/
disk.go
81 lines (73 loc) · 1.42 KB
/
disk.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
package main
import (
"bufio"
"os"
"time"
)
type diskBackend struct {
f *os.File
w *bufio.Writer
bulkRequest []*[]byte
ctx *Context
}
func NewDiskBackend(ctx *Context, filename string) *diskBackend {
f, err := os.Create(filename)
w := bufio.NewWriter(f)
if err != nil {
log.Debug(err)
}
return &diskBackend{f: f, w: w, ctx: ctx}
}
func (b *diskBackend) BulkAdd(line *[]byte) {
b.bulkRequest = append(b.bulkRequest, line)
}
func (b *diskBackend) Close() {
b.f.Close()
}
func (b *diskBackend) Flush() error {
for _, line := range b.bulkRequest {
b.w.Write(*line)
b.w.Write([]byte("\n"))
b.ctx.backupRate <- 1
}
err := b.w.Flush()
if err != nil {
return err
}
b.bulkRequest = b.bulkRequest[:0]
return nil
}
func (b *diskBackend) Size() int64 {
fi, _ := b.f.Stat()
return fi.Size()
}
func (b *diskBackend) Restore() {
filename := b.f.Name()
b.f.Close()
f, err := os.Open(filename)
if err != nil {
return
}
rd := bufio.NewReader(f)
defer f.Close()
log.Info("disk: restoring ", filename)
t0 := time.Now()
i := 0
for {
input, err := rd.ReadBytes('\n')
if err != nil {
f.Close()
log.Debug("disk: restored ", i, " removing ", filename)
os.Remove(filename)
b.ctx.restoreDone <- true
return
}
input2 := input[:len(input)-1]
b.ctx.parsedLines <- &input2
if time.Since(t0).Seconds() > 5 {
log.Info("disk: restoring ...")
t0 = time.Now()
}
i++
}
}