/
dedup_test.go
65 lines (54 loc) · 1.5 KB
/
dedup_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
package tm
import (
"fmt"
cv "github.com/glycerine/goconvey/convey"
"os"
"testing"
)
func Test030DedupStreams(t *testing.T) {
cv.Convey(`given a stream with some duplicate frames repeating previously seen frames, Dedup() filter out duplicates within the given window`, t, func() {
// setup the test streams, with duplicated Frames
nFrame := 100
expectedPath := "test.dedup.expected"
frames, _, _ := GenTestFrames(nFrame, &expectedPath)
// prep to write out duplicates
dupsPath := "test.dups"
writeMe, err := os.Create(dupsPath)
panicOn(err)
ds := NewFrameWriter(writeMe, 64*1024)
// generate duplicates, randomly
fold := 4
tot := nFrame * fold
k := 0
// first be sure we've seen all our frames, so we
// aren't jumping completely backwards before the first frame.
for i := 0; i < nFrame; i++ {
ds.Append(frames[i])
k++
}
for i := 1; i <= tot; i++ {
fr := frames[cryptoRandNonNegInt()%((i%(nFrame-1))+1)]
ds.Append(fr)
k++
}
ds.Flush()
ds.Sync()
writeMe.Seek(0, 0)
hasDupsFd := writeMe
obsPath := "test.dedup.obs"
dedupFd, err := os.Create(obsPath)
// dedup the stream
// all of it, so we give the full total as the window
p("k = %v", k)
err = Dedup(hasDupsFd, dedupFd, k+1, nil, false)
cv.So(err, cv.ShouldEqual, nil)
// check
diff := FilesDiff(expectedPath, obsPath)
if diff {
panic(fmt.Errorf("dedup failed, '%s' != '%s'", obsPath, expectedPath))
}
cv.So(diff, cv.ShouldBeFalse)
os.Remove(obsPath)
os.Remove(expectedPath)
})
}