/
filter.go
108 lines (77 loc) · 2.51 KB
/
filter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package blockcompressor
import (
"github.com/monkeybutter/snappy"
"sync"
)
func shuff(shuff_buf, block *Block, in, out DuplexPipe, wg *sync.WaitGroup) {
defer wg.Done()
if block.NBytes == in.BlockSize {
shuff_unit := in.BlockSize / in.TypeSize
for i := 0; i < in.TypeSize; i++ {
shuff_offset := i * shuff_unit
for j := 0; j < shuff_unit; j++ {
shuff_buf.Buf[shuff_offset+j] = block.Buf[j*in.TypeSize+i]
}
}
shuff_buf.NBytes = block.NBytes
shuff_buf.BlockID = block.BlockID
} else {
shuff_buf.NBytes = block.NBytes
shuff_buf.BlockID = block.BlockID
copy(shuff_buf.Buf[:shuff_buf.NBytes], block.Buf)
}
in.Upstream <- block
out.Downstream <- shuff_buf
}
func comp(comp_buf, block *Block, in, out DuplexPipe, wg *sync.WaitGroup) {
defer wg.Done()
if block.NBytes == in.BlockSize {
// We are allocating comp_chunk extra here to know length
// ! Fork snappy to return len(comp_buf.Buf) instead of
// the the actual slice of comp_buf.Buf
_, comp_buf.NBytes = snappy.Encode(comp_buf.Buf, block.Buf)
//fmt.Println("Compressing block", block.BlockID, block_size, len(comp_chunk))
// this misses the point of having reusable slices... :-(
//comp_buf.NBytes = len(comp_chunk)
comp_buf.BlockID = block.BlockID
} else {
comp_buf.NBytes = block.NBytes
comp_buf.BlockID = block.BlockID
copy(comp_buf.Buf[:comp_buf.NBytes], block.Buf)
}
in.Upstream <- block
out.Downstream <- comp_buf
}
func decomp(decomp_buf, block *Block, in, out DuplexPipe, wg *sync.WaitGroup) {
defer wg.Done()
nBlocks := (in.FileSize / int64(in.BlockSize)) + 1
if block.BlockID < int(nBlocks) - 1 {
snappy.Decode(decomp_buf.Buf, block.Buf[:block.NBytes])
decomp_buf.BlockID = block.BlockID
} else {
decomp_buf.NBytes = block.NBytes
decomp_buf.BlockID = block.BlockID
copy(decomp_buf.Buf[:decomp_buf.NBytes], block.Buf)
}
in.Upstream <- block
out.Downstream <- decomp_buf
}
func unshuff(unshuff_buf, block *Block, in, out DuplexPipe, wg *sync.WaitGroup) {
defer wg.Done()
nBlocks := (in.FileSize / int64(in.BlockSize)) + 1
if block.BlockID < int(nBlocks) - 1 {
for j := 0; j < in.TypeSize; j++ {
for k := 0; k < in.BlockSize/64; k++ {
unshuff_buf.Buf[k*in.TypeSize+j] = block.Buf[k+(j*in.BlockSize/in.TypeSize)]
}
}
unshuff_buf.NBytes = block.NBytes
unshuff_buf.BlockID = block.BlockID
} else {
unshuff_buf.NBytes = block.NBytes
unshuff_buf.BlockID = block.BlockID
copy(unshuff_buf.Buf[:unshuff_buf.NBytes], block.Buf)
}
in.Upstream <- block
out.Downstream <- unshuff_buf
}