func comp(block *Block, out chan *Block, block_size int, wg *sync.WaitGroup) { defer wg.Done() comp_len := snappy.MaxEncodedLen(block_size) comp_block := &Block{make([]byte, comp_len), 0, 0} if block.NBytes == block_size { comp_block.Buf, comp_block.NBytes = snappy.Encode(comp_block.Buf, block.Buf) comp_block.BlockID = block.BlockID } else { comp_block.NBytes = block.NBytes comp_block.BlockID = block.BlockID copy(comp_block.Buf[:comp_block.NBytes], block.Buf) } out <- comp_block }
func block_decomp(in DuplexPipe, block_size, conc_level int) DuplexPipe { // This is the output of the generator out := DuplexPipe{make(chan *Block, conc_level), make(chan *Block, conc_level)} comp_len := snappy.MaxEncodedLen(block_size) for i := 0; i < conc_level; i++ { out.Upstream <- &Block{make([]byte, comp_len), 0, 0} } var wg sync.WaitGroup go func() { for block := range in.Downstream { wg.Add(1) go comp(<-out.Upstream, block, in, out, block_size, &wg) } wg.Wait() close(out.Downstream) }() return out }