func NewSnappyEncodeOp() stream.Operator { generator := func() interface{} { fn := func(in []byte) [][]byte { compressed, err := snappy.Encode(nil, in) if err != nil { log.Printf("Error in snappy compression %v", err) } return [][]byte{compressed} } return fn } return mapper.NewOpFactory(generator, "NewSnappyEncodeOp") }
func NewJsonEncodeRop() stream.Operator { generator := func() interface{} { fn := func(in interface{}) [][]byte { out, err := json.Marshal(in) if err != nil { slog.Logf(logger.Levels.Error, "Error marshaling json %v\t%+v", err, in) } return [][]byte{out} } return fn } return mapper.NewOpFactory(generator, "NewJsonEncodeRop") }
func NewProtobufEncodeOp() stream.Operator { generator := func() interface{} { fn := func(obj stream.Object, outputer mapper.Outputer) { in := obj.(proto.Message) out, err := proto.Marshal(in) if err != nil { log.Printf("Error marshaling protobuf %v\t%#v", err, in) } outputer.Out(1) <- out } return fn } return mapper.NewOpFactory(generator, "NewProtobufEncodeOp") }
func NewTailDataOp() stream.Operator { gen := func() interface{} { logger := log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile) return func(input stream.Object, outputer mapper.Outputer) { if value, ok := input.([]byte); ok { logger.Printf("%s", string(value)) } else if value, ok := input.(string); ok { logger.Printf("%s", string(value)) } else { logger.Printf("%v", input) } outputer.Out(1) <- input } } op := mapper.NewOpFactory(gen, "TailDataOp") op.Parallel = false return op }
func NewGobEncodeRop() stream.InOutOperator { /* Each encoder provides a stateful stream. So we have to choices: Either run this operator not in parallel and get a stateful stream Or run this in parallel but use a new encoder for each input. We choose the latter but plan to buffer upstream so we get big streams coming out. We will compress each output separately here. */ generator := func() interface{} { var buf bytes.Buffer fn := func(in interface{}) [][]byte { enc := gob.NewEncoder(&buf) //each output is an indy stream err := enc.Encode(in) if err != nil { log.Printf("Error marshaling gob: %v\n", err.Error()) } n := buf.Len() out := make([]byte, n) if out == nil { log.Printf("Make failed") } newn, err := buf.Read(out) if newn != n || err != nil { if err == nil { log.Printf("Error marshaling gob on read: %v\t%v\n", newn, n) } else { log.Printf("Error marshaling gob on read: %v\t%v\t%v\n", newn, n, err.Error()) } } return [][]byte{out} } return fn } op := mapper.NewOpFactory(generator, "NewGobEncodeRop") //op.Parallel = false return op }
func NewJsonDecodeRop(gen interface{}) stream.Operator { //if outch is chan X, gen should be func() (func([]byte, chan<-bool) []X) return mapper.NewOpFactory(gen, "JsonDecodeRop") }
func NewGobDecodeRop(gen interface{}) stream.InOutOperator { //if outch is chan X, fun should be func([]byte, chan<-bool) []X op := mapper.NewOpFactory(gen, "GobDecodeRop") //op.Parallel = false return op }