func NewSnappyEncodeOp() stream.Operator {
generator := func() interface{} {
fn := func(in []byte) [][]byte {
compressed, err := snappy.Encode(nil, in)
if err != nil {
log.Printf("Error in snappy compression %v", err)
}
return [][]byte{compressed}
}
return fn
}
return mapper.NewOpFactory(generator, "NewSnappyEncodeOp")
}
func NewJsonEncodeRop() stream.Operator {
generator := func() interface{} {
fn := func(in interface{}) [][]byte {
out, err := json.Marshal(in)
if err != nil {
slog.Logf(logger.Levels.Error, "Error marshaling json %v\t%+v", err, in)
}
return [][]byte{out}
}
return fn
}
return mapper.NewOpFactory(generator, "NewJsonEncodeRop")
}
func NewProtobufEncodeOp() stream.Operator {
generator := func() interface{} {
fn := func(obj stream.Object, outputer mapper.Outputer) {
in := obj.(proto.Message)
out, err := proto.Marshal(in)
if err != nil {
log.Printf("Error marshaling protobuf %v\t%#v", err, in)
}
outputer.Out(1) <- out
}
return fn
}
return mapper.NewOpFactory(generator, "NewProtobufEncodeOp")
}
func NewTailDataOp() stream.Operator {
gen := func() interface{} {
logger := log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)
return func(input stream.Object, outputer mapper.Outputer) {
if value, ok := input.([]byte); ok {
logger.Printf("%s", string(value))
} else if value, ok := input.(string); ok {
logger.Printf("%s", string(value))
} else {
logger.Printf("%v", input)
}
outputer.Out(1) <- input
}
}
op := mapper.NewOpFactory(gen, "TailDataOp")
op.Parallel = false
return op
}
func NewGobEncodeRop() stream.InOutOperator {
/* Each encoder provides a stateful stream. So we have to choices:
Either run this operator not in parallel and get a stateful stream
Or run this in parallel but use a new encoder for each input. We
choose the latter but plan to buffer upstream so we get big streams
coming out. We will compress each output separately here.
*/
generator := func() interface{} {
var buf bytes.Buffer
fn := func(in interface{}) [][]byte {
enc := gob.NewEncoder(&buf) //each output is an indy stream
err := enc.Encode(in)
if err != nil {
log.Printf("Error marshaling gob: %v\n", err.Error())
}
n := buf.Len()
out := make([]byte, n)
if out == nil {
log.Printf("Make failed")
}
newn, err := buf.Read(out)
if newn != n || err != nil {
if err == nil {
log.Printf("Error marshaling gob on read: %v\t%v\n", newn, n)
} else {
log.Printf("Error marshaling gob on read: %v\t%v\t%v\n", newn, n, err.Error())
}
}
return [][]byte{out}
}
return fn
}
op := mapper.NewOpFactory(generator, "NewGobEncodeRop")
//op.Parallel = false
return op
}
func NewJsonDecodeRop(gen interface{}) stream.Operator { //if outch is chan X, gen should be func() (func([]byte, chan<-bool) []X)
return mapper.NewOpFactory(gen, "JsonDecodeRop")
}
func NewGobDecodeRop(gen interface{}) stream.InOutOperator { //if outch is chan X, fun should be func([]byte, chan<-bool) []X
op := mapper.NewOpFactory(gen, "GobDecodeRop")
//op.Parallel = false
return op
}