func NewTailDataOp() stream.Operator {
name := "TailDropOp"
createWorker := func() mapper.Worker {
logger := log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)
fn := func(input stream.Object, outputer mapper.Outputer) error {
if value, ok := input.([]byte); ok {
logger.Printf("%s", string(value))
} else if value, ok := input.(string); ok {
logger.Printf("%s", string(value))
} else {
logger.Printf("%v", input)
}
outputer.Sending(1).Send(input)
return nil
}
return mapper.NewWorker(fn, name)
}
op := mapper.NewClosureOp(createWorker, nil, name)
op.Parallel = false
return op
}
func NewSnappyEncodeOp() stream.Operator {
name := "SnappyEncodeOp"
generator := func() mapper.Worker {
fn := func(obj stream.Object, out mapper.Outputer) error {
compressed := snappy.Encode(nil, obj.([]byte))
out.Sending(1).Send(compressed)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(generator, nil, name)
}
func NewProtobufDecodeOp(decFn func([]byte, func([]byte, proto.Message)) stream.Object) stream.InOutOperator {
name := "ProtobufDecodeOp"
workerCreator := func() mapper.Worker {
decoder := ProtobufGeneralDecoder()
fn := func(obj stream.Object, out mapper.Outputer) error {
decoded := decFn(obj.([]byte), decoder)
out.Sending(1).Send(decoded)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(workerCreator, nil, name)
}
func NewJsonDecodeOp(decFn func([]byte, func([]byte, interface{})) stream.Object) stream.InOutOperator {
name := "JsonDecodeOp"
workerCreator := func() mapper.Worker {
decoder := JsonGeneralDecoder()
fn := func(obj stream.Object, out mapper.Outputer) error {
decoded := decFn(obj.([]byte), decoder)
out.Sending(1).Send(decoded)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(workerCreator, nil, name)
}
func NewSnappyDecodeOp() stream.Operator {
name := "SnappyDecodeOp"
generator := func() mapper.Worker {
fn := func(obj stream.Object, out mapper.Outputer) error {
decompressed, err := snappy.Decode(nil, obj.([]byte))
if err != nil {
return fmt.Errorf("Error in snappy decompression %v", err)
}
out.Sending(1).Send(decompressed)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(generator, nil, name)
}
func NewJsonEncodeOp() stream.Operator {
name := "JsonEncodeOp"
workerCreator := func() mapper.Worker {
fn := func(obj stream.Object, out mapper.Outputer) error {
res, err := json.Marshal(obj.([]byte))
if err != nil {
return fmt.Errorf("Error marshaling json %v\t%+v", err, obj)
}
out.Sending(1).Send(res)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(workerCreator, nil, name)
}
func NewProtobufEncodeOp() stream.Operator {
name := "ProtobufEncodeOp"
workerCreator := func() mapper.Worker {
fn := func(obj stream.Object, outputer mapper.Outputer) error {
in := obj.(proto.Message)
out, err := proto.Marshal(in)
if err != nil {
return fmt.Errorf("Error marshaling protobuf %v\t%#v", err, in)
}
outputer.Sending(1).Send(out)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(workerCreator, nil, name)
}
func NewGobEncodeOp() stream.InOutOperator {
/* Each encoder provides a stateful stream. So we have to choices:
Either run this operator not in parallel and get a stateful stream
Or run this in parallel but use a new encoder for each input. We
choose the latter but plan to buffer upstream so we get big streams
coming out. We will compress each output separately here.
*/
name := "GobEncodeOp"
workerCreator := func() mapper.Worker {
var buf bytes.Buffer
fn := func(obj stream.Object, outputter mapper.Outputer) error {
enc := gob.NewEncoder(&buf) //each output is an indy stream
err := enc.Encode(obj)
if err != nil {
return fmt.Errorf("Error marshaling gob: %v\n", err.Error())
}
n := buf.Len()
out := make([]byte, n)
if out == nil {
log.Printf("Make failed")
}
newn, err := buf.Read(out)
if newn != n || err != nil {
if err == nil {
return fmt.Errorf("Error marshaling gob on read: %v\t%v\n", newn, n)
} else {
return fmt.Errorf("Error marshaling gob on read: %v\t%v\t%v\n", newn, n, err.Error())
}
}
outputter.Sending(1).Send(out)
return nil
}
return mapper.NewWorker(fn, name)
}
op := mapper.NewClosureOp(workerCreator, nil, name)
//op.Parallel = false
return op
}