Esempio n. 1
0
func NewSnappyEncodeOp() stream.Operator {
	generator := func() interface{} {
		fn := func(in []byte) [][]byte {
			compressed, err := snappy.Encode(nil, in)
			if err != nil {
				log.Printf("Error in snappy compression %v", err)
			}
			return [][]byte{compressed}
		}
		return fn
	}
	return mapper.NewOpFactory(generator, "NewSnappyEncodeOp")
}
Esempio n. 2
0
func NewJsonEncodeRop() stream.Operator {
	generator := func() interface{} {
		fn := func(in interface{}) [][]byte {
			out, err := json.Marshal(in)
			if err != nil {
				slog.Logf(logger.Levels.Error, "Error marshaling json %v\t%+v", err, in)
			}
			return [][]byte{out}
		}
		return fn
	}

	return mapper.NewOpFactory(generator, "NewJsonEncodeRop")
}
Esempio n. 3
0
func NewProtobufEncodeOp() stream.Operator {
	generator := func() interface{} {
		fn := func(obj stream.Object, outputer mapper.Outputer) {
			in := obj.(proto.Message)
			out, err := proto.Marshal(in)
			if err != nil {
				log.Printf("Error marshaling protobuf %v\t%#v", err, in)
			}
			outputer.Out(1) <- out
		}
		return fn
	}

	return mapper.NewOpFactory(generator, "NewProtobufEncodeOp")
}
Esempio n. 4
0
func NewTailDataOp() stream.Operator {
	gen := func() interface{} {

		logger := log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)

		return func(input stream.Object, outputer mapper.Outputer) {

			if value, ok := input.([]byte); ok {
				logger.Printf("%s", string(value))
			} else if value, ok := input.(string); ok {
				logger.Printf("%s", string(value))
			} else {
				logger.Printf("%v", input)
			}

			outputer.Out(1) <- input
		}
	}
	op := mapper.NewOpFactory(gen, "TailDataOp")
	op.Parallel = false
	return op
}
Esempio n. 5
0
func NewGobEncodeRop() stream.InOutOperator {
	/* Each encoder provides a stateful stream. So we have to choices:
	   Either run this operator not in parallel and get  a stateful stream
	   Or run this in parallel but use a new encoder for each input. We
	   choose the latter but plan to buffer upstream so we get big streams
	   coming out. We will compress each output separately here.
	*/

	generator := func() interface{} {
		var buf bytes.Buffer
		fn := func(in interface{}) [][]byte {
			enc := gob.NewEncoder(&buf) //each output is an indy stream
			err := enc.Encode(in)
			if err != nil {
				log.Printf("Error marshaling gob: %v\n", err.Error())
			}
			n := buf.Len()
			out := make([]byte, n)
			if out == nil {
				log.Printf("Make failed")
			}
			newn, err := buf.Read(out)
			if newn != n || err != nil {
				if err == nil {
					log.Printf("Error marshaling gob on read: %v\t%v\n", newn, n)
				} else {
					log.Printf("Error marshaling gob on read: %v\t%v\t%v\n", newn, n, err.Error())
				}
			}

			return [][]byte{out}
		}
		return fn
	}

	op := mapper.NewOpFactory(generator, "NewGobEncodeRop")
	//op.Parallel = false
	return op
}
Esempio n. 6
0
func NewJsonDecodeRop(gen interface{}) stream.Operator { //if outch is chan X, gen should be func() (func([]byte, chan<-bool) []X)
	return mapper.NewOpFactory(gen, "JsonDecodeRop")
}
Esempio n. 7
0
func NewGobDecodeRop(gen interface{}) stream.InOutOperator { //if outch is chan X, fun should be func([]byte, chan<-bool) []X
	op := mapper.NewOpFactory(gen, "GobDecodeRop")
	//op.Parallel = false
	return op
}