Beispiel #1
0
func NewTailDataOp() stream.Operator {
	name := "TailDropOp"
	createWorker := func() mapper.Worker {

		logger := log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)

		fn := func(input stream.Object, outputer mapper.Outputer) error {

			if value, ok := input.([]byte); ok {
				logger.Printf("%s", string(value))
			} else if value, ok := input.(string); ok {
				logger.Printf("%s", string(value))
			} else {
				logger.Printf("%v", input)
			}

			outputer.Sending(1).Send(input)
			return nil
		}

		return mapper.NewWorker(fn, name)
	}
	op := mapper.NewClosureOp(createWorker, nil, name)
	op.Parallel = false
	return op
}
Beispiel #2
0
func NewSnappyEncodeOp() stream.Operator {
	name := "SnappyEncodeOp"
	generator := func() mapper.Worker {
		fn := func(obj stream.Object, out mapper.Outputer) error {
			compressed := snappy.Encode(nil, obj.([]byte))
			out.Sending(1).Send(compressed)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}
	return mapper.NewClosureOp(generator, nil, name)
}
Beispiel #3
0
func NewProtobufDecodeOp(decFn func([]byte, func([]byte, proto.Message)) stream.Object) stream.InOutOperator {
	name := "ProtobufDecodeOp"
	workerCreator := func() mapper.Worker {
		decoder := ProtobufGeneralDecoder()
		fn := func(obj stream.Object, out mapper.Outputer) error {
			decoded := decFn(obj.([]byte), decoder)
			out.Sending(1).Send(decoded)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}
	return mapper.NewClosureOp(workerCreator, nil, name)
}
Beispiel #4
0
func NewJsonDecodeOp(decFn func([]byte, func([]byte, interface{})) stream.Object) stream.InOutOperator {
	name := "JsonDecodeOp"
	workerCreator := func() mapper.Worker {
		decoder := JsonGeneralDecoder()
		fn := func(obj stream.Object, out mapper.Outputer) error {
			decoded := decFn(obj.([]byte), decoder)
			out.Sending(1).Send(decoded)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}
	return mapper.NewClosureOp(workerCreator, nil, name)
}
Beispiel #5
0
func NewSnappyDecodeOp() stream.Operator {
	name := "SnappyDecodeOp"
	generator := func() mapper.Worker {
		fn := func(obj stream.Object, out mapper.Outputer) error {
			decompressed, err := snappy.Decode(nil, obj.([]byte))
			if err != nil {
				return fmt.Errorf("Error in snappy decompression %v", err)
			}
			out.Sending(1).Send(decompressed)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}
	return mapper.NewClosureOp(generator, nil, name)
}
Beispiel #6
0
func NewJsonEncodeOp() stream.Operator {
	name := "JsonEncodeOp"
	workerCreator := func() mapper.Worker {
		fn := func(obj stream.Object, out mapper.Outputer) error {
			res, err := json.Marshal(obj.([]byte))
			if err != nil {
				return fmt.Errorf("Error marshaling json %v\t%+v", err, obj)
			}
			out.Sending(1).Send(res)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}

	return mapper.NewClosureOp(workerCreator, nil, name)
}
Beispiel #7
0
func NewProtobufEncodeOp() stream.Operator {
	name := "ProtobufEncodeOp"
	workerCreator := func() mapper.Worker {
		fn := func(obj stream.Object, outputer mapper.Outputer) error {
			in := obj.(proto.Message)
			out, err := proto.Marshal(in)
			if err != nil {
				return fmt.Errorf("Error marshaling protobuf %v\t%#v", err, in)
			}
			outputer.Sending(1).Send(out)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}

	return mapper.NewClosureOp(workerCreator, nil, name)
}
Beispiel #8
0
func NewGobEncodeOp() stream.InOutOperator {
	/* Each encoder provides a stateful stream. So we have to choices:
	   Either run this operator not in parallel and get  a stateful stream
	   Or run this in parallel but use a new encoder for each input. We
	   choose the latter but plan to buffer upstream so we get big streams
	   coming out. We will compress each output separately here.
	*/

	name := "GobEncodeOp"
	workerCreator := func() mapper.Worker {
		var buf bytes.Buffer
		fn := func(obj stream.Object, outputter mapper.Outputer) error {
			enc := gob.NewEncoder(&buf) //each output is an indy stream
			err := enc.Encode(obj)
			if err != nil {
				return fmt.Errorf("Error marshaling gob: %v\n", err.Error())
			}
			n := buf.Len()
			out := make([]byte, n)
			if out == nil {
				log.Printf("Make failed")
			}
			newn, err := buf.Read(out)
			if newn != n || err != nil {
				if err == nil {
					return fmt.Errorf("Error marshaling gob on read: %v\t%v\n", newn, n)
				} else {
					return fmt.Errorf("Error marshaling gob on read: %v\t%v\t%v\n", newn, n, err.Error())
				}
			}

			outputter.Sending(1).Send(out)
			return nil
		}
		return mapper.NewWorker(fn, name)
	}

	op := mapper.NewClosureOp(workerCreator, nil, name)
	//op.Parallel = false
	return op
}