func (this *protobufInput) ReadTaskIds() (taskIds []int32) { // Read a single json record from the input file data, err := this.readData() if err != nil { panic(err) } if isTuple(data) { // Since we want to dispose of the data slice to reuse it for // reading, we create a new slice for buffering. Creating a // new slice here is ok, since we probably didn't create one // when we actually read the data. bufferedData := make([]byte, len(data)) copy(bufferedData, data) this.tupleBuffer.PushBack(bufferedData) return this.ReadTaskIds() } taskIdsProto := &messages.TaskIds{} err = proto.Unmarshal(data, taskIdsProto) if err != nil { panic(err) } this.bufferPool.Dispose(data) return taskIdsProto.TaskIds }
func (this *protobufInput) decodeInput(contentList [][]byte, contentStructs ...interface{}) { for i, content := range contentStructs { err := proto.Unmarshal(contentList[i], content.(proto.Message)) if err != nil { panic(err) } } }
func (this *hybridInput) decodeInput(contentList []interface{}, contentStructs ...interface{}) { for i, content := range contentStructs { err := proto.Unmarshal(*contentList[i].(*[]byte), content.(proto.Message)) if err != nil { panic(err) } } }
func TestEmitGeneric(t *testing.T) { buffer := new(bytes.Buffer) output := NewProtobufOutput(buffer) input := NewProtobufInput(buffer) needTaskIds := true for i := 0; i < 100; i++ { num := rand.Int63() numStr := fmt.Sprintf("%d", num) outMsg := newTestObj(numStr, num, []byte(numStr)) outMeta := &messages.ShellMsgMeta{ Command: "emit", Anchors: []string{"1", "2"}, Id: &numStr, Stream: &numStr, Task: &num, NeedTaskIds: &needTaskIds, Msg: &numStr, } output.EmitGeneric(outMeta.Command, outMeta.GetId(), outMeta.GetStream(), outMeta.GetMsg(), outMeta.GetAnchors(), outMeta.GetTask(), true, outMsg) output.Flush() shellMsg := &messages.ShellMsg{ ShellMsgProto: &messages.ShellMsgProto{}, } err := input.ReadMsg(shellMsg) checkErr(err, t) if !shellMsg.ShellMsgProto.ShellMsgMeta.Equal(outMeta) { t.Fatalf("Emission metadata (%+v) does not equal read emission metadata (%+v)", outMeta, shellMsg.ShellMsgProto.ShellMsgMeta) } inMsg := &messages.Test{} err = proto.Unmarshal(shellMsg.ShellMsgProto.Contents[0], inMsg) checkErr(err, t) if !inMsg.Equal(outMsg) { t.Fatalf("Emission data (%+v) does not equal read emission data (%+v)", outMsg, inMsg) } } }
// readBytes reads data from stdin into the struct provided. func (this *protobufInput) ReadMsg(msg interface{}) (err error) { var data []byte var bufferPoolRead bool if this.tupleBuffer.Len() > 0 { // Read data from the tuple buffer e := this.tupleBuffer.Front() data = this.tupleBuffer.Remove(e).([]byte) bufferPoolRead = false } else { // if the tuple buffer is empty, read data from storm data, err = this.readData() if err != nil { return err } bufferPoolRead = true } err = proto.Unmarshal(data, msg.(proto.Message)) if bufferPoolRead { // If the buffer pool was mmapped, we don't want to mix heap and mmapped data this.bufferPool.Dispose(data) } return err }