Пример #1
0
func (this *protobufInput) ReadTaskIds() (taskIds []int32) {
	// Read a single json record from the input file
	data, err := this.readData()
	if err != nil {
		panic(err)
	}

	if isTuple(data) {
		// Since we want to dispose of the data slice to reuse it for
		// reading, we create a new slice for buffering. Creating a
		// new slice here is ok, since we probably didn't create one
		// when we actually read the data.
		bufferedData := make([]byte, len(data))
		copy(bufferedData, data)
		this.tupleBuffer.PushBack(bufferedData)
		return this.ReadTaskIds()
	}

	taskIdsProto := &messages.TaskIds{}
	err = proto.Unmarshal(data, taskIdsProto)
	if err != nil {
		panic(err)
	}
	this.bufferPool.Dispose(data)
	return taskIdsProto.TaskIds
}
Пример #2
0
func (this *protobufInput) decodeInput(contentList [][]byte, contentStructs ...interface{}) {
	for i, content := range contentStructs {
		err := proto.Unmarshal(contentList[i], content.(proto.Message))
		if err != nil {
			panic(err)
		}
	}
}
Пример #3
0
func (this *hybridInput) decodeInput(contentList []interface{}, contentStructs ...interface{}) {
	for i, content := range contentStructs {
		err := proto.Unmarshal(*contentList[i].(*[]byte), content.(proto.Message))
		if err != nil {
			panic(err)
		}
	}
}
Пример #4
0
func TestEmitGeneric(t *testing.T) {
	buffer := new(bytes.Buffer)
	output := NewProtobufOutput(buffer)
	input := NewProtobufInput(buffer)

	needTaskIds := true

	for i := 0; i < 100; i++ {
		num := rand.Int63()
		numStr := fmt.Sprintf("%d", num)
		outMsg := newTestObj(numStr, num, []byte(numStr))
		outMeta := &messages.ShellMsgMeta{
			Command:     "emit",
			Anchors:     []string{"1", "2"},
			Id:          &numStr,
			Stream:      &numStr,
			Task:        &num,
			NeedTaskIds: &needTaskIds,
			Msg:         &numStr,
		}
		output.EmitGeneric(outMeta.Command, outMeta.GetId(), outMeta.GetStream(), outMeta.GetMsg(), outMeta.GetAnchors(), outMeta.GetTask(), true, outMsg)
		output.Flush()

		shellMsg := &messages.ShellMsg{
			ShellMsgProto: &messages.ShellMsgProto{},
		}
		err := input.ReadMsg(shellMsg)
		checkErr(err, t)

		if !shellMsg.ShellMsgProto.ShellMsgMeta.Equal(outMeta) {
			t.Fatalf("Emission metadata (%+v) does not equal read emission metadata (%+v)", outMeta, shellMsg.ShellMsgProto.ShellMsgMeta)
		}

		inMsg := &messages.Test{}
		err = proto.Unmarshal(shellMsg.ShellMsgProto.Contents[0], inMsg)
		checkErr(err, t)
		if !inMsg.Equal(outMsg) {
			t.Fatalf("Emission data (%+v) does not equal read emission data (%+v)", outMsg, inMsg)
		}
	}
}
Пример #5
0
// readBytes reads data from stdin into the struct provided.
func (this *protobufInput) ReadMsg(msg interface{}) (err error) {
	var data []byte
	var bufferPoolRead bool
	if this.tupleBuffer.Len() > 0 {
		// Read data from the tuple buffer
		e := this.tupleBuffer.Front()
		data = this.tupleBuffer.Remove(e).([]byte)
		bufferPoolRead = false
	} else {
		// if the tuple buffer is empty, read data from storm
		data, err = this.readData()
		if err != nil {
			return err
		}
		bufferPoolRead = true
	}

	err = proto.Unmarshal(data, msg.(proto.Message))
	if bufferPoolRead {
		// If the buffer pool was mmapped, we don't want to mix heap and mmapped data
		this.bufferPool.Dispose(data)
	}
	return err
}