func (this *protobufInput) ReadTaskIds() (taskIds []int32) {
// Read a single json record from the input file
data, err := this.readData()
if err != nil {
panic(err)
}
if isTuple(data) {
// Since we want to dispose of the data slice to reuse it for
// reading, we create a new slice for buffering. Creating a
// new slice here is ok, since we probably didn't create one
// when we actually read the data.
bufferedData := make([]byte, len(data))
copy(bufferedData, data)
this.tupleBuffer.PushBack(bufferedData)
return this.ReadTaskIds()
}
taskIdsProto := &messages.TaskIds{}
err = proto.Unmarshal(data, taskIdsProto)
if err != nil {
panic(err)
}
this.bufferPool.Dispose(data)
return taskIdsProto.TaskIds
}
func (this *protobufInput) decodeInput(contentList [][]byte, contentStructs ...interface{}) {
for i, content := range contentStructs {
err := proto.Unmarshal(contentList[i], content.(proto.Message))
if err != nil {
panic(err)
}
}
}
func (this *hybridInput) decodeInput(contentList []interface{}, contentStructs ...interface{}) {
for i, content := range contentStructs {
err := proto.Unmarshal(*contentList[i].(*[]byte), content.(proto.Message))
if err != nil {
panic(err)
}
}
}
func TestEmitGeneric(t *testing.T) {
buffer := new(bytes.Buffer)
output := NewProtobufOutput(buffer)
input := NewProtobufInput(buffer)
needTaskIds := true
for i := 0; i < 100; i++ {
num := rand.Int63()
numStr := fmt.Sprintf("%d", num)
outMsg := newTestObj(numStr, num, []byte(numStr))
outMeta := &messages.ShellMsgMeta{
Command: "emit",
Anchors: []string{"1", "2"},
Id: &numStr,
Stream: &numStr,
Task: &num,
NeedTaskIds: &needTaskIds,
Msg: &numStr,
}
output.EmitGeneric(outMeta.Command, outMeta.GetId(), outMeta.GetStream(), outMeta.GetMsg(), outMeta.GetAnchors(), outMeta.GetTask(), true, outMsg)
output.Flush()
shellMsg := &messages.ShellMsg{
ShellMsgProto: &messages.ShellMsgProto{},
}
err := input.ReadMsg(shellMsg)
checkErr(err, t)
if !shellMsg.ShellMsgProto.ShellMsgMeta.Equal(outMeta) {
t.Fatalf("Emission metadata (%+v) does not equal read emission metadata (%+v)", outMeta, shellMsg.ShellMsgProto.ShellMsgMeta)
}
inMsg := &messages.Test{}
err = proto.Unmarshal(shellMsg.ShellMsgProto.Contents[0], inMsg)
checkErr(err, t)
if !inMsg.Equal(outMsg) {
t.Fatalf("Emission data (%+v) does not equal read emission data (%+v)", outMsg, inMsg)
}
}
}
// readBytes reads data from stdin into the struct provided.
func (this *protobufInput) ReadMsg(msg interface{}) (err error) {
var data []byte
var bufferPoolRead bool
if this.tupleBuffer.Len() > 0 {
// Read data from the tuple buffer
e := this.tupleBuffer.Front()
data = this.tupleBuffer.Remove(e).([]byte)
bufferPoolRead = false
} else {
// if the tuple buffer is empty, read data from storm
data, err = this.readData()
if err != nil {
return err
}
bufferPoolRead = true
}
err = proto.Unmarshal(data, msg.(proto.Message))
if bufferPoolRead {
// If the buffer pool was mmapped, we don't want to mix heap and mmapped data
this.bufferPool.Dispose(data)
}
return err
}