func TestCommandOp(t *testing.T) {
generator := newRecordedOpGenerator()
op := CommandOp{}
op.Database = "foo"
op.CommandName = "query"
metadata := bson.D{{"metadata", 1}}
op.Metadata = metadata
change := bson.D{{"updated", true}}
commandArgs := bson.D{{"$set", change}}
op.CommandArgs = commandArgs
inputDocs := []interface{}{}
for i := 0; i < 5; i++ {
inputDocs = append(inputDocs, &bson.D{{"inputDoc", 1}})
}
op.InputDocs = inputDocs
t.Logf("Generated CommandOp: %#v\n", op.CommandOp)
result, err := generator.fetchRecordedOpsFromConn(&op.CommandOp)
if err != nil {
t.Error(err)
}
receivedOp, err := result.RawOp.Parse()
if err != nil {
t.Error(err)
}
commandOp := receivedOp.(*CommandOp)
metadataAsBytes, _ := bson.Marshal(metadata)
metadataRaw := &bson.Raw{}
bson.Unmarshal(metadataAsBytes, metadataRaw)
commandArgsAsBytes, _ := bson.Marshal(commandArgs)
commandArgsRaw := &bson.Raw{}
bson.Unmarshal(commandArgsAsBytes, commandArgsRaw)
t.Log("Comparing parsed Command to original Command")
switch {
case commandOp.Database != op.Database:
t.Errorf("Databases not equal. Saw %v -- Expected %v\n", commandOp.Database, op.Database)
case commandOp.CommandName != op.CommandName:
t.Errorf("CommandNames not equal. Saw %v -- Expected %v\n", commandOp.CommandName, op.CommandName)
case !reflect.DeepEqual(commandOp.Metadata, metadataRaw):
t.Errorf("Metadata not equal. Saw %v -- Expected %v\n", commandOp.Metadata, metadataRaw)
case !reflect.DeepEqual(commandOp.CommandArgs, commandArgsRaw):
t.Errorf("CommandArgs not equal. Saw %v -- Expected %v\n", commandOp.CommandArgs, commandArgsRaw)
}
for i, doc := range commandOp.InputDocs {
marshaledAsBytes, _ := bson.Marshal(inputDocs[i])
unmarshaled := &bson.Raw{}
bson.Unmarshal(marshaledAsBytes, unmarshaled)
if !reflect.DeepEqual(unmarshaled, doc) {
t.Errorf("Document from InputDocs not matched. Saw %v -- Expected %v\n", unmarshaled, doc)
}
}
}
func TestRepeatGeneration(t *testing.T) {
recOp := &RecordedOp{
Seen: &PreciseTime{time.Now()},
}
bsonBytes, err := bson.Marshal(recOp)
if err != nil {
t.Errorf("couldn't marshal %v", err)
}
playbackReader := &PlaybackFileReader{bytes.NewReader(bsonBytes)}
repeat := 2
opChan, errChan := NewOpChanFromFile(playbackReader, repeat)
op1, ok := <-opChan
if !ok {
t.Errorf("read of 0-generation op failed")
}
if op1.Generation != 0 {
t.Errorf("generation of 0 generation op is %v", op1.Generation)
}
op2, ok := <-opChan
if !ok {
t.Errorf("read of 1-generation op failed")
}
if op2.Generation != 1 {
t.Errorf("generation of 1 generation op is %v", op2.Generation)
}
_, ok = <-opChan
if ok {
t.Errorf("Successfully read past end of op chan")
}
err = <-errChan
if err != io.EOF {
t.Errorf("should have eof at end, but got %v", err)
}
}
func TestOpCommandReplyGetCursorID(t *testing.T) {
testCursorID := int64(123)
doc := &struct {
Cursor struct {
ID int64 `bson:"id"`
} `bson:"cursor"`
}{}
doc.Cursor.ID = testCursorID
asByte, err := bson.Marshal(doc)
if err != nil {
t.Errorf("could not marshal bson: %v", err)
}
asRaw := &bson.Raw{}
bson.Unmarshal(asByte, asRaw)
commandReplyOp := &CommandReplyOp{}
commandReplyOp.CommandReply = asRaw
cursorID, err := commandReplyOp.getCursorID()
if err != nil {
t.Errorf("error fetching cursor %v", err)
}
if cursorID != testCursorID {
t.Errorf("cursorID did not match expected. Found: %v --- Expected: %v", cursorID, testCursorID)
}
t.Log("Ensuring cursorID consistent between multiple calls")
cursorID, err = commandReplyOp.getCursorID()
if err != nil {
t.Errorf("error fetching cursor %v", err)
}
if cursorID != testCursorID {
t.Errorf("cursorID did not match expected. Found: %v --- Expected: %v", cursorID, testCursorID)
}
}
func addBSON(b []byte, doc interface{}) ([]byte, error) {
if doc == nil {
return append(b, 5, 0, 0, 0, 0), nil
}
data, err := bson.Marshal(doc)
if err != nil {
return b, err
}
return append(b, data...), nil
}
// Record writes pcap data into a playback file
func Record(ctx *packetHandlerContext,
playbackWriter *PlaybackWriter,
noShortenReply bool) error {
ch := make(chan error)
go func() {
defer close(ch)
var fail error
for op := range ctx.mongoOpStream.Ops {
// since we don't currently have a way to shutdown packetHandler.Handle()
// continue to read from ctx.mongoOpStream.Ops even after a faltal error
if fail != nil {
toolDebugLogger.Logvf(DebugHigh, "not recording op because of record error %v", fail)
continue
}
if (op.Header.OpCode == OpCodeReply || op.Header.OpCode == OpCodeCommandReply) &&
!noShortenReply {
err := op.ShortenReply()
if err != nil {
userInfoLogger.Logvf(DebugLow, "stream %v problem shortening reply: %v", op.SeenConnectionNum, err)
continue
}
}
bsonBytes, err := bson.Marshal(op)
if err != nil {
userInfoLogger.Logvf(DebugLow, "stream %v error marshaling message: %v", op.SeenConnectionNum, err)
continue
}
_, err = playbackWriter.Write(bsonBytes)
if err != nil {
fail = fmt.Errorf("error writing message: %v", err)
userInfoLogger.Logvf(Always, "%v", err)
continue
}
}
ch <- fail
}()
if err := ctx.packetHandler.Handle(ctx.mongoOpStream, -1); err != nil {
return fmt.Errorf("record: error handling packet stream: %s", err)
}
stats, err := ctx.pcapHandle.Stats()
if err != nil {
toolDebugLogger.Logvf(Always, "Warning: got err %v getting pcap handle stats", err)
} else {
toolDebugLogger.Logvf(Info, "PCAP stats: %#v", stats)
}
err = <-ch
if err == nil && stats != nil && stats.PacketsDropped != 0 {
err = ErrPacketsDropped{stats.PacketsDropped}
}
return err
}
func objToDoc(obj interface{}) (d bson.D, err error) {
data, err := bson.Marshal(obj)
if err != nil {
return nil, err
}
err = bson.Unmarshal(data, &d)
if err != nil {
return nil, err
}
return d, err
}
// SetMeta changes the optional "metadata" field associated with the
// file. The meaning of keys under that field is user-defined.
// For example:
//
// file.SetMeta(bson.M{"inode": inode})
//
// It is a runtime error to call this function when the file is not open
// for writing.
func (file *GridFile) SetMeta(metadata interface{}) {
file.assertMode(gfsWriting)
data, err := bson.Marshal(metadata)
file.m.Lock()
if err != nil && file.err == nil {
file.err = err
} else {
file.doc.Metadata = &bson.Raw{Data: data}
}
file.m.Unlock()
}
func TestPlayOpEOF(t *testing.T) {
ops := []RecordedOp{{
Seen: &PreciseTime{time.Now()},
}, {
Seen: &PreciseTime{time.Now()},
EOF: true,
}}
var buf bytes.Buffer
for _, op := range ops {
bsonBytes, err := bson.Marshal(op)
if err != nil {
t.Errorf("couldn't marshal op %v", err)
}
buf.Write(bsonBytes)
}
playbackReader := &PlaybackFileReader{bytes.NewReader(buf.Bytes())}
repeat := 2
opChan, errChan := NewOpChanFromFile(playbackReader, repeat)
op1, ok := <-opChan
if !ok {
t.Errorf("read of op1 failed")
}
if op1.EOF {
t.Errorf("op1 should not be an EOF op")
}
op2, ok := <-opChan
if !ok {
t.Errorf("read op2 failed")
}
if op2.EOF {
t.Errorf("op2 should not be an EOF op")
}
op3, ok := <-opChan
if !ok {
t.Errorf("read of op3 failed")
}
if !op3.EOF {
t.Errorf("op3 is not an EOF op")
}
_, ok = <-opChan
if ok {
t.Errorf("Successfully read past end of op chan")
}
err := <-errChan
if err != io.EOF {
t.Errorf("should have eof at end, but got %v", err)
}
}
func TestLegacyOpReplyGetCursorID(t *testing.T) {
testCursorID := int64(123)
doc := &struct {
Cursor struct {
ID int64 `bson:"id"`
} `bson:"cursor"`
}{}
doc.Cursor.ID = testCursorID
asByte, err := bson.Marshal(doc)
if err != nil {
t.Errorf("could not marshal bson: %v", err)
}
asRaw := bson.Raw{}
bson.Unmarshal(asByte, &asRaw)
reply := &ReplyOp{}
reply.Docs = []bson.Raw{asRaw}
t.Log("Retrieving cursorID from reply docs")
cursorID, err := reply.getCursorID()
if err != nil {
t.Errorf("error fetching cursor %v", err)
}
if cursorID != testCursorID {
t.Errorf("cursorID did not match expected. Found: %v --- Expected: %v", cursorID, testCursorID)
}
t.Log("Ensuring cursorID consistent between multiple calls")
cursorID, err = reply.getCursorID()
if err != nil {
t.Errorf("error fetching cursor %v", err)
}
if cursorID != testCursorID {
t.Errorf("cursorID did not match expected. Found: %v --- Expected: %v", cursorID, testCursorID)
}
reply2 := &ReplyOp{}
reply2.CursorId = testCursorID
t.Log("Retrieving cursorID from reply field")
cursorID, err = reply.getCursorID()
if err != nil {
t.Errorf("error fetching cursor %v", err)
}
if cursorID != testCursorID {
t.Errorf("cursorID did not match expected. Found: %v --- Expected: %v", cursorID, testCursorID)
}
}
func TestPreciseTimeMarshal(t *testing.T) {
t1 := time.Date(2015, 4, 8, 15, 16, 23, 651387237, time.UTC)
preciseTime := &PreciseTime{t1}
asBson, err := bson.Marshal(preciseTime)
if err != nil {
t.Error(err)
}
result := &PreciseTime{}
err = bson.Unmarshal(asBson, result)
if err != nil {
t.Error(err)
}
if t1 != result.Time {
t.Errorf("Times not equal. Input: %v -- Result: %v", t1, result.Time)
}
}
func TestInsertOp(t *testing.T) {
generator := newRecordedOpGenerator()
op := InsertOp{}
op.Collection = "mongoreplay_test.test"
op.Flags = 7
documents := []interface{}(nil)
for i := 0; i < 10; i++ {
insertDoc := &testDoc{
DocumentNumber: i,
Success: true,
}
documents = append(documents, insertDoc)
}
op.Documents = documents
t.Logf("Generated Insert: %#v\n", op.InsertOp)
result, err := generator.fetchRecordedOpsFromConn(&op.InsertOp)
if err != nil {
t.Error(err)
}
receivedOp, err := result.RawOp.Parse()
if err != nil {
t.Error(err)
}
insertOp := receivedOp.(*InsertOp)
t.Log("Comparing parsed Insert to original Insert")
switch {
case insertOp.Collection != "mongoreplay_test.test":
t.Errorf("Collection not matched. Saw %v -- Expected %v\n", insertOp.Collection, "mongoreplay_test.test")
case insertOp.Flags != 7:
t.Errorf("Flags not matched. Saw %v -- Expected %v\n", insertOp.Flags, 7)
}
for i, doc := range insertOp.Documents {
marshaled, _ := bson.Marshal(documents[i])
unmarshaled := &bson.D{}
bson.Unmarshal(marshaled, unmarshaled)
if !reflect.DeepEqual(unmarshaled, doc) {
t.Errorf("Document not matched. Saw %v -- Expected %v\n", unmarshaled, doc)
}
}
}
// Record writes pcap data into a playback file
func Record(ctx *packetHandlerContext,
playbackWriter *PlaybackWriter,
noShortenReply bool) error {
ch := make(chan error)
go func() {
defer close(ch)
for op := range ctx.mongoOpStream.Ops {
if (op.Header.OpCode == OpCodeReply || op.Header.OpCode == OpCodeCommandReply) &&
!noShortenReply {
op.ShortenReply()
}
bsonBytes, err := bson.Marshal(op)
if err != nil {
ch <- fmt.Errorf("error marshaling message: %v", err)
return
}
_, err = playbackWriter.Write(bsonBytes)
if err != nil {
ch <- fmt.Errorf("error writing message: %v", err)
return
}
}
ch <- nil
}()
if err := ctx.packetHandler.Handle(ctx.mongoOpStream, -1); err != nil {
return fmt.Errorf("record: error handling packet stream: %s", err)
}
stats, err := ctx.pcapHandle.Stats()
if err != nil {
toolDebugLogger.Logvf(Always, "Warning: got err %v getting pcap handle stats", err)
} else {
toolDebugLogger.Logvf(Info, "PCAP stats: %#v", stats)
}
err = <-ch
if err == nil && stats != nil && stats.PacketsDropped != 0 {
err = ErrPacketsDropped{stats.PacketsDropped}
}
return err
}
func (file *GridFile) insertChunk(data []byte) {
n := file.chunk
file.chunk++
debugf("GridFile %p: adding to checksum: %q", file, string(data))
file.wsum.Write(data)
for file.doc.ChunkSize*file.wpending >= 1024*1024 {
// Hold on.. we got a MB pending.
file.c.Wait()
if file.err != nil {
return
}
}
file.wpending++
debugf("GridFile %p: inserting chunk %d with %d bytes", file, n, len(data))
// We may not own the memory of data, so rather than
// simply copying it, we'll marshal the document ahead of time.
data, err := bson.Marshal(gfsChunk{bson.NewObjectId(), file.doc.Id, n, data})
if err != nil {
file.err = err
return
}
go func() {
err := file.gfs.Chunks.Insert(bson.Raw{Data: data})
file.m.Lock()
file.wpending--
if err != nil && file.err == nil {
file.err = err
}
file.c.Broadcast()
file.m.Unlock()
}()
}
func TestShortenLegacyReply(t *testing.T) {
generator := newRecordedOpGenerator()
op := ReplyOp{}
op.ReplyDocs = 2
result, err := generator.fetchRecordedOpsFromConn(&op.ReplyOp)
doc1 := &testDoc{
Name: "Op Raw Short Reply Test 1",
DocumentNumber: 1,
Success: true,
}
doc2 := &testDoc{
Name: "Op Raw Short Reply Test 2",
DocumentNumber: 2,
Success: true,
}
asByte1, err := bson.Marshal(doc1)
if err != nil {
t.Errorf("could not marshal bson: %v", err)
}
asByte2, err := bson.Marshal(doc2)
if err != nil {
t.Errorf("could not marshal bson: %v", err)
}
// add the two docs as the docs from the reply
result.RawOp.Body = append(result.RawOp.Body, asByte1...)
result.RawOp.Body = append(result.RawOp.Body, asByte2...)
result.Header.MessageLength = int32(len(result.RawOp.Body))
// reply should be functional and parseable
parsed, err := result.RawOp.Parse()
if err != nil {
t.Errorf("error parsing op: %v", err)
}
fullReply, ok := parsed.(*ReplyOp)
if !ok {
t.Errorf("parsed op was wrong type")
}
if !(len(fullReply.Docs) == 2) {
t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.Docs))
}
// shorten the reply
result.ShortenReply()
parsed, err = result.RawOp.Parse()
if err != nil {
t.Errorf("error parsing op: %v", err)
}
fullReply, ok = parsed.(*ReplyOp)
if !ok {
t.Errorf("parsed op was wrong type")
}
// ensure that the reply now has only 1 document
if !(len(fullReply.Docs) == 1) {
t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.Docs))
}
}
func TestCommandOpGetMoreCursorsRewriteable(t *testing.T) {
oldCursorID := int64(1234)
newCursorID := int64(5678)
commandGM := &CommandGetMore{
CommandOp: CommandOp{},
}
doc := &bson.D{{"getMore", oldCursorID}}
asByte, err := bson.Marshal(doc)
if err != nil {
t.Errorf("could not marshal bson: %v", err)
}
asRaw := &bson.Raw{}
bson.Unmarshal(asByte, asRaw)
commandGM.CommandOp.CommandArgs = asRaw
t.Log("fetching getmore cursorID")
cursorIDs, err := commandGM.getCursorIDs()
if err != nil {
t.Errorf("error fetching cursorIDs: %v", err)
}
if len(cursorIDs) != 1 {
t.Errorf("differing number of cursorIDs found in commandlgetmore. Expected: %v --- Found: %v", 1, len(cursorIDs))
} else {
if oldCursorID != cursorIDs[0] {
t.Errorf("cursorIDs not matched when retrieved. Expected: %v --- Found: %v", oldCursorID, cursorIDs[0])
}
}
t.Log("setting getmore cursorID")
err = commandGM.setCursorIDs([]int64{newCursorID})
if err != nil {
t.Errorf("error setting cursorIDs: %v", err)
}
t.Log("fetching new getmore cursorID")
cursorIDs, err = commandGM.getCursorIDs()
if err != nil {
t.Errorf("error fetching cursorIDs: %v", err)
}
if len(cursorIDs) != 1 {
t.Errorf("differing number of cursorIDs found in killcursors. Expected: %v --- Found: %v", 1, len(cursorIDs))
} else {
if newCursorID != cursorIDs[0] {
t.Errorf("cursorIDs not matched when retrieved. Expected: %v --- Found: %v", newCursorID, cursorIDs[0])
}
}
commandArgs, ok := commandGM.CommandOp.CommandArgs.(*bson.D)
if !ok {
t.Errorf("commandArgs not a *bson.D")
} else {
for _, bsonDoc := range *commandArgs {
if bsonDoc.Name == "getMore" {
getmoreID, ok := bsonDoc.Value.(int64)
if !ok {
t.Errorf("cursorID in command is not int64")
}
if newCursorID != getmoreID {
t.Errorf("cursorIDs not matched when retrieved. Expected: %v --- Found: %v", newCursorID, getmoreID)
}
break
}
}
}
}
// ShortReplyFromReader reads an op from the given reader. It only holds on
// to header-related information and the first document.
func (op *RawOp) ShortenReply() error {
if op.Header.MessageLength < MsgHeaderLen {
return fmt.Errorf("expected message header to have length: %d bytes but was %d bytes", MsgHeaderLen, op.Header.MessageLength)
}
if op.Header.MessageLength > MaxMessageSize {
return fmt.Errorf("wire message size, %v, was greater then the maximum, %v bytes", op.Header.MessageLength, MaxMessageSize)
}
switch op.Header.OpCode {
case OpCodeReply:
if op.Header.MessageLength <= 20+MsgHeaderLen {
//there are no reply docs
return nil
}
firstDocSize := getInt32(op.Body, 20+MsgHeaderLen)
if 20+MsgHeaderLen+int(firstDocSize) > len(op.Body) || firstDocSize > maxBSONSize {
return fmt.Errorf("the size of the first document is greater then the size of the message")
}
op.Body = op.Body[0:(20 + MsgHeaderLen + firstDocSize)]
case OpCodeCommandReply:
// unmarshal the needed fields for replacing into the buffer
commandReply := &CommandReplyStruct{}
err := bson.Unmarshal(op.Body[MsgHeaderLen:], commandReply)
if err != nil {
return fmt.Errorf("unmarshaling op to shorten: %v", err)
}
switch {
case commandReply.Cursor.FirstBatch.Data != nil:
commandReply.Cursor.FirstBatch.Data, _ = bson.Marshal([0]byte{})
case commandReply.Cursor.NextBatch.Data != nil:
commandReply.Cursor.NextBatch.Data, _ = bson.Marshal([0]byte{})
default:
// it's not a findReply so we don't care about it
return nil
}
out, err := bson.Marshal(commandReply)
if err != nil {
return err
}
// calculate the new sizes for offsets into the new buffer
commandReplySize := getInt32(op.Body, MsgHeaderLen)
newCommandReplySize := getInt32(out, 0)
sizeDiff := commandReplySize - newCommandReplySize
newSize := op.Header.MessageLength - sizeDiff
newBody := make([]byte, newSize)
// copy the new data into a buffer that will replace the old buffer
copy(newBody, op.Body[:MsgHeaderLen])
copy(newBody[MsgHeaderLen:], out)
copy(newBody[MsgHeaderLen+newCommandReplySize:], op.Body[MsgHeaderLen+commandReplySize:])
// update the size of this message in the headers
SetInt32(newBody, 0, newSize)
op.Header.MessageLength = newSize
op.Body = newBody
default:
return fmt.Errorf("unexpected op type : %v", op.Header.OpCode)
}
return nil
}
