// get an id of an existing file, for _id access
func idOfFile(mf *MongoFiles, filename string) string {
session, _ := mf.SessionProvider.GetSession()
gfs := session.DB(mf.StorageOptions.DB).GridFS(mf.StorageOptions.GridFSPrefix)
gFile, _ := gfs.Open(filename)
bytes, _ := json.Marshal(gFile.Id())
return fmt.Sprintf("ObjectId(%v)", string(bytes))
}
// ExportDocument writes a line to output with the CSV representation of a document.
func (csvExporter *CSVExportOutput) ExportDocument(document bson.D) error {
rowOut := make([]string, 0, len(csvExporter.Fields))
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
for _, fieldName := range csvExporter.Fields {
fieldVal := extractFieldByName(fieldName, extendedDoc)
if fieldVal == nil {
rowOut = append(rowOut, "")
} else if reflect.TypeOf(fieldVal) == reflect.TypeOf(bson.M{}) ||
reflect.TypeOf(fieldVal) == reflect.TypeOf(bson.D{}) ||
reflect.TypeOf(fieldVal) == marshalDType ||
reflect.TypeOf(fieldVal) == reflect.TypeOf([]interface{}{}) {
buf, err := json.Marshal(fieldVal)
if err != nil {
rowOut = append(rowOut, "")
} else {
rowOut = append(rowOut, string(buf))
}
} else {
rowOut = append(rowOut, fmt.Sprintf("%v", fieldVal))
}
}
csvExporter.csvWriter.Write(rowOut)
csvExporter.NumExported++
return csvExporter.csvWriter.Error()
}
//ExportDocument converts the given document to extended json, and writes it
//to the output.
func (jsonExporter *JSONExportOutput) ExportDocument(document bson.M) error {
if jsonExporter.ArrayOutput {
if jsonExporter.NumExported >= 1 {
jsonExporter.Out.Write([]byte(","))
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
jsonOut, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("Error converting BSON to extended JSON: %v", err)
}
jsonExporter.Out.Write(jsonOut)
} else {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
err = jsonExporter.Encoder.Encode(extendedDoc)
if err != nil {
return err
}
}
jsonExporter.NumExported++
return nil
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
logger := log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile)
pcap, err := pcap.OpenOffline(*pcapFile)
if err != nil {
fmt.Fprintln(os.Stderr, "error opening pcap file:", err)
os.Exit(1)
}
h := mongocaputils.NewPacketHandler(pcap)
m := mongocaputils.NewMongoOpStream(*packetBufSize)
ch := make(chan struct{})
go func() {
defer close(ch)
for op := range m.Ops {
// TODO: add other op types
if opQuery, ok := op.Op.(*mongoproto.OpQuery); ok {
fbOp := map[string]interface{}{}
fbOp["ns"] = opQuery.FullCollectionName
fbOp["ntoskip"] = opQuery.NumberToSkip
fbOp["ntoreturn"] = opQuery.NumberToReturn
fbOp["ts"] = json.Date(op.Seen.Unix())
query, err := rawBSONToJSON(opQuery.Query)
if err != nil {
logger.Println(err)
if !*continueOnError {
os.Exit(1)
}
}
if strings.HasSuffix(opQuery.FullCollectionName, ".$cmd") {
fbOp["op"] = "command"
fbOp["command"] = query
} else {
fbOp["op"] = "query"
fbOp["query"] = query
}
fbOpStr, err := json.Marshal(fbOp)
if err != nil {
logger.Println(err)
if !*continueOnError {
os.Exit(1)
}
}
fmt.Println(string(fbOpStr))
}
}
}()
if err := h.Handle(m, -1); err != nil {
fmt.Fprintln(os.Stderr, "pcap_converter: error handling packet stream:", err)
}
<-ch
}
func (op *QueryOp) getOpBodyString() (string, error) {
queryAsJSON, err := ConvertBSONValueToJSON(op.Query)
if err != nil {
return "", fmt.Errorf("ConvertBSONValueToJSON err: %#v - %v", op, err)
}
asJSON, err := json.Marshal(queryAsJSON)
if err != nil {
return "", fmt.Errorf("json marshal err: %#v - %v", op, err)
}
return string(asJSON), nil
}
func dumpDoc(doc *bson.M, out io.Writer) error {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(doc)
if err != nil {
return fmt.Errorf("Error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("Error converting doc to JSON: %v", err)
}
_, err = out.Write(jsonBytes)
return err
}
// MarshalJSON makes the MarshalD type usable by
// the encoding/json package.
func (md MarshalD) MarshalJSON() ([]byte, error) {
var buff bytes.Buffer
buff.WriteString("{")
for i, item := range md {
key, err := json.Marshal(item.Name)
if err != nil {
return nil, fmt.Errorf("cannot marshal key %v: %v", item.Name, err)
}
val, err := json.Marshal(item.Value)
if err != nil {
return nil, fmt.Errorf("cannot marshal value %v: %v", item.Value, err)
}
buff.Write(key)
buff.WriteString(":")
buff.Write(val)
if i != len(md)-1 {
buff.WriteString(",")
}
}
buff.WriteString("}")
return buff.Bytes(), nil
}
func (op *OpQuery) String() string {
var query interface{}
if err := bson.Unmarshal(op.Query, &query); err != nil {
return "(error unmarshalling)"
}
queryAsJSON, err := bsonutil.ConvertBSONValueToJSON(query)
if err != nil {
return fmt.Sprintf("ConvertBSONValueToJSON err: %#v - %v", op, err)
}
asJSON, err := json.Marshal(queryAsJSON)
if err != nil {
return fmt.Sprintf("json marshal err: %#v - %v", op, err)
}
return fmt.Sprintf("OpQuery %v %v", op.FullCollectionName, string(asJSON))
}
func formatJSON(doc *bson.Raw, pretty bool) ([]byte, error) {
decodedDoc := bson.D{}
err := bson.Unmarshal(doc.Data, &decodedDoc)
if err != nil {
return nil, err
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(decodedDoc)
if err != nil {
return nil, fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if pretty {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonBytes, "", "\t")
jsonBytes = jsonFormatted.Bytes()
}
if err != nil {
return nil, fmt.Errorf("error converting doc to JSON: %v", err)
}
return jsonBytes, nil
}
func printJSON(doc *bson.Raw, out io.Writer, pretty bool) error {
decodedDoc := bson.D{}
err := bson.Unmarshal(doc.Data, &decodedDoc)
if err != nil {
return err
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(decodedDoc)
if err != nil {
return fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if pretty {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonBytes, "", "\t")
jsonBytes = jsonFormatted.Bytes()
}
if err != nil {
return fmt.Errorf("error converting doc to JSON: %v", err)
}
_, err = out.Write(jsonBytes)
return err
}
// ExportDocument converts the given document to extended JSON, and writes it
// to the output.
func (jsonExporter *JSONExportOutput) ExportDocument(document bson.M) error {
if jsonExporter.ArrayOutput || jsonExporter.PrettyOutput {
if jsonExporter.NumExported >= 1 {
if jsonExporter.ArrayOutput {
jsonExporter.Out.Write([]byte(","))
}
if jsonExporter.PrettyOutput {
jsonExporter.Out.Write([]byte("\n"))
}
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
jsonOut, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
if jsonExporter.PrettyOutput {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonOut, "", "\t")
jsonOut = jsonFormatted.Bytes()
}
jsonExporter.Out.Write(jsonOut)
} else {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
err = jsonExporter.Encoder.Encode(extendedDoc)
if err != nil {
return err
}
}
jsonExporter.NumExported++
return nil
}
func TestMongoDumpBSON(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
log.SetWriter(ioutil.Discard)
Convey("With a MongoDump instance", t, func() {
err := setUpMongoDumpTestData()
So(err, ShouldBeNil)
Convey("testing that using MongoDump WITHOUT giving a query dumps everything in the database and/or collection", func() {
md := simpleMongoDumpInstance()
md.InputOptions.Query = ""
Convey("and that for a particular collection", func() {
md.ToolOptions.Namespace.Collection = testCollectionNames[0]
err = md.Init()
So(err, ShouldBeNil)
Convey("it dumps to the default output directory", func() {
// we don't have to set this manually if parsing options via command line
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
err = readBSONIntoDatabase(dumpDBDir, testRestoreDB)
So(err, ShouldBeNil)
session, err := getBareSession()
So(err, ShouldBeNil)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
collOriginal := session.DB(testDB).C(testCollectionNames[0])
collRestore := session.DB(testRestoreDB).C(testCollectionNames[0])
Convey("with the correct number of documents", func() {
numDocsOrig, err := collOriginal.Count()
So(err, ShouldBeNil)
numDocsRestore, err := collRestore.Count()
So(err, ShouldBeNil)
So(numDocsOrig, ShouldEqual, numDocsRestore)
})
Convey("that are the same as the documents in the test database", func() {
iter := collOriginal.Find(nil).Iter()
var result bson.M
for iter.Next(&result) {
restoredCount, err := collRestore.Find(result).Count()
So(err, ShouldBeNil)
So(restoredCount, ShouldNotEqual, 0)
}
So(iter.Close(), ShouldBeNil)
})
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to a user-specified output directory", func() {
md.OutputOptions.Out = "dump_user"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump_user"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to standard output", func() {
md.OutputOptions.Out = "-"
stdoutBuf := &bytes.Buffer{}
md.stdout = stdoutBuf
err = md.Dump()
So(err, ShouldBeNil)
var count int
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(ioutil.NopCloser(stdoutBuf)))
defer bsonSource.Close()
var result bson.Raw
for bsonSource.Next(&result) {
count++
}
So(bsonSource.Err(), ShouldBeNil)
So(count, ShouldEqual, 10) //The 0th collection has 10 documents
Reset(func() {
})
})
})
Convey("for an entire database", func() {
md.ToolOptions.Namespace.Collection = ""
err = md.Init()
So(err, ShouldBeNil)
Convey("that exists. The dumped directory should contain the necessary bson files", func() {
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, len(testCollectionNames))
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("that does not exist. The dumped directory shouldn't be created", func() {
md.OutputOptions.Out = "dump"
md.ToolOptions.Namespace.DB = "nottestdb"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, "nottestdb"))
So(fileDirExists(dumpDir), ShouldBeFalse)
So(fileDirExists(dumpDBDir), ShouldBeFalse)
})
})
})
Convey("testing that using MongoDump WITH a query dumps a subset of documents in a database and/or collection", func() {
session, err := getBareSession()
So(err, ShouldBeNil)
md := simpleMongoDumpInstance()
// expect 10 documents per collection
bsonQuery := bson.M{"age": bson.M{"$lt": 10}}
jsonQuery, err := bsonutil.ConvertBSONValueToJSON(bsonQuery)
So(err, ShouldBeNil)
jsonQueryBytes, err := json.Marshal(jsonQuery)
So(err, ShouldBeNil)
Convey("using --query for all the collections in the database", func() {
md.InputOptions.Query = string(jsonQueryBytes)
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("using --queryFile for all the collections in the database", func() {
ioutil.WriteFile("example.json", jsonQueryBytes, 0777)
md.InputOptions.QueryFile = "example.json"
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
So(os.Remove("example.json"), ShouldBeNil)
})
})
})
Reset(func() {
So(tearDownMongoDumpTestData(), ShouldBeNil)
})
})
}
// dumpMetadata gets the metadata for a collection and writes it
// in readable JSON format.
func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
var err error
nsID := fmt.Sprintf("%v.%v", intent.DB, intent.C)
meta := Metadata{
// We have to initialize Indexes to an empty slice, not nil, so that an empty
// array is marshalled into json instead of null. That is, {indexes:[]} is okay
// but {indexes:null} will cause assertions in our legacy C++ mongotools
Indexes: []interface{}{},
}
// The collection options were already gathered while building the list of intents.
// We convert them to JSON so that they can be written to the metadata json file as text.
if intent.Options != nil {
if meta.Options, err = bsonutil.ConvertBSONValueToJSON(*intent.Options); err != nil {
return fmt.Errorf("error converting collection options to JSON: %v", err)
}
} else {
meta.Options = nil
}
// Second, we read the collection's index information by either calling
// listIndexes (pre-2.7 systems) or querying system.indexes.
// We keep a running list of all the indexes
// for the current collection as we iterate over the cursor, and include
// that list as the "indexes" field of the metadata document.
log.Logvf(log.DebugHigh, "\treading indexes for `%v`", nsID)
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
// get the indexes
indexesIter, err := db.GetIndexes(session.DB(intent.DB).C(intent.C))
if err != nil {
return err
}
if indexesIter == nil {
log.Logvf(log.Always, "the collection %v appears to have been dropped after the dump started", intent.Namespace())
return nil
}
indexOpts := &bson.D{}
for indexesIter.Next(indexOpts) {
convertedIndex, err := bsonutil.ConvertBSONValueToJSON(*indexOpts)
if err != nil {
return fmt.Errorf("error converting index (%#v): %v", convertedIndex, err)
}
meta.Indexes = append(meta.Indexes, convertedIndex)
}
if err := indexesIter.Err(); err != nil {
return fmt.Errorf("error getting indexes for collection `%v`: %v", nsID, err)
}
// Finally, we send the results to the writer as JSON bytes
jsonBytes, err := json.Marshal(meta)
if err != nil {
return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", nsID, err)
}
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
// make a buffered writer for nicer disk i/o
w := bufio.NewWriter(intent.MetadataFile)
_, err = w.Write(jsonBytes)
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
err = w.Flush()
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
return nil
}
