//ExportDocument converts the given document to extended json, and writes it
//to the output.
func (jsonExporter *JSONExportOutput) ExportDocument(document bson.M) error {
if jsonExporter.ArrayOutput {
if jsonExporter.NumExported >= 1 {
jsonExporter.Out.Write([]byte(","))
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
jsonOut, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("Error converting BSON to extended JSON: %v", err)
}
jsonExporter.Out.Write(jsonOut)
} else {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
err = jsonExporter.Encoder.Encode(extendedDoc)
if err != nil {
return err
}
}
jsonExporter.NumExported++
return nil
}
// ExportDocument writes a line to output with the CSV representation of a document.
func (csvExporter *CSVExportOutput) ExportDocument(document bson.D) error {
rowOut := make([]string, 0, len(csvExporter.Fields))
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
for _, fieldName := range csvExporter.Fields {
fieldVal := extractFieldByName(fieldName, extendedDoc)
if fieldVal == nil {
rowOut = append(rowOut, "")
} else if reflect.TypeOf(fieldVal) == reflect.TypeOf(bson.M{}) ||
reflect.TypeOf(fieldVal) == reflect.TypeOf(bson.D{}) ||
reflect.TypeOf(fieldVal) == marshalDType ||
reflect.TypeOf(fieldVal) == reflect.TypeOf([]interface{}{}) {
buf, err := json.Marshal(fieldVal)
if err != nil {
rowOut = append(rowOut, "")
} else {
rowOut = append(rowOut, string(buf))
}
} else {
rowOut = append(rowOut, fmt.Sprintf("%v", fieldVal))
}
}
csvExporter.csvWriter.Write(rowOut)
csvExporter.NumExported++
return csvExporter.csvWriter.Error()
}
func dumpDoc(doc *bson.M, out io.Writer) error {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(doc)
if err != nil {
return fmt.Errorf("Error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("Error converting doc to JSON: %v", err)
}
_, err = out.Write(jsonBytes)
return err
}
// Converts the raw BSON doc in op.Query to extended JSON
func rawBSONToJSON(rawBSON []byte) (interface{}, error) {
// Use bson.D to preserve order when unmarshalling
// http://godoc.org/github.com/mongodb/mongo-tools/common/bsonutil#MarshalD
var data bson.D
if err := bson.Unmarshal(rawBSON, &data); err != nil {
return nil, err
}
bsonAsJSON, err := bsonutil.ConvertBSONValueToJSON(data)
if err != nil {
return nil, err
}
return bsonAsJSON, nil
}
func (op *OpReply) String() string {
docs := make([]string, 0, len(op.Documents))
var doc interface{}
for _, d := range op.Documents {
_ = bson.Unmarshal(d, &doc)
jsonDoc, err := bsonutil.ConvertBSONValueToJSON(doc)
if err != nil {
return fmt.Sprintf("%#v - %v", op, err)
}
asJSON, _ := json.Marshal(jsonDoc)
docs = append(docs, string(asJSON))
}
return fmt.Sprintf("OpReply %v %v", op.Message, docs)
}
// ExportDocument converts the given document to extended JSON, and writes it
// to the output.
func (jsonExporter *JSONExportOutput) ExportDocument(document bson.M) error {
if jsonExporter.ArrayOutput || jsonExporter.PrettyOutput {
if jsonExporter.NumExported >= 1 {
if jsonExporter.ArrayOutput {
jsonExporter.Out.Write([]byte(","))
}
if jsonExporter.PrettyOutput {
jsonExporter.Out.Write([]byte("\n"))
}
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
jsonOut, err := json.Marshal(extendedDoc)
if err != nil {
return fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
if jsonExporter.PrettyOutput {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonOut, "", "\t")
jsonOut = jsonFormatted.Bytes()
}
jsonExporter.Out.Write(jsonOut)
} else {
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
err = jsonExporter.Encoder.Encode(extendedDoc)
if err != nil {
return err
}
}
jsonExporter.NumExported++
return nil
}
func (op *OpQuery) String() string {
var query interface{}
if err := bson.Unmarshal(op.Query, &query); err != nil {
return "(error unmarshalling)"
}
queryAsJSON, err := bsonutil.ConvertBSONValueToJSON(query)
if err != nil {
return fmt.Sprintf("ConvertBSONValueToJSON err: %#v - %v", op, err)
}
asJSON, err := json.Marshal(queryAsJSON)
if err != nil {
return fmt.Sprintf("json marshal err: %#v - %v", op, err)
}
return fmt.Sprintf("OpQuery %v %v", op.FullCollectionName, string(asJSON))
}
func (context *Context) dumpJsonToFile(collectionName string, id bson.ObjectId, writerImportScript *bufio.Writer) {
raw := bson.D{}
err := context.db.C(collectionName).Find(bson.M{"_id": id}).One(&raw)
if err != nil {
log.Fatalln("Could not open Marshal raw data", err)
}
output, err := bsonutil.ConvertBSONValueToJSON(raw)
if err != nil {
log.Fatalln("Could not convert to JSON", err)
}
var out []byte
out, err = json.Marshal(output)
if err != nil {
log.Fatalln("Could not Marshal JSON into bytes", err)
}
fmt.Fprintf(writerImportScript, "%s\n", out)
writerImportScript.Flush()
}
//ExportDocument writes a line to output with the CSV representation of a doc.
func (csvExporter *CSVExportOutput) ExportDocument(document bson.M) error {
rowOut := make([]string, 0, len(csvExporter.Fields))
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(document)
if err != nil {
return err
}
for _, fieldName := range csvExporter.Fields {
fieldVal, err := extractFieldByName(fieldName, extendedDoc)
if err != nil {
return nil
}
rowOut = append(rowOut, fmt.Sprintf("%s", fieldVal))
}
err = csvExporter.csvWriter.Write(rowOut)
if err != nil {
return err
}
csvExporter.NumExported++
return nil
}
func TestExtendedJSON(t *testing.T) {
Convey("Serializing a doc to extended JSON should work", t, func() {
x := bson.M{
"_id": bson.NewObjectId(),
"hey": "sup",
"subdoc": bson.M{
"subid": bson.NewObjectId(),
},
"array": []interface{}{
bson.NewObjectId(),
bson.Undefined,
},
}
out, err := bsonutil.ConvertBSONValueToJSON(x)
So(err, ShouldBeNil)
jsonEncoder := json.NewEncoder(os.Stdout)
jsonEncoder.Encode(out)
})
}
func formatJSON(doc *bson.Raw, pretty bool) ([]byte, error) {
decodedDoc := bson.D{}
err := bson.Unmarshal(doc.Data, &decodedDoc)
if err != nil {
return nil, err
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(decodedDoc)
if err != nil {
return nil, fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if pretty {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonBytes, "", "\t")
jsonBytes = jsonFormatted.Bytes()
}
if err != nil {
return nil, fmt.Errorf("error converting doc to JSON: %v", err)
}
return jsonBytes, nil
}
func printJSON(doc *bson.Raw, out io.Writer, pretty bool) error {
decodedDoc := bson.D{}
err := bson.Unmarshal(doc.Data, &decodedDoc)
if err != nil {
return err
}
extendedDoc, err := bsonutil.ConvertBSONValueToJSON(decodedDoc)
if err != nil {
return fmt.Errorf("error converting BSON to extended JSON: %v", err)
}
jsonBytes, err := json.Marshal(extendedDoc)
if pretty {
var jsonFormatted bytes.Buffer
json.Indent(&jsonFormatted, jsonBytes, "", "\t")
jsonBytes = jsonFormatted.Bytes()
}
if err != nil {
return fmt.Errorf("error converting doc to JSON: %v", err)
}
_, err = out.Write(jsonBytes)
return err
}
// CreateCollection creates the collection specified in the intent with the
// given options.
func (restore *MongoRestore) CreateCollection(intent *intents.Intent, options bson.D) error {
jsonCommand, err := bsonutil.ConvertBSONValueToJSON(
append(bson.D{{"create", intent.C}}, options...),
)
if err != nil {
return err
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
res := bson.M{}
err = session.DB(intent.DB).Run(jsonCommand, &res)
if err != nil {
return fmt.Errorf("error running create command: %v", err)
}
if util.IsFalsy(res["ok"]) {
return fmt.Errorf("create command: %v", res["errmsg"])
}
return nil
}
func TestMongoDumpBSON(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
log.SetWriter(ioutil.Discard)
Convey("With a MongoDump instance", t, func() {
err := setUpMongoDumpTestData()
So(err, ShouldBeNil)
Convey("testing that using MongoDump WITHOUT giving a query dumps everything in the database and/or collection", func() {
md := simpleMongoDumpInstance()
md.InputOptions.Query = ""
Convey("and that for a particular collection", func() {
md.ToolOptions.Namespace.Collection = testCollectionNames[0]
err = md.Init()
So(err, ShouldBeNil)
Convey("it dumps to the default output directory", func() {
// we don't have to set this manually if parsing options via command line
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
err = readBSONIntoDatabase(dumpDBDir, testRestoreDB)
So(err, ShouldBeNil)
session, err := getBareSession()
So(err, ShouldBeNil)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
collOriginal := session.DB(testDB).C(testCollectionNames[0])
collRestore := session.DB(testRestoreDB).C(testCollectionNames[0])
Convey("with the correct number of documents", func() {
numDocsOrig, err := collOriginal.Count()
So(err, ShouldBeNil)
numDocsRestore, err := collRestore.Count()
So(err, ShouldBeNil)
So(numDocsOrig, ShouldEqual, numDocsRestore)
})
Convey("that are the same as the documents in the test database", func() {
iter := collOriginal.Find(nil).Iter()
var result bson.M
for iter.Next(&result) {
restoredCount, err := collRestore.Find(result).Count()
So(err, ShouldBeNil)
So(restoredCount, ShouldNotEqual, 0)
}
So(iter.Close(), ShouldBeNil)
})
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to a user-specified output directory", func() {
md.OutputOptions.Out = "dump_user"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump_user"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to standard output", func() {
md.OutputOptions.Out = "-"
stdoutBuf := &bytes.Buffer{}
md.stdout = stdoutBuf
err = md.Dump()
So(err, ShouldBeNil)
var count int
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(ioutil.NopCloser(stdoutBuf)))
defer bsonSource.Close()
var result bson.Raw
for bsonSource.Next(&result) {
count++
}
So(bsonSource.Err(), ShouldBeNil)
So(count, ShouldEqual, 10) //The 0th collection has 10 documents
Reset(func() {
})
})
})
Convey("for an entire database", func() {
md.ToolOptions.Namespace.Collection = ""
err = md.Init()
So(err, ShouldBeNil)
Convey("that exists. The dumped directory should contain the necessary bson files", func() {
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, len(testCollectionNames))
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("that does not exist. The dumped directory shouldn't be created", func() {
md.OutputOptions.Out = "dump"
md.ToolOptions.Namespace.DB = "nottestdb"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, "nottestdb"))
So(fileDirExists(dumpDir), ShouldBeFalse)
So(fileDirExists(dumpDBDir), ShouldBeFalse)
})
})
})
Convey("testing that using MongoDump WITH a query dumps a subset of documents in a database and/or collection", func() {
session, err := getBareSession()
So(err, ShouldBeNil)
md := simpleMongoDumpInstance()
// expect 10 documents per collection
bsonQuery := bson.M{"age": bson.M{"$lt": 10}}
jsonQuery, err := bsonutil.ConvertBSONValueToJSON(bsonQuery)
So(err, ShouldBeNil)
jsonQueryBytes, err := json.Marshal(jsonQuery)
So(err, ShouldBeNil)
Convey("using --query for all the collections in the database", func() {
md.InputOptions.Query = string(jsonQueryBytes)
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("using --queryFile for all the collections in the database", func() {
ioutil.WriteFile("example.json", jsonQueryBytes, 0777)
md.InputOptions.QueryFile = "example.json"
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
So(os.Remove("example.json"), ShouldBeNil)
})
})
})
Reset(func() {
So(tearDownMongoDumpTestData(), ShouldBeNil)
})
})
}
// dumpMetadata gets the metadata for a collection and writes it
// in readable JSON format.
func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
var err error
nsID := fmt.Sprintf("%v.%v", intent.DB, intent.C)
meta := Metadata{
// We have to initialize Indexes to an empty slice, not nil, so that an empty
// array is marshalled into json instead of null. That is, {indexes:[]} is okay
// but {indexes:null} will cause assertions in our legacy C++ mongotools
Indexes: []interface{}{},
}
// The collection options were already gathered while building the list of intents.
// We convert them to JSON so that they can be written to the metadata json file as text.
if intent.Options != nil {
if meta.Options, err = bsonutil.ConvertBSONValueToJSON(*intent.Options); err != nil {
return fmt.Errorf("error converting collection options to JSON: %v", err)
}
} else {
meta.Options = nil
}
// Second, we read the collection's index information by either calling
// listIndexes (pre-2.7 systems) or querying system.indexes.
// We keep a running list of all the indexes
// for the current collection as we iterate over the cursor, and include
// that list as the "indexes" field of the metadata document.
log.Logvf(log.DebugHigh, "\treading indexes for `%v`", nsID)
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
// get the indexes
indexesIter, err := db.GetIndexes(session.DB(intent.DB).C(intent.C))
if err != nil {
return err
}
if indexesIter == nil {
log.Logvf(log.Always, "the collection %v appears to have been dropped after the dump started", intent.Namespace())
return nil
}
indexOpts := &bson.D{}
for indexesIter.Next(indexOpts) {
convertedIndex, err := bsonutil.ConvertBSONValueToJSON(*indexOpts)
if err != nil {
return fmt.Errorf("error converting index (%#v): %v", convertedIndex, err)
}
meta.Indexes = append(meta.Indexes, convertedIndex)
}
if err := indexesIter.Err(); err != nil {
return fmt.Errorf("error getting indexes for collection `%v`: %v", nsID, err)
}
// Finally, we send the results to the writer as JSON bytes
jsonBytes, err := json.Marshal(meta)
if err != nil {
return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", nsID, err)
}
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
// make a buffered writer for nicer disk i/o
w := bufio.NewWriter(intent.MetadataFile)
_, err = w.Write(jsonBytes)
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
err = w.Flush()
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
return nil
}