func TestCreateIntentsForDB(t *testing.T) {
// This tests creates intents based on the test file tree:
// db1
// db1/baddir
// db1/baddir/out.bson
// db1/c1.bson
// db1/c1.metadata.json
// db1/c2.bson
// db1/c3.bson
// db1/c3.metadata.json
var mr *MongoRestore
var buff bytes.Buffer
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
mr = newMongoRestore()
log.SetWriter(&buff)
Convey("running CreateIntentsForDB should succeed", func() {
ddl, err := newActualPath("testdata/testdirs/db1")
So(err, ShouldBeNil)
err = mr.CreateIntentsForDB("myDB", ddl)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("and reading the intents should show alphabetical order", func() {
i0 := mr.manager.Pop()
So(i0.C, ShouldEqual, "c1")
i1 := mr.manager.Pop()
So(i1.C, ShouldEqual, "c2")
i2 := mr.manager.Pop()
So(i2.C, ShouldEqual, "c3")
i3 := mr.manager.Pop()
So(i3, ShouldBeNil)
Convey("and all intents should have the supplied db name", func() {
So(i0.DB, ShouldEqual, "myDB")
So(i1.DB, ShouldEqual, "myDB")
So(i2.DB, ShouldEqual, "myDB")
})
Convey("with all the proper metadata + bson merges", func() {
So(i0.Location, ShouldNotEqual, "")
So(i0.MetadataLocation, ShouldNotEqual, "")
So(i1.Location, ShouldNotEqual, "")
So(i1.MetadataLocation, ShouldEqual, "") //no metadata for this file
So(i2.Location, ShouldNotEqual, "")
So(i2.MetadataLocation, ShouldNotEqual, "")
Convey("and skipped files all present in the logs", func() {
logs := buff.String()
So(strings.Contains(logs, "baddir"), ShouldEqual, true)
})
})
})
})
})
}
func TestCreateAllIntents(t *testing.T) {
// This tests creates intents based on the test file tree:
// testdirs/badfile.txt
// testdirs/oplog.bson
// testdirs/db1
// testdirs/db1/baddir
// testdirs/db1/baddir/out.bson
// testdirs/db1/c1.bson
// testdirs/db1/c1.metadata.json
// testdirs/db1/c2.bson
// testdirs/db1/c3.bson
// testdirs/db1/c3.metadata.json
// testdirs/db2
// testdirs/db2/c1.bin
// testdirs/db2/c2.txt
var mr *MongoRestore
var buff bytes.Buffer
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
mr = &MongoRestore{
manager: intents.NewIntentManager(),
InputOptions: &InputOptions{},
ToolOptions: &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
}
log.SetWriter(&buff)
Convey("running CreateAllIntents should succeed", func() {
ddl, err := newActualPath("testdata/testdirs/")
So(err, ShouldBeNil)
So(mr.CreateAllIntents(ddl, "", ""), ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("and reading the intents should show alphabetical order", func() {
i0 := mr.manager.Pop()
So(i0.DB, ShouldEqual, "db1")
So(i0.C, ShouldEqual, "c1")
i1 := mr.manager.Pop()
So(i1.DB, ShouldEqual, "db1")
So(i1.C, ShouldEqual, "c2")
i2 := mr.manager.Pop()
So(i2.DB, ShouldEqual, "db1")
So(i2.C, ShouldEqual, "c3")
i3 := mr.manager.Pop()
So(i3.DB, ShouldEqual, "db2")
So(i3.C, ShouldEqual, "c1")
i4 := mr.manager.Pop()
So(i4, ShouldBeNil)
Convey("with all the proper metadata + bson merges", func() {
So(i0.Location, ShouldNotEqual, "")
So(i0.MetadataLocation, ShouldNotEqual, "")
So(i1.Location, ShouldNotEqual, "")
So(i1.MetadataLocation, ShouldEqual, "") //no metadata for this file
So(i2.Location, ShouldNotEqual, "")
So(i2.MetadataLocation, ShouldNotEqual, "")
So(i3.Location, ShouldNotEqual, "")
So(i3.MetadataLocation, ShouldEqual, "") //no metadata for this file
Convey("and skipped files all present in the logs", func() {
logs := buff.String()
So(strings.Contains(logs, "badfile.txt"), ShouldEqual, true)
So(strings.Contains(logs, "baddir"), ShouldEqual, true)
So(strings.Contains(logs, "c2.txt"), ShouldEqual, true)
})
})
})
})
})
}
func TestCreateIntentsForCollection(t *testing.T) {
var mr *MongoRestore
var buff bytes.Buffer
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
buff = bytes.Buffer{}
mr = &MongoRestore{
manager: intents.NewIntentManager(),
ToolOptions: &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
InputOptions: &InputOptions{},
}
log.SetWriter(&buff)
Convey("running CreateIntentForCollection on a file without metadata", func() {
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection("myDB", "myC", ddl)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("should create one intent with 'myDb' and 'myC' fields", func() {
i0 := mr.manager.Pop()
So(i0, ShouldNotBeNil)
So(i0.DB, ShouldEqual, "myDB")
So(i0.C, ShouldEqual, "myC")
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
So(err, ShouldBeNil)
So(i0.Location, ShouldEqual, ddl.Path())
i1 := mr.manager.Pop()
So(i1, ShouldBeNil)
Convey("and no Metadata path", func() {
So(i0.MetadataLocation, ShouldEqual, "")
logs := buff.String()
So(strings.Contains(logs, "without metadata"), ShouldEqual, true)
})
})
})
Convey("running CreateIntentForCollection on a file *with* metadata", func() {
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection("myDB", "myC", ddl)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("should create one intent with 'myDb' and 'myC' fields", func() {
i0 := mr.manager.Pop()
So(i0, ShouldNotBeNil)
So(i0.DB, ShouldEqual, "myDB")
So(i0.C, ShouldEqual, "myC")
So(i0.Location, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
i1 := mr.manager.Pop()
So(i1, ShouldBeNil)
Convey("and a set Metadata path", func() {
So(i0.MetadataLocation, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.metadata.json"))
logs := buff.String()
So(strings.Contains(logs, "found metadata"), ShouldEqual, true)
})
})
})
Convey("running CreateIntentForCollection on a non-existent file", func() {
_, err := newActualPath("aaaaaaaaaaaaaa.bson")
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
Convey("running CreateIntentForCollection on a directory", func() {
ddl, err := newActualPath("testdata")
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection(
"myDB", "myC", ddl)
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
Convey("running CreateIntentForCollection on non-bson file", func() {
ddl, err := newActualPath("testdata/testdirs/db1/c1.metadata.json")
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection(
"myDB", "myC", ddl)
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
})
}
func TestMongoDumpMetaData(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
log.SetWriter(ioutil.Discard)
Convey("With a MongoDump instance", t, func() {
err := setUpMongoDumpTestData()
So(err, ShouldBeNil)
Convey("testing that the dumped directory contains information about indexes", func() {
md := simpleMongoDumpInstance()
md.OutputOptions.Out = "dump"
err = md.Init()
So(err, ShouldBeNil)
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
Convey("having one metadata file per collection", func() {
c1, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
c2, err := countMetaDataFiles(dumpDBDir)
So(err, ShouldBeNil)
So(c1, ShouldEqual, c2)
Convey("and that the JSON in a metadata file is valid", func() {
metaFiles, err := getMatchingFiles(dumpDBDir, ".*\\.metadata\\.json")
So(err, ShouldBeNil)
So(len(metaFiles), ShouldBeGreaterThan, 0)
oneMetaFile, err := os.Open(util.ToUniversalPath(filepath.Join(dumpDBDir, metaFiles[0])))
So(err, ShouldBeNil)
contents, err := ioutil.ReadAll(oneMetaFile)
var jsonResult map[string]interface{}
err = json.Unmarshal(contents, &jsonResult)
So(err, ShouldBeNil)
Convey("and contains an 'indexes' key", func() {
_, ok := jsonResult["indexes"]
So(ok, ShouldBeTrue)
So(oneMetaFile.Close(), ShouldBeNil)
})
})
})
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Reset(func() {
So(tearDownMongoDumpTestData(), ShouldBeNil)
})
})
}
func TestMongoDumpBSON(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
log.SetWriter(ioutil.Discard)
Convey("With a MongoDump instance", t, func() {
err := setUpMongoDumpTestData()
So(err, ShouldBeNil)
Convey("testing that using MongoDump WITHOUT giving a query dumps everything in the database and/or collection", func() {
md := simpleMongoDumpInstance()
md.InputOptions.Query = ""
Convey("and that for a particular collection", func() {
md.ToolOptions.Namespace.Collection = testCollectionNames[0]
err = md.Init()
So(err, ShouldBeNil)
Convey("it dumps to the default output directory", func() {
// we don't have to set this manually if parsing options via command line
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
err = readBSONIntoDatabase(dumpDBDir, testRestoreDB)
So(err, ShouldBeNil)
session, err := getBareSession()
So(err, ShouldBeNil)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
collOriginal := session.DB(testDB).C(testCollectionNames[0])
collRestore := session.DB(testRestoreDB).C(testCollectionNames[0])
Convey("with the correct number of documents", func() {
numDocsOrig, err := collOriginal.Count()
So(err, ShouldBeNil)
numDocsRestore, err := collRestore.Count()
So(err, ShouldBeNil)
So(numDocsOrig, ShouldEqual, numDocsRestore)
})
Convey("that are the same as the documents in the test database", func() {
iter := collOriginal.Find(nil).Iter()
var result bson.M
for iter.Next(&result) {
restoredCount, err := collRestore.Find(result).Count()
So(err, ShouldBeNil)
So(restoredCount, ShouldNotEqual, 0)
}
So(iter.Close(), ShouldBeNil)
})
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to a user-specified output directory", func() {
md.OutputOptions.Out = "dump_user"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump_user"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to standard output", func() {
md.OutputOptions.Out = "-"
stdoutBuf := &bytes.Buffer{}
md.stdout = stdoutBuf
err = md.Dump()
So(err, ShouldBeNil)
var count int
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(ioutil.NopCloser(stdoutBuf)))
defer bsonSource.Close()
var result bson.Raw
for bsonSource.Next(&result) {
count++
}
So(bsonSource.Err(), ShouldBeNil)
So(count, ShouldEqual, 10) //The 0th collection has 10 documents
Reset(func() {
})
})
})
Convey("for an entire database", func() {
md.ToolOptions.Namespace.Collection = ""
err = md.Init()
So(err, ShouldBeNil)
Convey("that exists. The dumped directory should contain the necessary bson files", func() {
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, len(testCollectionNames))
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("that does not exist. The dumped directory shouldn't be created", func() {
md.OutputOptions.Out = "dump"
md.ToolOptions.Namespace.DB = "nottestdb"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, "nottestdb"))
So(fileDirExists(dumpDir), ShouldBeFalse)
So(fileDirExists(dumpDBDir), ShouldBeFalse)
})
})
})
Convey("testing that using MongoDump WITH a query dumps a subset of documents in a database and/or collection", func() {
session, err := getBareSession()
So(err, ShouldBeNil)
md := simpleMongoDumpInstance()
// expect 10 documents per collection
bsonQuery := bson.M{"age": bson.M{"$lt": 10}}
jsonQuery, err := bsonutil.ConvertBSONValueToJSON(bsonQuery)
So(err, ShouldBeNil)
jsonQueryBytes, err := json.Marshal(jsonQuery)
So(err, ShouldBeNil)
Convey("using --query for all the collections in the database", func() {
md.InputOptions.Query = string(jsonQueryBytes)
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("using --queryFile for all the collections in the database", func() {
ioutil.WriteFile("example.json", jsonQueryBytes, 0777)
md.InputOptions.QueryFile = "example.json"
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
So(os.Remove("example.json"), ShouldBeNil)
})
})
})
Reset(func() {
So(tearDownMongoDumpTestData(), ShouldBeNil)
})
})
}