Example #1
0
func TestCreateIntentsForDB(t *testing.T) {
	// This tests creates intents based on the test file tree:
	//   db1
	//   db1/baddir
	//   db1/baddir/out.bson
	//   db1/c1.bson
	//   db1/c1.metadata.json
	//   db1/c2.bson
	//   db1/c3.bson
	//   db1/c3.metadata.json

	var mr *MongoRestore
	var buff bytes.Buffer

	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a test MongoRestore", t, func() {
		mr = newMongoRestore()
		log.SetWriter(&buff)

		Convey("running CreateIntentsForDB should succeed", func() {
			ddl, err := newActualPath("testdata/testdirs/db1")
			So(err, ShouldBeNil)
			err = mr.CreateIntentsForDB("myDB", ddl)
			So(err, ShouldBeNil)
			mr.manager.Finalize(intents.Legacy)

			Convey("and reading the intents should show alphabetical order", func() {
				i0 := mr.manager.Pop()
				So(i0.C, ShouldEqual, "c1")
				i1 := mr.manager.Pop()
				So(i1.C, ShouldEqual, "c2")
				i2 := mr.manager.Pop()
				So(i2.C, ShouldEqual, "c3")
				i3 := mr.manager.Pop()
				So(i3, ShouldBeNil)

				Convey("and all intents should have the supplied db name", func() {
					So(i0.DB, ShouldEqual, "myDB")
					So(i1.DB, ShouldEqual, "myDB")
					So(i2.DB, ShouldEqual, "myDB")
				})

				Convey("with all the proper metadata + bson merges", func() {
					So(i0.Location, ShouldNotEqual, "")
					So(i0.MetadataLocation, ShouldNotEqual, "")
					So(i1.Location, ShouldNotEqual, "")
					So(i1.MetadataLocation, ShouldEqual, "") //no metadata for this file
					So(i2.Location, ShouldNotEqual, "")
					So(i2.MetadataLocation, ShouldNotEqual, "")

					Convey("and skipped files all present in the logs", func() {
						logs := buff.String()
						So(strings.Contains(logs, "baddir"), ShouldEqual, true)
					})
				})
			})
		})
	})
}
Example #2
0
func TestCreateAllIntents(t *testing.T) {
	// This tests creates intents based on the test file tree:
	//   testdirs/badfile.txt
	//   testdirs/oplog.bson
	//   testdirs/db1
	//   testdirs/db1/baddir
	//   testdirs/db1/baddir/out.bson
	//   testdirs/db1/c1.bson
	//   testdirs/db1/c1.metadata.json
	//   testdirs/db1/c2.bson
	//   testdirs/db1/c3.bson
	//   testdirs/db1/c3.metadata.json
	//   testdirs/db2
	//   testdirs/db2/c1.bin
	//   testdirs/db2/c2.txt

	var mr *MongoRestore
	var buff bytes.Buffer

	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a test MongoRestore", t, func() {
		mr = &MongoRestore{
			manager:      intents.NewIntentManager(),
			InputOptions: &InputOptions{},
			ToolOptions:  &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
		}
		log.SetWriter(&buff)

		Convey("running CreateAllIntents should succeed", func() {
			ddl, err := newActualPath("testdata/testdirs/")
			So(err, ShouldBeNil)
			So(mr.CreateAllIntents(ddl, "", ""), ShouldBeNil)
			mr.manager.Finalize(intents.Legacy)

			Convey("and reading the intents should show alphabetical order", func() {
				i0 := mr.manager.Pop()
				So(i0.DB, ShouldEqual, "db1")
				So(i0.C, ShouldEqual, "c1")
				i1 := mr.manager.Pop()
				So(i1.DB, ShouldEqual, "db1")
				So(i1.C, ShouldEqual, "c2")
				i2 := mr.manager.Pop()
				So(i2.DB, ShouldEqual, "db1")
				So(i2.C, ShouldEqual, "c3")
				i3 := mr.manager.Pop()
				So(i3.DB, ShouldEqual, "db2")
				So(i3.C, ShouldEqual, "c1")
				i4 := mr.manager.Pop()
				So(i4, ShouldBeNil)

				Convey("with all the proper metadata + bson merges", func() {
					So(i0.Location, ShouldNotEqual, "")
					So(i0.MetadataLocation, ShouldNotEqual, "")
					So(i1.Location, ShouldNotEqual, "")
					So(i1.MetadataLocation, ShouldEqual, "") //no metadata for this file
					So(i2.Location, ShouldNotEqual, "")
					So(i2.MetadataLocation, ShouldNotEqual, "")
					So(i3.Location, ShouldNotEqual, "")
					So(i3.MetadataLocation, ShouldEqual, "") //no metadata for this file

					Convey("and skipped files all present in the logs", func() {
						logs := buff.String()
						So(strings.Contains(logs, "badfile.txt"), ShouldEqual, true)
						So(strings.Contains(logs, "baddir"), ShouldEqual, true)
						So(strings.Contains(logs, "c2.txt"), ShouldEqual, true)
					})
				})
			})
		})
	})
}
Example #3
0
func TestCreateIntentsForCollection(t *testing.T) {
	var mr *MongoRestore
	var buff bytes.Buffer

	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a test MongoRestore", t, func() {
		buff = bytes.Buffer{}
		mr = &MongoRestore{
			manager:      intents.NewIntentManager(),
			ToolOptions:  &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
			InputOptions: &InputOptions{},
		}
		log.SetWriter(&buff)

		Convey("running CreateIntentForCollection on a file without metadata", func() {
			ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
			So(err, ShouldBeNil)
			err = mr.CreateIntentForCollection("myDB", "myC", ddl)
			So(err, ShouldBeNil)
			mr.manager.Finalize(intents.Legacy)

			Convey("should create one intent with 'myDb' and 'myC' fields", func() {
				i0 := mr.manager.Pop()
				So(i0, ShouldNotBeNil)
				So(i0.DB, ShouldEqual, "myDB")
				So(i0.C, ShouldEqual, "myC")
				ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
				So(err, ShouldBeNil)
				So(i0.Location, ShouldEqual, ddl.Path())
				i1 := mr.manager.Pop()
				So(i1, ShouldBeNil)

				Convey("and no Metadata path", func() {
					So(i0.MetadataLocation, ShouldEqual, "")
					logs := buff.String()
					So(strings.Contains(logs, "without metadata"), ShouldEqual, true)
				})
			})
		})

		Convey("running CreateIntentForCollection on a file *with* metadata", func() {
			ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
			So(err, ShouldBeNil)
			err = mr.CreateIntentForCollection("myDB", "myC", ddl)
			So(err, ShouldBeNil)
			mr.manager.Finalize(intents.Legacy)

			Convey("should create one intent with 'myDb' and 'myC' fields", func() {
				i0 := mr.manager.Pop()
				So(i0, ShouldNotBeNil)
				So(i0.DB, ShouldEqual, "myDB")
				So(i0.C, ShouldEqual, "myC")
				So(i0.Location, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
				i1 := mr.manager.Pop()
				So(i1, ShouldBeNil)

				Convey("and a set Metadata path", func() {
					So(i0.MetadataLocation, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.metadata.json"))
					logs := buff.String()
					So(strings.Contains(logs, "found metadata"), ShouldEqual, true)
				})
			})
		})

		Convey("running CreateIntentForCollection on a non-existent file", func() {
			_, err := newActualPath("aaaaaaaaaaaaaa.bson")
			Convey("should fail", func() {
				So(err, ShouldNotBeNil)
			})
		})

		Convey("running CreateIntentForCollection on a directory", func() {
			ddl, err := newActualPath("testdata")
			So(err, ShouldBeNil)
			err = mr.CreateIntentForCollection(
				"myDB", "myC", ddl)

			Convey("should fail", func() {
				So(err, ShouldNotBeNil)
			})
		})

		Convey("running CreateIntentForCollection on non-bson file", func() {
			ddl, err := newActualPath("testdata/testdirs/db1/c1.metadata.json")
			So(err, ShouldBeNil)
			err = mr.CreateIntentForCollection(
				"myDB", "myC", ddl)

			Convey("should fail", func() {
				So(err, ShouldNotBeNil)
			})
		})

	})
}
Example #4
0
func TestMongoDumpMetaData(t *testing.T) {
	testutil.VerifyTestType(t, testutil.IntegrationTestType)
	log.SetWriter(ioutil.Discard)

	Convey("With a MongoDump instance", t, func() {
		err := setUpMongoDumpTestData()
		So(err, ShouldBeNil)

		Convey("testing that the dumped directory contains information about indexes", func() {
			md := simpleMongoDumpInstance()
			md.OutputOptions.Out = "dump"
			err = md.Init()
			So(err, ShouldBeNil)

			err = md.Dump()
			So(err, ShouldBeNil)

			path, err := os.Getwd()
			So(err, ShouldBeNil)
			dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
			dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
			So(fileDirExists(dumpDir), ShouldBeTrue)
			So(fileDirExists(dumpDBDir), ShouldBeTrue)

			Convey("having one metadata file per collection", func() {
				c1, err := countNonIndexBSONFiles(dumpDBDir)
				So(err, ShouldBeNil)

				c2, err := countMetaDataFiles(dumpDBDir)
				So(err, ShouldBeNil)

				So(c1, ShouldEqual, c2)

				Convey("and that the JSON in a metadata file is valid", func() {
					metaFiles, err := getMatchingFiles(dumpDBDir, ".*\\.metadata\\.json")
					So(err, ShouldBeNil)
					So(len(metaFiles), ShouldBeGreaterThan, 0)

					oneMetaFile, err := os.Open(util.ToUniversalPath(filepath.Join(dumpDBDir, metaFiles[0])))
					So(err, ShouldBeNil)
					contents, err := ioutil.ReadAll(oneMetaFile)
					var jsonResult map[string]interface{}
					err = json.Unmarshal(contents, &jsonResult)
					So(err, ShouldBeNil)

					Convey("and contains an 'indexes' key", func() {
						_, ok := jsonResult["indexes"]
						So(ok, ShouldBeTrue)
						So(oneMetaFile.Close(), ShouldBeNil)
					})

				})

			})

			Reset(func() {
				So(os.RemoveAll(dumpDir), ShouldBeNil)
			})
		})

		Reset(func() {
			So(tearDownMongoDumpTestData(), ShouldBeNil)
		})

	})
}
Example #5
0
func TestMongoDumpBSON(t *testing.T) {
	testutil.VerifyTestType(t, testutil.IntegrationTestType)
	log.SetWriter(ioutil.Discard)

	Convey("With a MongoDump instance", t, func() {
		err := setUpMongoDumpTestData()
		So(err, ShouldBeNil)

		Convey("testing that using MongoDump WITHOUT giving a query dumps everything in the database and/or collection", func() {
			md := simpleMongoDumpInstance()
			md.InputOptions.Query = ""

			Convey("and that for a particular collection", func() {
				md.ToolOptions.Namespace.Collection = testCollectionNames[0]
				err = md.Init()
				So(err, ShouldBeNil)

				Convey("it dumps to the default output directory", func() {
					// we don't have to set this manually if parsing options via command line
					md.OutputOptions.Out = "dump"
					err = md.Dump()
					So(err, ShouldBeNil)
					path, err := os.Getwd()
					So(err, ShouldBeNil)

					dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
					dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
					So(fileDirExists(dumpDir), ShouldBeTrue)
					So(fileDirExists(dumpDBDir), ShouldBeTrue)

					err = readBSONIntoDatabase(dumpDBDir, testRestoreDB)
					So(err, ShouldBeNil)

					session, err := getBareSession()
					So(err, ShouldBeNil)

					countColls, err := countNonIndexBSONFiles(dumpDBDir)
					So(err, ShouldBeNil)
					So(countColls, ShouldEqual, 1)

					collOriginal := session.DB(testDB).C(testCollectionNames[0])
					collRestore := session.DB(testRestoreDB).C(testCollectionNames[0])

					Convey("with the correct number of documents", func() {
						numDocsOrig, err := collOriginal.Count()
						So(err, ShouldBeNil)

						numDocsRestore, err := collRestore.Count()
						So(err, ShouldBeNil)

						So(numDocsOrig, ShouldEqual, numDocsRestore)
					})

					Convey("that are the same as the documents in the test database", func() {
						iter := collOriginal.Find(nil).Iter()

						var result bson.M
						for iter.Next(&result) {
							restoredCount, err := collRestore.Find(result).Count()
							So(err, ShouldBeNil)
							So(restoredCount, ShouldNotEqual, 0)
						}
						So(iter.Close(), ShouldBeNil)
					})

					Reset(func() {
						So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
						So(os.RemoveAll(dumpDir), ShouldBeNil)
					})
				})

				Convey("it dumps to a user-specified output directory", func() {
					md.OutputOptions.Out = "dump_user"
					err = md.Dump()
					So(err, ShouldBeNil)
					path, err := os.Getwd()
					So(err, ShouldBeNil)

					dumpDir := util.ToUniversalPath(filepath.Join(path, "dump_user"))
					dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
					So(fileDirExists(dumpDir), ShouldBeTrue)
					So(fileDirExists(dumpDBDir), ShouldBeTrue)

					countColls, err := countNonIndexBSONFiles(dumpDBDir)
					So(err, ShouldBeNil)
					So(countColls, ShouldEqual, 1)

					Reset(func() {
						So(os.RemoveAll(dumpDir), ShouldBeNil)
					})

				})

				Convey("it dumps to standard output", func() {
					md.OutputOptions.Out = "-"
					stdoutBuf := &bytes.Buffer{}
					md.stdout = stdoutBuf
					err = md.Dump()
					So(err, ShouldBeNil)
					var count int
					bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(ioutil.NopCloser(stdoutBuf)))
					defer bsonSource.Close()

					var result bson.Raw
					for bsonSource.Next(&result) {
						count++
					}
					So(bsonSource.Err(), ShouldBeNil)
					So(count, ShouldEqual, 10) //The 0th collection has 10 documents

					Reset(func() {
					})

				})

			})

			Convey("for an entire database", func() {
				md.ToolOptions.Namespace.Collection = ""
				err = md.Init()
				So(err, ShouldBeNil)

				Convey("that exists. The dumped directory should contain the necessary bson files", func() {
					md.OutputOptions.Out = "dump"
					err = md.Dump()
					So(err, ShouldBeNil)
					path, err := os.Getwd()
					So(err, ShouldBeNil)

					dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
					dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
					So(fileDirExists(dumpDir), ShouldBeTrue)
					So(fileDirExists(dumpDBDir), ShouldBeTrue)

					countColls, err := countNonIndexBSONFiles(dumpDBDir)
					So(err, ShouldBeNil)
					So(countColls, ShouldEqual, len(testCollectionNames))

					Reset(func() {
						So(os.RemoveAll(dumpDir), ShouldBeNil)
					})

				})

				Convey("that does not exist. The dumped directory shouldn't be created", func() {
					md.OutputOptions.Out = "dump"
					md.ToolOptions.Namespace.DB = "nottestdb"
					err = md.Dump()
					So(err, ShouldBeNil)

					path, err := os.Getwd()
					So(err, ShouldBeNil)

					dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
					dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, "nottestdb"))

					So(fileDirExists(dumpDir), ShouldBeFalse)
					So(fileDirExists(dumpDBDir), ShouldBeFalse)
				})

			})
		})

		Convey("testing that using MongoDump WITH a query dumps a subset of documents in a database and/or collection", func() {
			session, err := getBareSession()
			So(err, ShouldBeNil)
			md := simpleMongoDumpInstance()

			// expect 10 documents per collection
			bsonQuery := bson.M{"age": bson.M{"$lt": 10}}
			jsonQuery, err := bsonutil.ConvertBSONValueToJSON(bsonQuery)
			So(err, ShouldBeNil)
			jsonQueryBytes, err := json.Marshal(jsonQuery)
			So(err, ShouldBeNil)

			Convey("using --query for all the collections in the database", func() {
				md.InputOptions.Query = string(jsonQueryBytes)
				md.ToolOptions.Namespace.DB = testDB
				md.OutputOptions.Out = "dump"
				dumpDir := testQuery(md, session)

				Reset(func() {
					So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
					So(os.RemoveAll(dumpDir), ShouldBeNil)
				})

			})

			Convey("using --queryFile for all the collections in the database", func() {
				ioutil.WriteFile("example.json", jsonQueryBytes, 0777)
				md.InputOptions.QueryFile = "example.json"
				md.ToolOptions.Namespace.DB = testDB
				md.OutputOptions.Out = "dump"
				dumpDir := testQuery(md, session)

				Reset(func() {
					So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
					So(os.RemoveAll(dumpDir), ShouldBeNil)
					So(os.Remove("example.json"), ShouldBeNil)
				})

			})
		})

		Reset(func() {
			So(tearDownMongoDumpTestData(), ShouldBeNil)
		})
	})
}