// Init performs preliminary setup operations for MongoDump.
func (dump *MongoDump) Init() error {
err := dump.ValidateOptions()
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
if dump.stdout == nil {
dump.stdout = os.Stdout
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
return fmt.Errorf("can't create session: %v", err)
}
// temporarily allow secondary reads for the isMongos check
dump.sessionProvider.SetReadPreference(mgo.Nearest)
dump.isMongos, err = dump.sessionProvider.IsMongos()
if err != nil {
return err
}
if dump.isMongos && dump.OutputOptions.Oplog {
return fmt.Errorf("can't use --oplog option when dumping from a mongos")
}
var mode mgo.Mode
if dump.ToolOptions.ReplicaSetName != "" || dump.isMongos {
mode = mgo.Primary
} else {
mode = mgo.Nearest
}
var tags bson.D
if dump.InputOptions.ReadPreference != "" {
mode, tags, err = db.ParseReadPreference(dump.InputOptions.ReadPreference)
if err != nil {
return fmt.Errorf("error parsing --readPreference : %v", err)
}
if len(tags) > 0 {
dump.sessionProvider.SetTags(tags)
}
}
// warn if we are trying to dump from a secondary in a sharded cluster
if dump.isMongos && mode != mgo.Primary {
log.Logf(log.Always, db.WarningNonPrimaryMongosConnection)
}
dump.sessionProvider.SetReadPreference(mode)
dump.sessionProvider.SetTags(tags)
dump.sessionProvider.SetFlags(db.DisableSocketTimeout)
// return a helpful error message for mongos --repair
if dump.OutputOptions.Repair && dump.isMongos {
return fmt.Errorf("--repair flag cannot be used on a mongos")
}
dump.manager = intents.NewIntentManager()
dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime)
return nil
}
// Init performs preliminary setup operations for MongoDump.
func (dump *MongoDump) Init() error {
err := dump.ValidateOptions()
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
return fmt.Errorf("can't create session: %v", err)
}
// allow secondary reads for the isMongos check
dump.sessionProvider.SetFlags(db.Monotonic)
dump.isMongos, err = dump.sessionProvider.IsMongos()
if err != nil {
return err
}
// ensure we allow secondary reads on mongods and disable TCP timeouts
flags := db.DisableSocketTimeout
if dump.isMongos {
log.Logf(log.Info, "connecting to mongos; secondary reads disabled")
} else {
flags |= db.Monotonic
}
dump.sessionProvider.SetFlags(flags)
// return a helpful error message for mongos --repair
if dump.OutputOptions.Repair && dump.isMongos {
return fmt.Errorf("--repair flag cannot be used on a mongos")
}
dump.manager = intents.NewIntentManager()
dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime)
return nil
}
func newMongoRestore() *MongoRestore {
renamer, _ := ns.NewRenamer([]string{}, []string{})
includer, _ := ns.NewMatcher([]string{"*"})
excluder, _ := ns.NewMatcher([]string{})
return &MongoRestore{
manager: intents.NewIntentManager(),
InputOptions: &InputOptions{},
ToolOptions: &commonOpts.ToolOptions{},
NSOptions: &NSOptions{},
renamer: renamer,
includer: includer,
excluder: excluder,
}
}
func TestHandlingBSON(t *testing.T) {
var mr *MongoRestore
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
mr = &MongoRestore{
manager: intents.NewIntentManager(),
ToolOptions: &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
InputOptions: &InputOptions{},
}
Convey("with a target path to a bson file instead of a directory", func() {
err := mr.handleBSONInsteadOfDirectory("testdata/testdirs/db1/c2.bson")
So(err, ShouldBeNil)
Convey("the proper DB and Coll should be inferred", func() {
So(mr.ToolOptions.DB, ShouldEqual, "db1")
So(mr.ToolOptions.Collection, ShouldEqual, "c2")
})
})
Convey("but pre-existing settings should not be overwritten", func() {
mr.ToolOptions.DB = "a"
Convey("either collection settings", func() {
mr.ToolOptions.Collection = "b"
err := mr.handleBSONInsteadOfDirectory("testdata/testdirs/db1/c1.bson")
So(err, ShouldBeNil)
So(mr.ToolOptions.DB, ShouldEqual, "a")
So(mr.ToolOptions.Collection, ShouldEqual, "b")
})
Convey("or db settings", func() {
err := mr.handleBSONInsteadOfDirectory("testdata/testdirs/db1/c1.bson")
So(err, ShouldBeNil)
So(mr.ToolOptions.DB, ShouldEqual, "a")
So(mr.ToolOptions.Collection, ShouldEqual, "c1")
})
})
})
}
func TestCreateIntentsForDB(t *testing.T) {
// This tests creates intents based on the test file tree:
// db1
// db1/baddir
// db1/baddir/out.bson
// db1/c1.bson
// db1/c1.metadata.json
// db1/c2.bson
// db1/c3.bson
// db1/c3.metadata.json
var mr *MongoRestore
var buff bytes.Buffer
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
mr = &MongoRestore{
InputOptions: &InputOptions{},
manager: intents.NewIntentManager(),
ToolOptions: &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
}
log.SetWriter(&buff)
Convey("running CreateIntentsForDB should succeed", func() {
ddl, err := newActualPath("testdata/testdirs/db1")
So(err, ShouldBeNil)
err = mr.CreateIntentsForDB("myDB", "", ddl, false)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("and reading the intents should show alphabetical order", func() {
i0 := mr.manager.Pop()
So(i0.C, ShouldEqual, "c1")
i1 := mr.manager.Pop()
So(i1.C, ShouldEqual, "c2")
i2 := mr.manager.Pop()
So(i2.C, ShouldEqual, "c3")
i3 := mr.manager.Pop()
So(i3, ShouldBeNil)
Convey("and all intents should have the supplied db name", func() {
So(i0.DB, ShouldEqual, "myDB")
So(i1.DB, ShouldEqual, "myDB")
So(i2.DB, ShouldEqual, "myDB")
})
Convey("with all the proper metadata + bson merges", func() {
So(i0.Location, ShouldNotEqual, "")
So(i0.MetadataLocation, ShouldNotEqual, "")
So(i1.Location, ShouldNotEqual, "")
So(i1.MetadataLocation, ShouldEqual, "") //no metadata for this file
So(i2.Location, ShouldNotEqual, "")
So(i2.MetadataLocation, ShouldNotEqual, "")
Convey("and skipped files all present in the logs", func() {
logs := buff.String()
So(strings.Contains(logs, "baddir"), ShouldEqual, true)
})
})
})
})
})
}
func TestCreateIntentsForCollection(t *testing.T) {
var mr *MongoRestore
var buff bytes.Buffer
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a test MongoRestore", t, func() {
buff = bytes.Buffer{}
mr = &MongoRestore{
manager: intents.NewIntentManager(),
ToolOptions: &commonOpts.ToolOptions{Namespace: &commonOpts.Namespace{}},
InputOptions: &InputOptions{},
}
log.SetWriter(&buff)
Convey("running CreateIntentForCollection on a file without metadata", func() {
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection("myDB", "myC", ddl)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("should create one intent with 'myDb' and 'myC' fields", func() {
i0 := mr.manager.Pop()
So(i0, ShouldNotBeNil)
So(i0.DB, ShouldEqual, "myDB")
So(i0.C, ShouldEqual, "myC")
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c2.bson"))
So(err, ShouldBeNil)
So(i0.Location, ShouldEqual, ddl.Path())
i1 := mr.manager.Pop()
So(i1, ShouldBeNil)
Convey("and no Metadata path", func() {
So(i0.MetadataLocation, ShouldEqual, "")
logs := buff.String()
So(strings.Contains(logs, "without metadata"), ShouldEqual, true)
})
})
})
Convey("running CreateIntentForCollection on a file *with* metadata", func() {
ddl, err := newActualPath(util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection("myDB", "myC", ddl)
So(err, ShouldBeNil)
mr.manager.Finalize(intents.Legacy)
Convey("should create one intent with 'myDb' and 'myC' fields", func() {
i0 := mr.manager.Pop()
So(i0, ShouldNotBeNil)
So(i0.DB, ShouldEqual, "myDB")
So(i0.C, ShouldEqual, "myC")
So(i0.Location, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.bson"))
i1 := mr.manager.Pop()
So(i1, ShouldBeNil)
Convey("and a set Metadata path", func() {
So(i0.MetadataLocation, ShouldEqual, util.ToUniversalPath("testdata/testdirs/db1/c1.metadata.json"))
logs := buff.String()
So(strings.Contains(logs, "found metadata"), ShouldEqual, true)
})
})
})
Convey("running CreateIntentForCollection on a non-existent file", func() {
_, err := newActualPath("aaaaaaaaaaaaaa.bson")
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
Convey("running CreateIntentForCollection on a directory", func() {
ddl, err := newActualPath("testdata")
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection(
"myDB", "myC", ddl)
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
Convey("running CreateIntentForCollection on non-bson file", func() {
ddl, err := newActualPath("testdata/testdirs/db1/c1.metadata.json")
So(err, ShouldBeNil)
err = mr.CreateIntentForCollection(
"myDB", "myC", ddl)
Convey("should fail", func() {
So(err, ShouldNotBeNil)
})
})
})
}
func TestGetDumpAuthVersion(t *testing.T) {
testutil.VerifyTestType(t, testutil.UnitTestType)
restore := &MongoRestore{}
Convey("With a test mongorestore", t, func() {
Convey("and no --restoreDbUsersAndRoles", func() {
restore = &MongoRestore{
InputOptions: &InputOptions{},
ToolOptions: &commonOpts.ToolOptions{},
NSOptions: &NSOptions{},
}
Convey("auth version 1 should be detected", func() {
restore.manager = intents.NewIntentManager()
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 1)
})
Convey("auth version 3 should be detected", func() {
restore.manager = intents.NewIntentManager()
intent := &intents.Intent{
DB: "admin",
C: "system.version",
Location: "testdata/auth_version_3.bson",
}
intent.BSONFile = &realBSONFile{path: "testdata/auth_version_3.bson", intent: intent}
restore.manager.Put(intent)
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 3)
})
Convey("auth version 5 should be detected", func() {
restore.manager = intents.NewIntentManager()
intent := &intents.Intent{
DB: "admin",
C: "system.version",
Location: "testdata/auth_version_5.bson",
}
intent.BSONFile = &realBSONFile{path: "testdata/auth_version_5.bson", intent: intent}
restore.manager.Put(intent)
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 5)
})
})
Convey("using --restoreDbUsersAndRoles", func() {
restore = &MongoRestore{
InputOptions: &InputOptions{
RestoreDBUsersAndRoles: true,
},
ToolOptions: &commonOpts.ToolOptions{},
NSOptions: &NSOptions{
DB: "TestDB",
},
}
Convey("auth version 3 should be detected when no file exists", func() {
restore.manager = intents.NewIntentManager()
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 3)
})
Convey("auth version 3 should be detected when a version 3 file exists", func() {
restore.manager = intents.NewIntentManager()
intent := &intents.Intent{
DB: "admin",
C: "system.version",
Location: "testdata/auth_version_3.bson",
}
intent.BSONFile = &realBSONFile{path: "testdata/auth_version_3.bson", intent: intent}
restore.manager.Put(intent)
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 3)
})
Convey("auth version 5 should be detected", func() {
restore.manager = intents.NewIntentManager()
intent := &intents.Intent{
DB: "admin",
C: "system.version",
Location: "testdata/auth_version_5.bson",
}
intent.BSONFile = &realBSONFile{path: "testdata/auth_version_5.bson", intent: intent}
restore.manager.Put(intent)
version, err := restore.GetDumpAuthVersion()
So(err, ShouldBeNil)
So(version, ShouldEqual, 5)
})
})
})
}
// Restore runs the mongorestore program.
func (restore *MongoRestore) Restore() error {
var target archive.DirLike
err := restore.ParseAndValidateOptions()
if err != nil {
log.Logvf(log.DebugLow, "got error from options parsing: %v", err)
return err
}
// Build up all intents to be restored
restore.manager = intents.NewIntentManager()
if restore.InputOptions.Archive == "" && restore.InputOptions.OplogReplay {
restore.manager.SetSmartPickOplog(true)
}
if restore.InputOptions.Archive != "" {
archiveReader, err := restore.getArchiveReader()
if err != nil {
return err
}
restore.archive = &archive.Reader{
In: archiveReader,
Prelude: &archive.Prelude{},
}
err = restore.archive.Prelude.Read(restore.archive.In)
if err != nil {
return err
}
log.Logvf(log.DebugLow, `archive format version "%v"`, restore.archive.Prelude.Header.FormatVersion)
log.Logvf(log.DebugLow, `archive server version "%v"`, restore.archive.Prelude.Header.ServerVersion)
log.Logvf(log.DebugLow, `archive tool version "%v"`, restore.archive.Prelude.Header.ToolVersion)
target, err = restore.archive.Prelude.NewPreludeExplorer()
if err != nil {
return err
}
} else if restore.TargetDirectory != "-" {
var usedDefaultTarget bool
if restore.TargetDirectory == "" {
restore.TargetDirectory = "dump"
log.Logv(log.Always, "using default 'dump' directory")
usedDefaultTarget = true
}
target, err = newActualPath(restore.TargetDirectory)
if err != nil {
if usedDefaultTarget {
log.Logv(log.Always, "see mongorestore --help for usage information")
}
return fmt.Errorf("mongorestore target '%v' invalid: %v", restore.TargetDirectory, err)
}
// handle cases where the user passes in a file instead of a directory
if !target.IsDir() {
log.Logv(log.DebugLow, "mongorestore target is a file, not a directory")
err = restore.handleBSONInsteadOfDirectory(restore.TargetDirectory)
if err != nil {
return err
}
} else {
log.Logv(log.DebugLow, "mongorestore target is a directory, not a file")
}
}
if restore.NSOptions.Collection != "" &&
restore.OutputOptions.NumParallelCollections > 1 &&
restore.OutputOptions.NumInsertionWorkers == 1 {
// handle special parallelization case when we are only restoring one collection
// by mapping -j to insertion workers rather than parallel collections
log.Logvf(log.DebugHigh,
"setting number of insertions workers to number of parallel collections (%v)",
restore.OutputOptions.NumParallelCollections)
restore.OutputOptions.NumInsertionWorkers = restore.OutputOptions.NumParallelCollections
}
if restore.InputOptions.Archive != "" {
if int(restore.archive.Prelude.Header.ConcurrentCollections) > restore.OutputOptions.NumParallelCollections {
restore.OutputOptions.NumParallelCollections = int(restore.archive.Prelude.Header.ConcurrentCollections)
restore.OutputOptions.NumInsertionWorkers = int(restore.archive.Prelude.Header.ConcurrentCollections)
log.Logvf(log.Always,
"setting number of parallel collections to number of parallel collections in archive (%v)",
restore.archive.Prelude.Header.ConcurrentCollections,
)
}
}
// Create the demux before intent creation, because muted archive intents need
// to register themselves with the demux directly
if restore.InputOptions.Archive != "" {
restore.archive.Demux = &archive.Demultiplexer{
In: restore.archive.In,
}
}
switch {
case restore.InputOptions.Archive != "":
log.Logvf(log.Always, "preparing collections to restore from")
err = restore.CreateAllIntents(target)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection == "":
log.Logvf(log.Always,
"building a list of collections to restore from %v dir",
target.Path())
err = restore.CreateIntentsForDB(
restore.NSOptions.DB,
target,
)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "" && restore.TargetDirectory == "-":
log.Logvf(log.Always, "setting up a collection to be read from standard input")
err = restore.CreateStdinIntentForCollection(
restore.NSOptions.DB,
restore.NSOptions.Collection,
)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "":
log.Logvf(log.Always, "checking for collection data in %v", target.Path())
err = restore.CreateIntentForCollection(
restore.NSOptions.DB,
restore.NSOptions.Collection,
target,
)
default:
log.Logvf(log.Always, "preparing collections to restore from")
err = restore.CreateAllIntents(target)
}
if err != nil {
return fmt.Errorf("error scanning filesystem: %v", err)
}
if restore.isMongos && restore.manager.HasConfigDBIntent() && restore.NSOptions.DB == "" {
return fmt.Errorf("cannot do a full restore on a sharded system - " +
"remove the 'config' directory from the dump directory first")
}
if restore.InputOptions.OplogFile != "" {
err = restore.CreateIntentForOplog()
if err != nil {
return fmt.Errorf("error reading oplog file: %v", err)
}
}
if restore.InputOptions.OplogReplay && restore.manager.Oplog() == nil {
return fmt.Errorf("no oplog file to replay; make sure you run mongodump with --oplog")
}
if restore.manager.GetOplogConflict() {
return fmt.Errorf("cannot provide both an oplog.bson file and an oplog file with --oplogFile, " +
"nor can you provide both a local/oplog.rs.bson and a local/oplog.$main.bson file.")
}
conflicts := restore.manager.GetDestinationConflicts()
if len(conflicts) > 0 {
for _, conflict := range conflicts {
log.Logvf(log.Always, "%s", conflict.Error())
}
return fmt.Errorf("cannot restore with conflicting namespace destinations")
}
if restore.OutputOptions.DryRun {
log.Logvf(log.Always, "dry run completed")
return nil
}
if restore.InputOptions.Archive != "" {
namespaceChan := make(chan string, 1)
namespaceErrorChan := make(chan error)
restore.archive.Demux.NamespaceChan = namespaceChan
restore.archive.Demux.NamespaceErrorChan = namespaceErrorChan
go restore.archive.Demux.Run()
// consume the new namespace announcement from the demux for all of the special collections
// that get cached when being read out of the archive.
// The first regular collection found gets pushed back on to the namespaceChan
// consume the new namespace announcement from the demux for all of the collections that get cached
for {
ns, ok := <-namespaceChan
// the archive can have only special collections. In that case we keep reading until
// the namespaces are exhausted, indicated by the namespaceChan being closed.
if !ok {
break
}
intent := restore.manager.IntentForNamespace(ns)
if intent == nil {
return fmt.Errorf("no intent for collection in archive: %v", ns)
}
if intent.IsSystemIndexes() ||
intent.IsUsers() ||
intent.IsRoles() ||
intent.IsAuthVersion() {
log.Logvf(log.DebugLow, "special collection %v found", ns)
namespaceErrorChan <- nil
} else {
// Put the ns back on the announcement chan so that the
// demultiplexer can start correctly
log.Logvf(log.DebugLow, "first non special collection %v found."+
" The demultiplexer will handle it and the remainder", ns)
namespaceChan <- ns
break
}
}
}
// If restoring users and roles, make sure we validate auth versions
if restore.ShouldRestoreUsersAndRoles() {
log.Logv(log.Info, "comparing auth version of the dump directory and target server")
restore.authVersions.Dump, err = restore.GetDumpAuthVersion()
if err != nil {
return fmt.Errorf("error getting auth version from dump: %v", err)
}
restore.authVersions.Server, err = auth.GetAuthVersion(restore.SessionProvider)
if err != nil {
return fmt.Errorf("error getting auth version of server: %v", err)
}
err = restore.ValidateAuthVersions()
if err != nil {
return fmt.Errorf(
"the users and roles collections in the dump have an incompatible auth version with target server: %v",
err)
}
}
err = restore.LoadIndexesFromBSON()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
// Restore the regular collections
if restore.InputOptions.Archive != "" {
restore.manager.UsePrioritizer(restore.archive.Demux.NewPrioritizer(restore.manager))
} else if restore.OutputOptions.NumParallelCollections > 1 {
restore.manager.Finalize(intents.MultiDatabaseLTF)
} else {
// use legacy restoration order if we are single-threaded
restore.manager.Finalize(intents.Legacy)
}
restore.termChan = make(chan struct{})
if err := restore.RestoreIntents(); err != nil {
return err
}
// Restore users/roles
if restore.ShouldRestoreUsersAndRoles() {
err = restore.RestoreUsersOrRoles(restore.manager.Users(), restore.manager.Roles())
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
// Restore oplog
if restore.InputOptions.OplogReplay {
err = restore.RestoreOplog()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
log.Logv(log.Always, "done")
return nil
}