// Dump handles some final options checking and executes MongoDump func (dump *MongoDump) Dump() error { err := dump.ValidateOptions() if err != nil { return fmt.Errorf("Bad Option: %v", err) } if dump.InputOptions.Query != "" { // TODO, check for extended json support... // gonna need to do some exploring later on, since im 95% sure // this is undefined in the current tools err = json.Unmarshal([]byte(dump.InputOptions.Query), &dump.query) if err != nil { return fmt.Errorf("error parsing query: %v", err) } } if dump.OutputOptions.Out == "-" { dump.useStdout = true } if dump.OutputOptions.DumpDBUsersAndRoles { //first make sure this is possible with the connected database dump.authVersion, err = auth.GetAuthVersion(dump.SessionProvider.GetSession()) if err != nil { return fmt.Errorf("error getting auth schema version for dumpDbUsersAndRoles: %v", err) } log.Logf(2, "using auth schema version %v", dump.authVersion) if dump.authVersion != 3 { return fmt.Errorf("backing up users and roles is only supported for "+ "deployments with auth schema versions 3, found: %v", dump.authVersion) } } //switch on what kind of execution to do switch { case dump.ToolOptions.DB == "" && dump.ToolOptions.Collection == "": err = dump.DumpEverything() case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection == "": err = dump.DumpDatabase(dump.ToolOptions.DB) case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection != "": err = dump.DumpCollection(dump.ToolOptions.DB, dump.ToolOptions.Collection) } if dump.OutputOptions.DumpDBUsersAndRoles { log.Logf(0, "dumping users and roles for %v", dump.ToolOptions.DB) if dump.ToolOptions.DB == "admin" { log.Logf(0, "skipping users/roles dump, already dumped admin database") } else { err = dump.DumpUsersAndRolesForDB(dump.ToolOptions.DB) if err != nil { return fmt.Errorf("error dumping users and roles: %v", err) } } } log.Logf(1, "done") return err }
// Dump handles some final options checking and executes MongoDump. func (dump *MongoDump) Dump() (err error) { defer dump.sessionProvider.Close() dump.shutdownIntentsNotifier = newNotifier() if dump.InputOptions.HasQuery() { // parse JSON then convert extended JSON values var asJSON interface{} content, err := dump.InputOptions.GetQuery() if err != nil { return err } err = json.Unmarshal(content, &asJSON) if err != nil { return fmt.Errorf("error parsing query as json: %v", err) } convertedJSON, err := bsonutil.ConvertJSONValueToBSON(asJSON) if err != nil { return fmt.Errorf("error converting query to bson: %v", err) } asMap, ok := convertedJSON.(map[string]interface{}) if !ok { // unlikely to be reached return fmt.Errorf("query is not in proper format") } dump.query = bson.M(asMap) } if dump.OutputOptions.DumpDBUsersAndRoles { // first make sure this is possible with the connected database dump.authVersion, err = auth.GetAuthVersion(dump.sessionProvider) if err == nil { err = auth.VerifySystemAuthVersion(dump.sessionProvider) } if err != nil { return fmt.Errorf("error getting auth schema version for dumpDbUsersAndRoles: %v", err) } log.Logvf(log.DebugLow, "using auth schema version %v", dump.authVersion) if dump.authVersion < 3 { return fmt.Errorf("backing up users and roles is only supported for "+ "deployments with auth schema versions >= 3, found: %v", dump.authVersion) } } if dump.OutputOptions.Archive != "" { //getArchiveOut gives us a WriteCloser to which we should write the archive var archiveOut io.WriteCloser archiveOut, err = dump.getArchiveOut() if err != nil { return err } dump.archive = &archive.Writer{ // The archive.Writer needs its own copy of archiveOut because things // like the prelude are not written by the multiplexer. Out: archiveOut, Mux: archive.NewMultiplexer(archiveOut, dump.shutdownIntentsNotifier), } go dump.archive.Mux.Run() defer func() { // The Mux runs until its Control is closed close(dump.archive.Mux.Control) muxErr := <-dump.archive.Mux.Completed archiveOut.Close() if muxErr != nil { if err != nil { err = fmt.Errorf("archive writer: %v / %v", err, muxErr) } else { err = fmt.Errorf("archive writer: %v", muxErr) } log.Logvf(log.DebugLow, "%v", err) } else { log.Logvf(log.DebugLow, "mux completed successfully") } }() } // switch on what kind of execution to do switch { case dump.ToolOptions.DB == "" && dump.ToolOptions.Collection == "": err = dump.CreateAllIntents() case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection == "": err = dump.CreateIntentsForDatabase(dump.ToolOptions.DB) case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection != "": err = dump.CreateCollectionIntent(dump.ToolOptions.DB, dump.ToolOptions.Collection) } if err != nil { return err } if dump.OutputOptions.Oplog { err = dump.CreateOplogIntents() if err != nil { return err } } if dump.OutputOptions.DumpDBUsersAndRoles && dump.ToolOptions.DB != "admin" { err = dump.CreateUsersRolesVersionIntentsForDB(dump.ToolOptions.DB) if err != nil { return err } } // verify we can use repair cursors if dump.OutputOptions.Repair { log.Logv(log.DebugLow, "verifying that the connected server supports repairCursor") if dump.isMongos { return fmt.Errorf("cannot use --repair on mongos") } exampleIntent := dump.manager.Peek() if exampleIntent != nil { supported, err := dump.sessionProvider.SupportsRepairCursor( exampleIntent.DB, exampleIntent.C) if !supported { return err // no extra context needed } } } // IO Phase I // metadata, users, roles, and versions // TODO, either remove this debug or improve the language log.Logvf(log.DebugHigh, "dump phase I: metadata, indexes, users, roles, version") err = dump.DumpMetadata() if err != nil { return fmt.Errorf("error dumping metadata: %v", err) } if dump.OutputOptions.Archive != "" { session, err := dump.sessionProvider.GetSession() if err != nil { return err } defer session.Close() buildInfo, err := session.BuildInfo() var serverVersion string if err != nil { log.Logvf(log.Always, "warning, couldn't get version information from server: %v", err) serverVersion = "unknown" } else { serverVersion = buildInfo.Version } dump.archive.Prelude, err = archive.NewPrelude(dump.manager, dump.OutputOptions.NumParallelCollections, serverVersion) if err != nil { return fmt.Errorf("creating archive prelude: %v", err) } err = dump.archive.Prelude.Write(dump.archive.Out) if err != nil { return fmt.Errorf("error writing metadata into archive: %v", err) } } err = dump.DumpSystemIndexes() if err != nil { return fmt.Errorf("error dumping system indexes: %v", err) } if dump.ToolOptions.DB == "admin" || dump.ToolOptions.DB == "" { err = dump.DumpUsersAndRoles() if err != nil { return fmt.Errorf("error dumping users and roles: %v", err) } } if dump.OutputOptions.DumpDBUsersAndRoles { log.Logvf(log.Always, "dumping users and roles for %v", dump.ToolOptions.DB) if dump.ToolOptions.DB == "admin" { log.Logvf(log.Always, "skipping users/roles dump, already dumped admin database") } else { err = dump.DumpUsersAndRolesForDB(dump.ToolOptions.DB) if err != nil { return fmt.Errorf("error dumping users and roles for db: %v", err) } } } // If oplog capturing is enabled, we first check the most recent // oplog entry and save its timestamp, this will let us later // copy all oplog entries that occurred while dumping, creating // what is effectively a point-in-time snapshot. if dump.OutputOptions.Oplog { err := dump.determineOplogCollectionName() if err != nil { return fmt.Errorf("error finding oplog: %v", err) } log.Logvf(log.Info, "getting most recent oplog timestamp") dump.oplogStart, err = dump.getOplogStartTime() if err != nil { return fmt.Errorf("error getting oplog start: %v", err) } } if failpoint.Enabled(failpoint.PauseBeforeDumping) { time.Sleep(15 * time.Second) } // IO Phase II // regular collections // TODO, either remove this debug or improve the language log.Logvf(log.DebugHigh, "dump phase II: regular collections") // begin dumping intents if err := dump.DumpIntents(); err != nil { return err } // IO Phase III // oplog // TODO, either remove this debug or improve the language log.Logvf(log.DebugLow, "dump phase III: the oplog") // If we are capturing the oplog, we dump all oplog entries that occurred // while dumping the database. Before and after dumping the oplog, // we check to see if the oplog has rolled over (i.e. the most recent entry when // we started still exist, so we know we haven't lost data) if dump.OutputOptions.Oplog { log.Logvf(log.DebugLow, "checking if oplog entry %v still exists", dump.oplogStart) exists, err := dump.checkOplogTimestampExists(dump.oplogStart) if !exists { return fmt.Errorf( "oplog overflow: mongodump was unable to capture all new oplog entries during execution") } if err != nil { return fmt.Errorf("unable to check oplog for overflow: %v", err) } log.Logvf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart) log.Logvf(log.Always, "writing captured oplog to %v", dump.manager.Oplog().Location) err = dump.DumpOplogAfterTimestamp(dump.oplogStart) if err != nil { return fmt.Errorf("error dumping oplog: %v", err) } // check the oplog for a rollover one last time, to avoid a race condition // wherein the oplog rolls over in the time after our first check, but before // we copy it. log.Logvf(log.DebugLow, "checking again if oplog entry %v still exists", dump.oplogStart) exists, err = dump.checkOplogTimestampExists(dump.oplogStart) if !exists { return fmt.Errorf( "oplog overflow: mongodump was unable to capture all new oplog entries during execution") } if err != nil { return fmt.Errorf("unable to check oplog for overflow: %v", err) } log.Logvf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart) } log.Logvf(log.DebugLow, "finishing dump") return err }
// Restore runs the mongorestore program. func (restore *MongoRestore) Restore() error { var target archive.DirLike err := restore.ParseAndValidateOptions() if err != nil { log.Logvf(log.DebugLow, "got error from options parsing: %v", err) return err } // Build up all intents to be restored restore.manager = intents.NewIntentManager() if restore.InputOptions.Archive == "" && restore.InputOptions.OplogReplay { restore.manager.SetSmartPickOplog(true) } if restore.InputOptions.Archive != "" { archiveReader, err := restore.getArchiveReader() if err != nil { return err } restore.archive = &archive.Reader{ In: archiveReader, Prelude: &archive.Prelude{}, } err = restore.archive.Prelude.Read(restore.archive.In) if err != nil { return err } log.Logvf(log.DebugLow, `archive format version "%v"`, restore.archive.Prelude.Header.FormatVersion) log.Logvf(log.DebugLow, `archive server version "%v"`, restore.archive.Prelude.Header.ServerVersion) log.Logvf(log.DebugLow, `archive tool version "%v"`, restore.archive.Prelude.Header.ToolVersion) target, err = restore.archive.Prelude.NewPreludeExplorer() if err != nil { return err } } else if restore.TargetDirectory != "-" { var usedDefaultTarget bool if restore.TargetDirectory == "" { restore.TargetDirectory = "dump" log.Logv(log.Always, "using default 'dump' directory") usedDefaultTarget = true } target, err = newActualPath(restore.TargetDirectory) if err != nil { if usedDefaultTarget { log.Logv(log.Always, "see mongorestore --help for usage information") } return fmt.Errorf("mongorestore target '%v' invalid: %v", restore.TargetDirectory, err) } // handle cases where the user passes in a file instead of a directory if !target.IsDir() { log.Logv(log.DebugLow, "mongorestore target is a file, not a directory") err = restore.handleBSONInsteadOfDirectory(restore.TargetDirectory) if err != nil { return err } } else { log.Logv(log.DebugLow, "mongorestore target is a directory, not a file") } } if restore.NSOptions.Collection != "" && restore.OutputOptions.NumParallelCollections > 1 && restore.OutputOptions.NumInsertionWorkers == 1 { // handle special parallelization case when we are only restoring one collection // by mapping -j to insertion workers rather than parallel collections log.Logvf(log.DebugHigh, "setting number of insertions workers to number of parallel collections (%v)", restore.OutputOptions.NumParallelCollections) restore.OutputOptions.NumInsertionWorkers = restore.OutputOptions.NumParallelCollections } if restore.InputOptions.Archive != "" { if int(restore.archive.Prelude.Header.ConcurrentCollections) > restore.OutputOptions.NumParallelCollections { restore.OutputOptions.NumParallelCollections = int(restore.archive.Prelude.Header.ConcurrentCollections) restore.OutputOptions.NumInsertionWorkers = int(restore.archive.Prelude.Header.ConcurrentCollections) log.Logvf(log.Always, "setting number of parallel collections to number of parallel collections in archive (%v)", restore.archive.Prelude.Header.ConcurrentCollections, ) } } // Create the demux before intent creation, because muted archive intents need // to register themselves with the demux directly if restore.InputOptions.Archive != "" { restore.archive.Demux = &archive.Demultiplexer{ In: restore.archive.In, } } switch { case restore.InputOptions.Archive != "": log.Logvf(log.Always, "preparing collections to restore from") err = restore.CreateAllIntents(target) case restore.NSOptions.DB != "" && restore.NSOptions.Collection == "": log.Logvf(log.Always, "building a list of collections to restore from %v dir", target.Path()) err = restore.CreateIntentsForDB( restore.NSOptions.DB, target, ) case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "" && restore.TargetDirectory == "-": log.Logvf(log.Always, "setting up a collection to be read from standard input") err = restore.CreateStdinIntentForCollection( restore.NSOptions.DB, restore.NSOptions.Collection, ) case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "": log.Logvf(log.Always, "checking for collection data in %v", target.Path()) err = restore.CreateIntentForCollection( restore.NSOptions.DB, restore.NSOptions.Collection, target, ) default: log.Logvf(log.Always, "preparing collections to restore from") err = restore.CreateAllIntents(target) } if err != nil { return fmt.Errorf("error scanning filesystem: %v", err) } if restore.isMongos && restore.manager.HasConfigDBIntent() && restore.NSOptions.DB == "" { return fmt.Errorf("cannot do a full restore on a sharded system - " + "remove the 'config' directory from the dump directory first") } if restore.InputOptions.OplogFile != "" { err = restore.CreateIntentForOplog() if err != nil { return fmt.Errorf("error reading oplog file: %v", err) } } if restore.InputOptions.OplogReplay && restore.manager.Oplog() == nil { return fmt.Errorf("no oplog file to replay; make sure you run mongodump with --oplog") } if restore.manager.GetOplogConflict() { return fmt.Errorf("cannot provide both an oplog.bson file and an oplog file with --oplogFile, " + "nor can you provide both a local/oplog.rs.bson and a local/oplog.$main.bson file.") } conflicts := restore.manager.GetDestinationConflicts() if len(conflicts) > 0 { for _, conflict := range conflicts { log.Logvf(log.Always, "%s", conflict.Error()) } return fmt.Errorf("cannot restore with conflicting namespace destinations") } if restore.OutputOptions.DryRun { log.Logvf(log.Always, "dry run completed") return nil } if restore.InputOptions.Archive != "" { namespaceChan := make(chan string, 1) namespaceErrorChan := make(chan error) restore.archive.Demux.NamespaceChan = namespaceChan restore.archive.Demux.NamespaceErrorChan = namespaceErrorChan go restore.archive.Demux.Run() // consume the new namespace announcement from the demux for all of the special collections // that get cached when being read out of the archive. // The first regular collection found gets pushed back on to the namespaceChan // consume the new namespace announcement from the demux for all of the collections that get cached for { ns, ok := <-namespaceChan // the archive can have only special collections. In that case we keep reading until // the namespaces are exhausted, indicated by the namespaceChan being closed. if !ok { break } intent := restore.manager.IntentForNamespace(ns) if intent == nil { return fmt.Errorf("no intent for collection in archive: %v", ns) } if intent.IsSystemIndexes() || intent.IsUsers() || intent.IsRoles() || intent.IsAuthVersion() { log.Logvf(log.DebugLow, "special collection %v found", ns) namespaceErrorChan <- nil } else { // Put the ns back on the announcement chan so that the // demultiplexer can start correctly log.Logvf(log.DebugLow, "first non special collection %v found."+ " The demultiplexer will handle it and the remainder", ns) namespaceChan <- ns break } } } // If restoring users and roles, make sure we validate auth versions if restore.ShouldRestoreUsersAndRoles() { log.Logv(log.Info, "comparing auth version of the dump directory and target server") restore.authVersions.Dump, err = restore.GetDumpAuthVersion() if err != nil { return fmt.Errorf("error getting auth version from dump: %v", err) } restore.authVersions.Server, err = auth.GetAuthVersion(restore.SessionProvider) if err != nil { return fmt.Errorf("error getting auth version of server: %v", err) } err = restore.ValidateAuthVersions() if err != nil { return fmt.Errorf( "the users and roles collections in the dump have an incompatible auth version with target server: %v", err) } } err = restore.LoadIndexesFromBSON() if err != nil { return fmt.Errorf("restore error: %v", err) } // Restore the regular collections if restore.InputOptions.Archive != "" { restore.manager.UsePrioritizer(restore.archive.Demux.NewPrioritizer(restore.manager)) } else if restore.OutputOptions.NumParallelCollections > 1 { restore.manager.Finalize(intents.MultiDatabaseLTF) } else { // use legacy restoration order if we are single-threaded restore.manager.Finalize(intents.Legacy) } restore.termChan = make(chan struct{}) if err := restore.RestoreIntents(); err != nil { return err } // Restore users/roles if restore.ShouldRestoreUsersAndRoles() { err = restore.RestoreUsersOrRoles(restore.manager.Users(), restore.manager.Roles()) if err != nil { return fmt.Errorf("restore error: %v", err) } } // Restore oplog if restore.InputOptions.OplogReplay { err = restore.RestoreOplog() if err != nil { return fmt.Errorf("restore error: %v", err) } } log.Logv(log.Always, "done") return nil }