// CreateIntentForCollection builds an intent for the given database and collection name // along with a path to a .bson collection file. It searches the file's parent directory // for a matching metadata file. // // This method is not called by CreateIntentsForDB, // it is only used in the case where --db and --collection flags are set. func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, dir archive.DirLike) error { log.Logf(log.DebugLow, "reading collection %v for database %v from %v", collection, db, dir.Path()) // first make sure the bson file exists and is valid _, err := dir.Stat() if err != nil { return err } if dir.IsDir() { return fmt.Errorf("file %v is a directory, not a bson file", dir.Path()) } baseName, fileType := restore.getInfoFromFilename(dir.Name()) if fileType != BSONFileType { return fmt.Errorf("file %v does not have .bson extension", dir.Path()) } // then create its intent intent := &intents.Intent{ DB: db, C: collection, Size: dir.Size(), Location: dir.Path(), } intent.BSONFile = &realBSONFile{path: dir.Path(), intent: intent, gzip: restore.InputOptions.Gzip} // finally, check if it has a .metadata.json file in its folder log.Logf(log.DebugLow, "scanning directory %v for metadata", dir.Name()) entries, err := dir.Parent().ReadDir() if err != nil { // try and carry on if we can log.Logf(log.Info, "error attempting to locate metadata for file: %v", err) log.Log(log.Info, "restoring collection without metadata") restore.manager.Put(intent) return nil } metadataName := baseName + ".metadata.json" if restore.InputOptions.Gzip { metadataName += ".gz" } for _, entry := range entries { if entry.Name() == metadataName { metadataPath := entry.Path() log.Logf(log.Info, "found metadata for collection at %v", metadataPath) intent.MetadataLocation = metadataPath intent.MetadataFile = &realMetadataFile{path: metadataPath, intent: intent, gzip: restore.InputOptions.Gzip} break } } if intent.MetadataFile == nil { log.Log(log.Info, "restoring collection without metadata") } restore.manager.Put(intent) return nil }
// HandleSignals listens for either SIGTERM, SIGINT or the // SIGHUP signal. It ends restore reads for all goroutines // as soon as any of those signals is received. func HandleSignals(dump *MongoDump) { log.Log(log.DebugLow, "will listen for SIGTERM, SIGINT and SIGHUP") sigChan := make(chan os.Signal, 2) signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) // first signal cleanly terminates dump writes <-sigChan log.Log(log.Always, "ending dump writes") close(dump.termChan) // second signal exits immediately <-sigChan log.Log(log.Always, "forcefully terminating mongodump") os.Exit(util.ExitKill) }
// handleSignals listens for either SIGTERM, SIGINT or the // SIGHUP signal. It ends restore reads for all goroutines // as soon as any of those signals is received. func (restore *MongoRestore) handleSignals() { log.Log(log.DebugLow, "will listen for SIGTERM, SIGINT and SIGHUP") sigChan := make(chan os.Signal, 2) signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) // first signal cleanly terminates restore reads <-sigChan log.Log(log.Always, "ending restore reads") close(restore.termChan) // second signal exits immediately <-sigChan log.Log(log.Always, "forcefully terminating mongorestore") os.Exit(util.ExitKill) }
// Prompt displays a prompt asking for the password and returns the // password the user enters as a string. func Prompt() string { var pass string if IsTerminal() { log.Log(log.DebugLow, "standard input is a terminal; reading password from terminal") fmt.Fprintf(os.Stderr, "Enter password:"******"reading password from standard input") fmt.Fprintf(os.Stderr, "Enter password:") pass = readPassFromStdin() } fmt.Fprintln(os.Stderr) return pass }
// getTargetDirFromArgs handles the logic and error cases of figuring out // the target restore directory. func getTargetDirFromArgs(extraArgs []string, dirFlag string) (string, error) { // This logic is in a switch statement so that the rules are understandable. // We start by handling error cases, and then handle the different ways the target // directory can be legally set. switch { case len(extraArgs) > 1: // error on cases when there are too many positional arguments return "", fmt.Errorf("too many positional arguments") case dirFlag != "" && len(extraArgs) > 0: // error when positional arguments and --dir are used return "", fmt.Errorf( "cannot use both --dir and a positional argument to set the target directory") case len(extraArgs) == 1: // a nice, simple case where one argument is given, so we use it return extraArgs[0], nil case dirFlag != "": // if we have no extra args and a --dir flag, use the --dir flag log.Log(log.Info, "using --dir flag instead of arguments") return dirFlag, nil default: return "", nil } }
// Finalize processes the intents for prioritization. Currently only two // kinds of prioritizers are supported. No more "Put" operations may be done // after finalize is called. func (manager *Manager) Finalize(pType PriorityType) { switch pType { case Legacy: log.Log(log.DebugHigh, "finalizing intent manager with legacy prioritizer") manager.prioritizer = NewLegacyPrioritizer(manager.intentsByDiscoveryOrder) case LongestTaskFirst: log.Log(log.DebugHigh, "finalizing intent manager with longest task first prioritizer") manager.prioritizer = NewLongestTaskFirstPrioritizer(manager.intentsByDiscoveryOrder) case MultiDatabaseLTF: log.Log(log.DebugHigh, "finalizing intent manager with multi-database longest task first prioritizer") manager.prioritizer = NewMultiDatabaseLTFPrioritizer(manager.intentsByDiscoveryOrder) default: panic("cannot initialize IntentPrioritizer with unknown type") } // release these for the garbage collector and to ensure code correctness manager.intents = nil manager.intentsByDiscoveryOrder = nil }
// ValidateSettings returns an error if any settings specified on the command line // were invalid, or nil if they are valid. func (exp *MongoExport) ValidateSettings() error { // Namespace must have a valid database if none is specified, // use 'test' if exp.ToolOptions.Namespace.DB == "" { exp.ToolOptions.Namespace.DB = "test" } if exp.ToolOptions.Namespace.Collection == "" { return fmt.Errorf("must specify a collection") } if err := util.ValidateCollectionName(exp.ToolOptions.Namespace.Collection); err != nil { return err } exp.OutputOpts.Type = strings.ToLower(exp.OutputOpts.Type) if exp.ToolOptions.HiddenOptions.CSVOutputType { log.Log(log.Always, "csv flag is deprecated; please use --type=csv instead") exp.OutputOpts.Type = CSV } if exp.OutputOpts.Type == "" { // special error for an empty type value return fmt.Errorf("--type cannot be empty") } if exp.OutputOpts.Type != CSV && exp.OutputOpts.Type != JSON { return fmt.Errorf("invalid output type '%v', choose 'json' or 'csv'", exp.OutputOpts.Type) } if exp.InputOpts.Query != "" && exp.InputOpts.ForceTableScan { return fmt.Errorf("cannot use --forceTableScan when specifying --query") } if exp.InputOpts.Query != "" && exp.InputOpts.QueryFile != "" { return fmt.Errorf("either --query or --queryFile can be specified as a query option") } if exp.InputOpts != nil && exp.InputOpts.HasQuery() { content, err := exp.InputOpts.GetQuery() if err != nil { return err } _, err2 := getObjectFromByteArg(content) if err2 != nil { return err2 } } if exp.InputOpts != nil && exp.InputOpts.Sort != "" { _, err := getSortFromArg(exp.InputOpts.Sort) if err != nil { return err } } return nil }
// CreateIndexes takes in an intent and an array of index documents and // attempts to create them using the createIndexes command. If that command // fails, we fall back to individual index creation. func (restore *MongoRestore) CreateIndexes(intent *intents.Intent, indexes []IndexDocument) error { // first, sanitize the indexes for _, index := range indexes { // update the namespace of the index before inserting index.Options["ns"] = intent.Namespace() // check for length violations before building the command fullIndexName := fmt.Sprintf("%v.$%v", index.Options["ns"], index.Options["name"]) if len(fullIndexName) > 127 { return fmt.Errorf( "cannot restore index with namespace '%v': "+ "namespace is too long (max size is 127 bytes)", fullIndexName) } // remove the index version, forcing an update, // unless we specifically want to keep it if !restore.OutputOptions.KeepIndexVersion { delete(index.Options, "v") } } session, err := restore.SessionProvider.GetSession() if err != nil { return fmt.Errorf("error establishing connection: %v", err) } session.SetSafe(&mgo.Safe{}) defer session.Close() // then attempt the createIndexes command rawCommand := bson.D{ {"createIndexes", intent.C}, {"indexes", indexes}, } results := bson.M{} err = session.DB(intent.DB).Run(rawCommand, &results) if err == nil { return nil } if err.Error() != "no such cmd: createIndexes" { return fmt.Errorf("createIndex error: %v", err) } // if we're here, the connected server does not support the command, so we fall back log.Log(log.Info, "\tcreateIndexes command not supported, attemping legacy index insertion") for _, idx := range indexes { log.Logf(log.Info, "\tmanually creating index %v", idx.Options["name"]) err = restore.LegacyInsertIndex(intent, idx) if err != nil { return fmt.Errorf("error creating index %v: %v", idx.Options["name"], err) } } return nil }
// GetDumpAuthVersion reads the admin.system.version collection in the dump directory // to determine the authentication version of the files in the dump. If that collection is not // present in the dump, we try to infer the authentication version based on its absence. // Returns the authentication version number and any errors that occur. func (restore *MongoRestore) GetDumpAuthVersion() (int, error) { // first handle the case where we have no auth version intent := restore.manager.AuthVersion() if intent == nil { if restore.InputOptions.RestoreDBUsersAndRoles { // If we are using --restoreDbUsersAndRoles, we cannot guarantee an // $admin.system.version collection from a 2.6 server, // so we can assume up to version 3. log.Logf(log.Always, "no system.version bson file found in '%v' database dump", restore.ToolOptions.DB) log.Log(log.Always, "warning: assuming users and roles collections are of auth version 3") log.Log(log.Always, "if users are from an earlier version of MongoDB, they may not restore properly") return 3, nil } log.Log(log.Info, "no system.version bson file found in dump") log.Log(log.Always, "assuming users in the dump directory are from <= 2.4 (auth version 1)") return 1, nil } err := intent.BSONFile.Open() if err != nil { return 0, err } defer intent.BSONFile.Close() bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile)) defer bsonSource.Close() versionDoc := struct { CurrentVersion int `bson:"currentVersion"` }{} bsonSource.Next(&versionDoc) if err := bsonSource.Err(); err != nil { return 0, fmt.Errorf("error reading version bson file %v: %v", intent.BSONPath, err) } authVersion := versionDoc.CurrentVersion if authVersion == 0 { // 0 is not a possible valid version number, so this can only indicate bad input return 0, fmt.Errorf("system.version bson file does not have 'currentVersion' field") } return authVersion, nil }
// ValidateAuthVersions compares the authentication version of the dump files and the // authentication version of the target server, and returns an error if the versions // are incompatible. func (restore *MongoRestore) ValidateAuthVersions() error { if restore.authVersions.Dump == 2 || restore.authVersions.Dump == 4 { return fmt.Errorf( "cannot restore users and roles from a dump file with auth version %v; "+ "finish the upgrade or roll it back", restore.authVersions.Dump) } if restore.authVersions.Server == 2 || restore.authVersions.Server == 4 { return fmt.Errorf( "cannot restore users and roles to a server with auth version %v; "+ "finish the upgrade or roll it back", restore.authVersions.Server) } switch restore.authVersions { case authVersionPair{3, 5}: log.Log(log.Info, "restoring users and roles of auth version 3 to a server of auth version 5") case authVersionPair{5, 5}: log.Log(log.Info, "restoring users and roles of auth version 5 to a server of auth version 5") case authVersionPair{3, 3}: log.Log(log.Info, "restoring users and roles of auth version 3 to a server of auth version 3") case authVersionPair{1, 1}: log.Log(log.Info, "restoring users and roles of auth version 1 to a server of auth version 1") case authVersionPair{1, 5}: return fmt.Errorf("cannot restore users of auth version 1 to a server of auth version 5") case authVersionPair{5, 3}: return fmt.Errorf("cannot restore users of auth version 5 to a server of auth version 3") case authVersionPair{1, 3}: log.Log(log.Info, "restoring users and roles of auth version 1 to a server of auth version 3") log.Log(log.Always, "users and roles will have to be updated with the authSchemaUpgrade command") case authVersionPair{5, 1}: fallthrough case authVersionPair{3, 1}: return fmt.Errorf( "cannot restore users and roles dump file >= auth version 3 to a server of auth version 1") default: return fmt.Errorf("invalid auth pair: dump=%v, server=%v", restore.authVersions.Dump, restore.authVersions.Server) } return nil }
func main() { go signals.Handle() // initialize command-line opts opts := options.New("bsondump", bsondump.Usage, options.EnabledOptions{}) bsonDumpOpts := &bsondump.BSONDumpOptions{} opts.AddOptions(bsonDumpOpts) args, err := opts.Parse() if err != nil { log.Logf(log.Always, "error parsing command line options: %v", err) log.Logf(log.Always, "try 'bsondump --help' for more information") os.Exit(util.ExitBadOptions) } // print help, if specified if opts.PrintHelp(false) { return } // print version, if specified if opts.PrintVersion() { return } log.SetVerbosity(opts.Verbosity) // pull out the filename if len(args) == 0 { log.Logf(log.Always, "must provide a filename") log.Logf(log.Always, "try 'bsondump --help' for more information") os.Exit(util.ExitBadOptions) } else if len(args) > 1 { log.Logf(log.Always, "too many positional arguments: %v", args) log.Logf(log.Always, "try 'bsondump --help' for more information") os.Exit(util.ExitBadOptions) } dumper := bsondump.BSONDump{ ToolOptions: opts, BSONDumpOptions: bsonDumpOpts, FileName: args[0], Out: os.Stdout, } log.Logf(log.DebugLow, "running bsondump with --objcheck: %v", bsonDumpOpts.ObjCheck) if len(bsonDumpOpts.Type) != 0 && bsonDumpOpts.Type != "debug" && bsonDumpOpts.Type != "json" { log.Logf(log.Always, "Unsupported output type '%v'. Must be either 'debug' or 'json'", bsonDumpOpts.Type) os.Exit(util.ExitBadOptions) } err = dumper.Open() if err != nil { log.Logf(log.Always, "Failed: %v", err) os.Exit(util.ExitError) } var numFound int if bsonDumpOpts.Type == "debug" { numFound, err = dumper.Debug() } else { numFound, err = dumper.JSON() } log.Logf(log.Always, "%v objects found", numFound) if err != nil { log.Log(log.Always, err.Error()) os.Exit(util.ExitError) } }
// Dump handles some final options checking and executes MongoDump. func (dump *MongoDump) Dump() (err error) { if dump.InputOptions.HasQuery() { // parse JSON then convert extended JSON values var asJSON interface{} content, err := dump.InputOptions.GetQuery() if err != nil { return err } err = json.Unmarshal(content, &asJSON) if err != nil { return fmt.Errorf("error parsing query as json: %v", err) } convertedJSON, err := bsonutil.ConvertJSONValueToBSON(asJSON) if err != nil { return fmt.Errorf("error converting query to bson: %v", err) } asMap, ok := convertedJSON.(map[string]interface{}) if !ok { // unlikely to be reached return fmt.Errorf("query is not in proper format") } dump.query = bson.M(asMap) } if dump.OutputOptions.DumpDBUsersAndRoles { // first make sure this is possible with the connected database dump.authVersion, err = auth.GetAuthVersion(dump.sessionProvider) if err != nil { return fmt.Errorf("error getting auth schema version for dumpDbUsersAndRoles: %v", err) } log.Logf(log.DebugLow, "using auth schema version %v", dump.authVersion) if dump.authVersion < 3 { return fmt.Errorf("backing up users and roles is only supported for "+ "deployments with auth schema versions >= 3, found: %v", dump.authVersion) } } if dump.OutputOptions.Archive != "" { //getArchiveOut gives us a WriteCloser to which we should write the archive var archiveOut io.WriteCloser archiveOut, err = dump.getArchiveOut() if err != nil { return err } dump.archive = &archive.Writer{ // The archive.Writer needs its own copy of archiveOut because things // like the prelude are not written by the multiplexer. Out: archiveOut, Mux: archive.NewMultiplexer(archiveOut), } go dump.archive.Mux.Run() defer func() { // The Mux runs until its Control is closed close(dump.archive.Mux.Control) muxErr := <-dump.archive.Mux.Completed archiveOut.Close() if muxErr != nil { if err != nil { err = fmt.Errorf("%v && %v", err, muxErr) } else { err = muxErr } log.Logf(log.DebugLow, "mux returned an error: %v", err) } else { log.Logf(log.DebugLow, "mux completed successfully") } }() } // switch on what kind of execution to do switch { case dump.ToolOptions.DB == "" && dump.ToolOptions.Collection == "": err = dump.CreateAllIntents() case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection == "": err = dump.CreateIntentsForDatabase(dump.ToolOptions.DB) case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection != "": err = dump.CreateCollectionIntent(dump.ToolOptions.DB, dump.ToolOptions.Collection) } if err != nil { return err } if dump.OutputOptions.Oplog { err = dump.CreateOplogIntents() if err != nil { return err } } if dump.OutputOptions.DumpDBUsersAndRoles && dump.ToolOptions.DB != "admin" { err = dump.CreateUsersRolesVersionIntentsForDB(dump.ToolOptions.DB) if err != nil { return err } } // verify we can use repair cursors if dump.OutputOptions.Repair { log.Log(log.DebugLow, "verifying that the connected server supports repairCursor") if dump.isMongos { return fmt.Errorf("cannot use --repair on mongos") } exampleIntent := dump.manager.Peek() if exampleIntent != nil { supported, err := dump.sessionProvider.SupportsRepairCursor( exampleIntent.DB, exampleIntent.C) if !supported { return err // no extra context needed } } } // IO Phase I // metadata, users, roles, and versions // TODO, either remove this debug or improve the language log.Logf(log.DebugHigh, "dump phase I: metadata, indexes, users, roles, version") err = dump.DumpMetadata() if err != nil { return fmt.Errorf("error dumping metadata: %v", err) } if dump.OutputOptions.Archive != "" { dump.archive.Prelude, err = archive.NewPrelude(dump.manager, dump.ToolOptions.HiddenOptions.MaxProcs) if err != nil { return fmt.Errorf("creating archive prelude: %v", err) } err = dump.archive.Prelude.Write(dump.archive.Out) if err != nil { return fmt.Errorf("error writing metadata into archive: %v", err) } } err = dump.DumpSystemIndexes() if err != nil { return fmt.Errorf("error dumping system indexes: %v", err) } if dump.ToolOptions.DB == "admin" || dump.ToolOptions.DB == "" { err = dump.DumpUsersAndRoles() if err != nil { return fmt.Errorf("error dumping users and roles: %v", err) } } if dump.OutputOptions.DumpDBUsersAndRoles { log.Logf(log.Always, "dumping users and roles for %v", dump.ToolOptions.DB) if dump.ToolOptions.DB == "admin" { log.Logf(log.Always, "skipping users/roles dump, already dumped admin database") } else { err = dump.DumpUsersAndRolesForDB(dump.ToolOptions.DB) if err != nil { return fmt.Errorf("error dumping users and roles for db: %v", err) } } } // If oplog capturing is enabled, we first check the most recent // oplog entry and save its timestamp, this will let us later // copy all oplog entries that occurred while dumping, creating // what is effectively a point-in-time snapshot. if dump.OutputOptions.Oplog { err := dump.determineOplogCollectionName() if err != nil { return fmt.Errorf("error finding oplog: %v", err) } log.Logf(log.Info, "getting most recent oplog timestamp") dump.oplogStart, err = dump.getOplogStartTime() if err != nil { return fmt.Errorf("error getting oplog start: %v", err) } } // IO Phase II // regular collections // TODO, either remove this debug or improve the language log.Logf(log.DebugHigh, "dump phase II: regular collections") // kick off the progress bar manager and begin dumping intents if dump.ProgressManager != nil { dump.ProgressManager.Start() defer dump.ProgressManager.Stop() } dump.termChan = make(chan struct{}) if dump.HandleSignals != nil { go dump.HandleSignals(dump) } if err := dump.DumpIntents(); err != nil { return err } // IO Phase III // oplog // TODO, either remove this debug or improve the language log.Logf(log.DebugLow, "dump phase III: the oplog") // If we are capturing the oplog, we dump all oplog entries that occurred // while dumping the database. Before and after dumping the oplog, // we check to see if the oplog has rolled over (i.e. the most recent entry when // we started still exist, so we know we haven't lost data) if dump.OutputOptions.Oplog { log.Logf(log.DebugLow, "checking if oplog entry %v still exists", dump.oplogStart) exists, err := dump.checkOplogTimestampExists(dump.oplogStart) if !exists { return fmt.Errorf( "oplog overflow: mongodump was unable to capture all new oplog entries during execution") } if err != nil { return fmt.Errorf("unable to check oplog for overflow: %v", err) } log.Logf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart) log.Logf(log.Always, "writing captured oplog to %v", dump.manager.Oplog().BSONPath) err = dump.DumpOplogAfterTimestamp(dump.oplogStart) if err != nil { return fmt.Errorf("error dumping oplog: %v", err) } // check the oplog for a rollover one last time, to avoid a race condition // wherein the oplog rolls over in the time after our first check, but before // we copy it. log.Logf(log.DebugLow, "checking again if oplog entry %v still exists", dump.oplogStart) exists, err = dump.checkOplogTimestampExists(dump.oplogStart) if !exists { return fmt.Errorf( "oplog overflow: mongodump was unable to capture all new oplog entries during execution") } if err != nil { return fmt.Errorf("unable to check oplog for overflow: %v", err) } log.Logf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart) } log.Logf(log.Info, "done") return err }
// CreateAllIntents drills down into a dump folder, creating intents for all of // the databases and collections it finds. func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike, filterDB string, filterCollection string) error { log.Logf(log.DebugHigh, "using %v as dump root directory", dir.Path()) foundOplog := false entries, err := dir.ReadDir() if err != nil { return fmt.Errorf("error reading root dump folder: %v", err) } for _, entry := range entries { if entry.IsDir() { if err = util.ValidateDBName(entry.Name()); err != nil { return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err) } if filterDB == "" || entry.Name() == filterDB { err = restore.CreateIntentsForDB(entry.Name(), filterCollection, entry, false) } else { err = restore.CreateIntentsForDB(entry.Name(), "", entry, true) } if err != nil { return err } } else { if entry.Name() == "oplog.bson" { if restore.InputOptions.OplogReplay { log.Log(log.DebugLow, "found oplog.bson file to replay") } foundOplog = true oplogIntent := &intents.Intent{ C: "oplog", Size: entry.Size(), Location: entry.Path(), } // filterDB is used to mimic CreateIntentsForDB, and since CreateIntentsForDB wouldn't // apply the oplog, even when asked, we don't either. if filterDB != "" || !restore.InputOptions.OplogReplay { if restore.InputOptions.Archive == "" { continue } else { mutedOut := &archive.MutedCollection{ Intent: oplogIntent, Demux: restore.archive.Demux, } restore.archive.Demux.Open( oplogIntent.Namespace(), mutedOut, ) continue } } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { oplogIntent.Location = "archive on stdin" } else { oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } // no need to check that we want to cache here oplogIntent.BSONFile = &archive.RegularCollectionReceiver{ Intent: oplogIntent, Demux: restore.archive.Demux, } } else { oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip} } restore.manager.Put(oplogIntent) } else { log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } if restore.InputOptions.OplogReplay && !foundOplog { return fmt.Errorf("no %v/oplog.bson file to replay; make sure you run mongodump with --oplog", dir.Path()) } return nil }
// RestoreOplog attempts to restore a MongoDB oplog. func (restore *MongoRestore) RestoreOplog() error { log.Log(log.Always, "replaying oplog") intent := restore.manager.Oplog() if intent == nil { // this should not be reached log.Log(log.Always, "no oplog.bson file in root of the dump directory, skipping oplog application") return nil } if err := intent.BSONFile.Open(); err != nil { return err } defer intent.BSONFile.Close() bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile)) defer bsonSource.Close() entryArray := make([]interface{}, 0, 1024) rawOplogEntry := &bson.Raw{} var totalOps int64 var entrySize, bufferedBytes int oplogProgressor := progress.NewCounter(intent.BSONSize) bar := progress.Bar{ Name: "oplog", Watching: oplogProgressor, WaitTime: 3 * time.Second, Writer: log.Writer(0), BarLength: progressBarLength, IsBytes: true, } bar.Start() defer bar.Stop() session, err := restore.SessionProvider.GetSession() if err != nil { return fmt.Errorf("error establishing connection: %v", err) } defer session.Close() // To restore the oplog, we iterate over the oplog entries, // filling up a buffer. Once the buffer reaches max document size, // apply the current buffered ops and reset the buffer. for bsonSource.Next(rawOplogEntry) { entrySize = len(rawOplogEntry.Data) if bufferedBytes+entrySize > oplogMaxCommandSize { err = restore.ApplyOps(session, entryArray) if err != nil { return fmt.Errorf("error applying oplog: %v", err) } entryArray = make([]interface{}, 0, 1024) bufferedBytes = 0 } entryAsOplog := db.Oplog{} err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog) if err != nil { return fmt.Errorf("error reading oplog: %v", err) } if entryAsOplog.Operation == "n" { //skip no-ops continue } if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) { log.Logf( log.DebugLow, "timestamp %v is not below limit of %v; ending oplog restoration", entryAsOplog.Timestamp, restore.oplogLimit, ) break } totalOps++ bufferedBytes += entrySize oplogProgressor.Inc(int64(entrySize)) entryArray = append(entryArray, entryAsOplog) } // finally, flush the remaining entries if len(entryArray) > 0 { err = restore.ApplyOps(session, entryArray) if err != nil { return fmt.Errorf("error applying oplog: %v", err) } } log.Logf(log.Info, "applied %v ops", totalOps) return nil }
// RestoreIntent attempts to restore a given intent into MongoDB. func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error { collectionExists, err := restore.CollectionExists(intent) if err != nil { return fmt.Errorf("error reading database: %v", err) } if restore.safety == nil && !restore.OutputOptions.Drop && collectionExists { log.Logf(log.Always, "restoring to existing collection %v without dropping", intent.Namespace()) log.Log(log.Always, "Important: restored data will be inserted without raising errors; check your server log") } if restore.OutputOptions.Drop { if collectionExists { if strings.HasPrefix(intent.C, "system.") { log.Logf(log.Always, "cannot drop system collection %v, skipping", intent.Namespace()) } else { log.Logf(log.Info, "dropping collection %v before restoring", intent.Namespace()) err = restore.DropCollection(intent) if err != nil { return err // no context needed } collectionExists = false } } else { log.Logf(log.DebugLow, "collection %v doesn't exist, skipping drop command", intent.Namespace()) } } var options bson.D var indexes []IndexDocument // get indexes from system.indexes dump if we have it but don't have metadata files if intent.MetadataPath == "" { if _, ok := restore.dbCollectionIndexes[intent.DB]; ok { if indexes, ok = restore.dbCollectionIndexes[intent.DB][intent.C]; ok { log.Logf(log.Always, "no metadata; falling back to system.indexes") } } } // first create the collection with options from the metadata file if intent.MetadataPath != "" { err = intent.MetadataFile.Open() if err != nil { return err } defer intent.MetadataFile.Close() log.Logf(log.Always, "reading metadata for %v from %v", intent.Namespace(), intent.Location) metadata, err := ioutil.ReadAll(intent.MetadataFile) if err != nil { return fmt.Errorf("error reading metadata from %v: %v", intent.Location, err) } options, indexes, err = restore.MetadataFromJSON(metadata) if err != nil { return fmt.Errorf("error parsing metadata from %v: %v", intent.Location, err) } if !restore.OutputOptions.NoOptionsRestore { if options != nil { if !collectionExists { log.Logf(log.Info, "creating collection %v using options from metadata", intent.Namespace()) err = restore.CreateCollection(intent, options) if err != nil { return fmt.Errorf("error creating collection %v: %v", intent.Namespace(), err) } } else { log.Logf(log.Info, "collection %v already exists", intent.Namespace()) } } else { log.Log(log.Info, "no collection options to restore") } } else { log.Log(log.Info, "skipping options restoration") } } if intent.BSONPath != "" { err = intent.BSONFile.Open() if err != nil { return err } defer intent.BSONFile.Close() log.Logf(log.Always, "restoring %v from %v", intent.Namespace(), intent.Location) var size int64 bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile)) defer bsonSource.Close() err = restore.RestoreCollectionToDB(intent.DB, intent.C, bsonSource, size) if err != nil { return fmt.Errorf("error restoring from %v: %v", intent.BSONPath, err) } } // finally, add indexes if len(indexes) > 0 && !restore.OutputOptions.NoIndexRestore { log.Logf(log.Always, "restoring indexes for collection %v from metadata", intent.Namespace()) err = restore.CreateIndexes(intent, indexes) if err != nil { return fmt.Errorf("error creating indexes for %v: %v", intent.Namespace(), err) } } else { log.Log(log.Always, "no indexes to restore") } log.Logf(log.Always, "finished restoring %v", intent.Namespace()) return nil }
func main() { go signals.Handle() // initialize command-line opts opts := options.New("bsondump", bsondump.Usage, options.EnabledOptions{}) bsonDumpOpts := &bsondump.BSONDumpOptions{} opts.AddOptions(bsonDumpOpts) args, err := opts.Parse() if err != nil { log.Logf(log.Always, "error parsing command line options: %v", err) log.Logf(log.Always, "try 'bsondump --help' for more information") os.Exit(util.ExitBadOptions) } // print help, if specified if opts.PrintHelp(false) { return } // print version, if specified if opts.PrintVersion() { return } log.SetVerbosity(opts.Verbosity) if len(args) > 1 { log.Logf(log.Always, "too many positional arguments: %v", args) log.Logf(log.Always, "try 'bsondump --help' for more information") os.Exit(util.ExitBadOptions) } // If the user specified a bson input file if len(args) == 1 { if bsonDumpOpts.BSONFileName != "" { log.Logf(log.Always, "Cannot specify both a positional argument and --bsonFile") os.Exit(util.ExitBadOptions) } bsonDumpOpts.BSONFileName = args[0] } dumper := bsondump.BSONDump{ ToolOptions: opts, BSONDumpOptions: bsonDumpOpts, } reader, err := bsonDumpOpts.GetBSONReader() if err != nil { log.Logf(log.Always, "Getting BSON Reader Failed: %v", err) os.Exit(util.ExitError) } dumper.BSONSource = db.NewBSONSource(reader) defer dumper.BSONSource.Close() writer, err := bsonDumpOpts.GetWriter() if err != nil { log.Logf(log.Always, "Getting Writer Failed: %v", err) os.Exit(util.ExitError) } dumper.Out = writer defer dumper.Out.Close() log.Logf(log.DebugLow, "running bsondump with --objcheck: %v", bsonDumpOpts.ObjCheck) if len(bsonDumpOpts.Type) != 0 && bsonDumpOpts.Type != "debug" && bsonDumpOpts.Type != "json" { log.Logf(log.Always, "Unsupported output type '%v'. Must be either 'debug' or 'json'", bsonDumpOpts.Type) os.Exit(util.ExitBadOptions) } var numFound int if bsonDumpOpts.Type == "debug" { numFound, err = dumper.Debug() } else { numFound, err = dumper.JSON() } log.Logf(log.Always, "%v objects found", numFound) if err != nil { log.Log(log.Always, err.Error()) os.Exit(util.ExitError) } }
// Restore runs the mongorestore program. func (restore *MongoRestore) Restore() error { var target archive.DirLike err := restore.ParseAndValidateOptions() if err != nil { log.Logf(log.DebugLow, "got error from options parsing: %v", err) return err } // Build up all intents to be restored restore.manager = intents.NewIntentManager() if restore.InputOptions.Archive != "" { archiveReader, err := restore.getArchiveReader() if err != nil { return err } restore.archive = &archive.Reader{ In: archiveReader, Prelude: &archive.Prelude{}, } err = restore.archive.Prelude.Read(restore.archive.In) if err != nil { return err } target, err = restore.archive.Prelude.NewPreludeExplorer() if err != nil { return err } } else { if restore.TargetDirectory == "" { restore.TargetDirectory = "dump" log.Log(log.Always, "using default 'dump' directory") } target, err = newActualPath(restore.TargetDirectory) if err != nil { return err } // handle cases where the user passes in a file instead of a directory if !target.IsDir() { log.Log(log.DebugLow, "mongorestore target is a file, not a directory") err = restore.handleBSONInsteadOfDirectory(restore.TargetDirectory) if err != nil { return err } } else { log.Log(log.DebugLow, "mongorestore target is a directory, not a file") } } if restore.ToolOptions.Collection != "" && restore.OutputOptions.NumParallelCollections > 1 && restore.OutputOptions.NumInsertionWorkers == 1 { // handle special parallelization case when we are only restoring one collection // by mapping -j to insertion workers rather than parallel collections log.Logf(log.DebugHigh, "setting number of insertions workers to number of parallel collections (%v)", restore.OutputOptions.NumParallelCollections) restore.OutputOptions.NumInsertionWorkers = restore.OutputOptions.NumParallelCollections } if restore.InputOptions.Archive != "" { if int(restore.archive.Prelude.Header.ConcurrentCollections) > restore.OutputOptions.NumParallelCollections { restore.OutputOptions.NumParallelCollections = int(restore.archive.Prelude.Header.ConcurrentCollections) restore.OutputOptions.NumInsertionWorkers = int(restore.archive.Prelude.Header.ConcurrentCollections) log.Logf(log.Always, "setting number of parallel collections to number of parallel collections in archive (%v)", restore.archive.Prelude.Header.ConcurrentCollections, ) } } // Create the demux before intent creation, because muted archive intents need // to register themselves with the demux directly if restore.InputOptions.Archive != "" { restore.archive.Demux = &archive.Demultiplexer{ In: restore.archive.In, } } switch { case restore.InputOptions.Archive != "": log.Logf(log.Always, "creating intents for archive") err = restore.CreateAllIntents(target, restore.ToolOptions.DB, restore.ToolOptions.Collection) case restore.ToolOptions.DB == "" && restore.ToolOptions.Collection == "": log.Logf(log.Always, "building a list of dbs and collections to restore from %v dir", target.Path()) err = restore.CreateAllIntents(target, "", "") case restore.ToolOptions.DB != "" && restore.ToolOptions.Collection == "": log.Logf(log.Always, "building a list of collections to restore from %v dir", target.Path()) err = restore.CreateIntentsForDB( restore.ToolOptions.DB, "", target, false, ) case restore.ToolOptions.DB != "" && restore.ToolOptions.Collection != "": log.Logf(log.Always, "checking for collection data in %v", target.Path()) err = restore.CreateIntentForCollection( restore.ToolOptions.DB, restore.ToolOptions.Collection, target, ) } if err != nil { return fmt.Errorf("error scanning filesystem: %v", err) } if restore.isMongos && restore.manager.HasConfigDBIntent() && restore.ToolOptions.DB == "" { return fmt.Errorf("cannot do a full restore on a sharded system - " + "remove the 'config' directory from the dump directory first") } if restore.InputOptions.Archive != "" { namespaceChan := make(chan string, 1) namespaceErrorChan := make(chan error) restore.archive.Demux.NamespaceChan = namespaceChan restore.archive.Demux.NamespaceErrorChan = namespaceErrorChan go restore.archive.Demux.Run() // consume the new namespace announcement from the demux for all of the special collections // that get cached when being read out of the archive. // The first regular collection found gets pushed back on to the namespaceChan // consume the new namespace announcement from the demux for all of the collections that get cached for { ns, ok := <-namespaceChan // the archive can have only special collections. In that case we keep reading until // the namespaces are exhausted, indicated by the namespaceChan being closed. if !ok { break } intent := restore.manager.IntentForNamespace(ns) if intent == nil { return fmt.Errorf("no intent for collection in archive: %v", ns) } if intent.IsSystemIndexes() || intent.IsUsers() || intent.IsRoles() || intent.IsAuthVersion() { log.Logf(log.DebugLow, "special collection %v found", ns) namespaceErrorChan <- nil } else { // Put the ns back on the announcement chan so that the // demultiplexer can start correctly log.Logf(log.DebugLow, "first non special collection %v found."+ " The demultiplexer will handle it and the remainder", ns) namespaceChan <- ns break } } } // If restoring users and roles, make sure we validate auth versions if restore.ShouldRestoreUsersAndRoles() { log.Log(log.Info, "comparing auth version of the dump directory and target server") restore.authVersions.Dump, err = restore.GetDumpAuthVersion() if err != nil { return fmt.Errorf("error getting auth version from dump: %v", err) } restore.authVersions.Server, err = auth.GetAuthVersion(restore.SessionProvider) if err != nil { return fmt.Errorf("error getting auth version of server: %v", err) } err = restore.ValidateAuthVersions() if err != nil { return fmt.Errorf( "the users and roles collections in the dump have an incompatible auth version with target server: %v", err) } } err = restore.LoadIndexesFromBSON() if err != nil { return fmt.Errorf("restore error: %v", err) } // Restore the regular collections if restore.InputOptions.Archive != "" { restore.manager.UsePrioritizer(restore.archive.Demux.NewPrioritizer(restore.manager)) } else if restore.OutputOptions.NumParallelCollections > 1 { restore.manager.Finalize(intents.MultiDatabaseLTF) } else { // use legacy restoration order if we are single-threaded restore.manager.Finalize(intents.Legacy) } restore.termChan = make(chan struct{}) go restore.handleSignals() if err := restore.RestoreIntents(); err != nil { return err } // Restore users/roles if restore.ShouldRestoreUsersAndRoles() { if restore.manager.Users() != nil { err = restore.RestoreUsersOrRoles(Users, restore.manager.Users()) if err != nil { return fmt.Errorf("restore error: %v", err) } } if restore.manager.Roles() != nil { err = restore.RestoreUsersOrRoles(Roles, restore.manager.Roles()) if err != nil { return fmt.Errorf("restore error: %v", err) } } } // Restore oplog if restore.InputOptions.OplogReplay { err = restore.RestoreOplog() if err != nil { return fmt.Errorf("restore error: %v", err) } } log.Log(log.Always, "done") return nil }
// ParseAndValidateOptions returns a non-nil error if user-supplied options are invalid. func (restore *MongoRestore) ParseAndValidateOptions() error { // Can't use option pkg defaults for --objcheck because it's two separate flags, // and we need to be able to see if they're both being used. We default to // true here and then see if noobjcheck is enabled. log.Log(log.DebugHigh, "checking options") if restore.InputOptions.Objcheck { restore.objCheck = true log.Log(log.DebugHigh, "\tdumping with object check enabled") } else { log.Log(log.DebugHigh, "\tdumping with object check disabled") } if restore.ToolOptions.DB == "" && restore.ToolOptions.Collection != "" { return fmt.Errorf("cannot dump a collection without a specified database") } if restore.ToolOptions.DB != "" { if err := util.ValidateDBName(restore.ToolOptions.DB); err != nil { return fmt.Errorf("invalid db name: %v", err) } } if restore.ToolOptions.Collection != "" { if err := util.ValidateCollectionGrammar(restore.ToolOptions.Collection); err != nil { return fmt.Errorf("invalid collection name: %v", err) } } if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles without a specified database") } if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "admin" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles with the admin database") } var err error restore.isMongos, err = restore.SessionProvider.IsMongos() if err != nil { return err } if restore.isMongos { log.Log(log.DebugLow, "restoring to a sharded system") } if restore.InputOptions.OplogLimit != "" { if !restore.InputOptions.OplogReplay { return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled") } restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit) if err != nil { return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err) } } // check if we are using a replica set and fall back to w=1 if we aren't (for <= 2.4) nodeType, err := restore.SessionProvider.GetNodeType() if err != nil { return fmt.Errorf("error determining type of connected node: %v", err) } log.Logf(log.DebugLow, "connected to node type: %v", nodeType) restore.safety, err = db.BuildWriteConcern(restore.OutputOptions.WriteConcern, nodeType) if err != nil { return fmt.Errorf("error parsing write concern: %v", err) } // handle the hidden auth collection flags if restore.ToolOptions.HiddenOptions.TempUsersColl == nil { restore.tempUsersCol = "tempusers" } else { restore.tempUsersCol = *restore.ToolOptions.HiddenOptions.TempUsersColl } if restore.ToolOptions.HiddenOptions.TempRolesColl == nil { restore.tempRolesCol = "temproles" } else { restore.tempRolesCol = *restore.ToolOptions.HiddenOptions.TempRolesColl } if restore.OutputOptions.NumInsertionWorkers < 0 { return fmt.Errorf( "cannot specify a negative number of insertion workers per collection") } // a single dash signals reading from stdin if restore.TargetDirectory == "-" { restore.useStdin = true if restore.InputOptions.Archive != "" { return fmt.Errorf( "cannot restore from \"-\" when --archive is specified") } if restore.ToolOptions.Collection == "" { return fmt.Errorf("cannot restore from stdin without a specified collection") } } return nil }
// Run executes the mongooplog program. func (mo *MongoOplog) Run() error { // split up the oplog namespace we are using oplogDB, oplogColl, err := util.SplitAndValidateNamespace(mo.SourceOptions.OplogNS) if err != nil { return err } // the full oplog namespace needs to be specified if oplogColl == "" { return fmt.Errorf("the oplog namespace must specify a collection") } log.Logf(log.DebugLow, "using oplog namespace `%v.%v`", oplogDB, oplogColl) // connect to the destination server toSession, err := mo.SessionProviderTo.GetSession() if err != nil { return fmt.Errorf("error connecting to destination db: %v", err) } defer toSession.Close() toSession.SetSocketTimeout(0) // purely for logging destServerStr := mo.ToolOptions.Host if mo.ToolOptions.Port != "" { destServerStr = destServerStr + ":" + mo.ToolOptions.Port } log.Logf(log.DebugLow, "successfully connected to destination server `%v`", destServerStr) // connect to the source server fromSession, err := mo.SessionProviderFrom.GetSession() if err != nil { return fmt.Errorf("error connecting to source db: %v", err) } defer fromSession.Close() fromSession.SetSocketTimeout(0) log.Logf(log.DebugLow, "successfully connected to source server `%v`", mo.SourceOptions.From) // set slave ok fromSession.SetMode(mgo.Eventual, true) // get the tailing cursor for the source server's oplog tail := buildTailingCursor(fromSession.DB(oplogDB).C(oplogColl), mo.SourceOptions) defer tail.Close() // read the cursor dry, applying ops to the destination // server in the process oplogEntry := &db.Oplog{} res := &db.ApplyOpsResponse{} log.Log(log.DebugLow, "applying oplog entries...") for tail.Next(oplogEntry) { // skip noops if oplogEntry.Operation == "n" { log.Logf(log.DebugHigh, "skipping no-op for namespace `%v`", oplogEntry.Namespace) continue } // prepare the op to be applied opsToApply := []db.Oplog{*oplogEntry} // apply the operation err := toSession.Run(bson.M{"applyOps": opsToApply}, res) if err != nil { return fmt.Errorf("error applying ops: %v", err) } // check the server's response for an issue if !res.Ok { return fmt.Errorf("server gave error applying ops: %v", res.ErrMsg) } } // make sure there was no tailing error if err := tail.Err(); err != nil { return fmt.Errorf("error querying oplog: %v", err) } log.Log(log.DebugLow, "done applying oplog entries") return nil }