// Init performs preliminary setup operations for MongoDump. func (dump *MongoDump) Init() error { err := dump.ValidateOptions() if err != nil { return fmt.Errorf("bad option: %v", err) } if dump.stdout == nil { dump.stdout = os.Stdout } dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions) if err != nil { return fmt.Errorf("can't create session: %v", err) } // temporarily allow secondary reads for the isMongos check dump.sessionProvider.SetReadPreference(mgo.Nearest) dump.isMongos, err = dump.sessionProvider.IsMongos() if err != nil { return err } if dump.isMongos && dump.OutputOptions.Oplog { return fmt.Errorf("can't use --oplog option when dumping from a mongos") } var mode mgo.Mode if dump.ToolOptions.ReplicaSetName != "" || dump.isMongos { mode = mgo.Primary } else { mode = mgo.Nearest } var tags bson.D if dump.InputOptions.ReadPreference != "" { mode, tags, err = db.ParseReadPreference(dump.InputOptions.ReadPreference) if err != nil { return fmt.Errorf("error parsing --readPreference : %v", err) } if len(tags) > 0 { dump.sessionProvider.SetTags(tags) } } // warn if we are trying to dump from a secondary in a sharded cluster if dump.isMongos && mode != mgo.Primary { log.Logf(log.Always, db.WarningNonPrimaryMongosConnection) } dump.sessionProvider.SetReadPreference(mode) dump.sessionProvider.SetTags(tags) dump.sessionProvider.SetFlags(db.DisableSocketTimeout) // return a helpful error message for mongos --repair if dump.OutputOptions.Repair && dump.isMongos { return fmt.Errorf("--repair flag cannot be used on a mongos") } dump.manager = intents.NewIntentManager() dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime) return nil }
func simpleMongoDumpInstance() *MongoDump { ssl := testutil.GetSSLOptions() auth := testutil.GetAuthOptions() namespace := &options.Namespace{ DB: testDB, } connection := &options.Connection{ Host: testServer, Port: testPort, } toolOptions := &options.ToolOptions{ SSL: &ssl, Namespace: namespace, Connection: connection, Auth: &auth, HiddenOptions: &options.HiddenOptions{}, Verbosity: &options.Verbosity{}, } outputOptions := &OutputOptions{} inputOptions := &InputOptions{} log.SetVerbosity(toolOptions.Verbosity) return &MongoDump{ ToolOptions: toolOptions, InputOptions: inputOptions, OutputOptions: outputOptions, ProgressManager: progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime), HandleSignals: HandleSignals, } }
// Init performs preliminary setup operations for MongoDump. func (dump *MongoDump) Init() error { err := dump.ValidateOptions() if err != nil { return fmt.Errorf("bad option: %v", err) } dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions) if err != nil { return fmt.Errorf("can't create session: %v", err) } // allow secondary reads for the isMongos check dump.sessionProvider.SetFlags(db.Monotonic) dump.isMongos, err = dump.sessionProvider.IsMongos() if err != nil { return err } // ensure we allow secondary reads on mongods and disable TCP timeouts flags := db.DisableSocketTimeout if dump.isMongos { log.Logf(log.Info, "connecting to mongos; secondary reads disabled") } else { flags |= db.Monotonic } dump.sessionProvider.SetFlags(flags) // return a helpful error message for mongos --repair if dump.OutputOptions.Repair && dump.isMongos { return fmt.Errorf("--repair flag cannot be used on a mongos") } dump.manager = intents.NewIntentManager() dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime) return nil }
// RestoreIntents iterates through all of the intents stored in the IntentManager, and restores them. func (restore *MongoRestore) RestoreIntents() error { // start up the progress bar manager restore.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime) restore.progressManager.Start() defer restore.progressManager.Stop() log.Logf(log.DebugLow, "restoring up to %v collections in parallel", restore.OutputOptions.NumParallelCollections) if restore.OutputOptions.NumParallelCollections > 0 { resultChan := make(chan error) // start a goroutine for each job thread for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ { go func(id int) { log.Logf(log.DebugHigh, "starting restore routine with id=%v", id) for { intent := restore.manager.Pop() if intent == nil { log.Logf(log.DebugHigh, "ending restore routine with id=%v, no more work to do", id) resultChan <- nil // done return } err := restore.RestoreIntent(intent) if err != nil { resultChan <- fmt.Errorf("%v: %v", intent.Namespace(), err) return } restore.manager.Finish(intent) } }(i) } // wait until all goroutines are done or one of them errors out for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ { if err := <-resultChan; err != nil { return err } } return nil } // single-threaded for { intent := restore.manager.Pop() if intent == nil { return nil } err := restore.RestoreIntent(intent) if err != nil { return fmt.Errorf("%v: %v", intent.Namespace(), err) } restore.manager.Finish(intent) } return nil }
// Internal function that handles exporting to the given writer. Used primarily // for testing, because it bypasses writing to the file system. func (exp *MongoExport) exportInternal(out io.Writer) (int64, error) { max, err := exp.getCount() if err != nil { return 0, err } progressManager := progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime) progressManager.Start() defer progressManager.Stop() watchProgressor := progress.NewCounter(int64(max)) bar := &progress.Bar{ Name: fmt.Sprintf("%v.%v", exp.ToolOptions.Namespace.DB, exp.ToolOptions.Namespace.Collection), Watching: watchProgressor, BarLength: progressBarLength, } progressManager.Attach(bar) defer progressManager.Detach(bar) exportOutput, err := exp.getExportOutput(out) if err != nil { return 0, err } cursor, session, err := exp.getCursor() if err != nil { return 0, err } defer session.Close() defer cursor.Close() connURL := exp.ToolOptions.Host if connURL == "" { connURL = util.DefaultHost } if exp.ToolOptions.Port != "" { connURL = connURL + ":" + exp.ToolOptions.Port } log.Logf(log.Always, "connected to: %v", connURL) // Write headers err = exportOutput.WriteHeader() if err != nil { return 0, err } var result bson.M docsCount := int64(0) // Write document content for cursor.Next(&result) { err := exportOutput.ExportDocument(result) if err != nil { return docsCount, err } docsCount++ if docsCount%watchProgressorUpdateFrequency == 0 { watchProgressor.Set(docsCount) } } watchProgressor.Set(docsCount) if err := cursor.Err(); err != nil { return docsCount, err } // Write footers err = exportOutput.WriteFooter() if err != nil { return docsCount, err } exportOutput.Flush() return docsCount, nil }
func main() { // initialize command-line opts opts := options.New("mongodump", mongodump.Usage, options.EnabledOptions{true, true, true}) inputOpts := &mongodump.InputOptions{} opts.AddOptions(inputOpts) outputOpts := &mongodump.OutputOptions{} opts.AddOptions(outputOpts) args, err := opts.Parse() if err != nil { log.Logf(log.Always, "error parsing command line options: %v", err) log.Logf(log.Always, "try 'mongodump --help' for more information") os.Exit(util.ExitBadOptions) } if len(args) > 0 { log.Logf(log.Always, "positional arguments not allowed: %v", args) log.Logf(log.Always, "try 'mongodump --help' for more information") os.Exit(util.ExitBadOptions) } // print help, if specified if opts.PrintHelp(false) { return } // print version, if specified if opts.PrintVersion() { return } // init logger log.SetVerbosity(opts.Verbosity) // connect directly, unless a replica set name is explicitly specified _, setName := util.ParseConnectionString(opts.Host) opts.Direct = (setName == "") opts.ReplicaSetName = setName dump := mongodump.MongoDump{ ToolOptions: opts, OutputOptions: outputOpts, InputOptions: inputOpts, ProgressManager: progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime), HandleSignals: mongodump.HandleSignals, } if err = dump.Init(); err != nil { log.Logf(log.Always, "Failed: %v", err) os.Exit(util.ExitError) } if err = dump.Dump(); err != nil { log.Logf(log.Always, "Failed: %v", err) if err == util.ErrTerminated { os.Exit(util.ExitKill) } os.Exit(util.ExitError) } }