// ValidateSettings returns an error if any settings specified on the command line // were invalid, or nil if they are valid. func (exp *MongoExport) ValidateSettings() error { // Namespace must have a valid database if none is specified, // use 'test' if exp.ToolOptions.Namespace.DB == "" { exp.ToolOptions.Namespace.DB = "test" } err := util.ValidateDBName(exp.ToolOptions.Namespace.DB) if err != nil { return err } if exp.ToolOptions.Namespace.Collection == "" { return fmt.Errorf("must specify a collection") } if err = util.ValidateCollectionGrammar(exp.ToolOptions.Namespace.Collection); err != nil { return err } exp.OutputOpts.Type = strings.ToLower(exp.OutputOpts.Type) if exp.OutputOpts.CSVOutputType { log.Logv(log.Always, "csv flag is deprecated; please use --type=csv instead") exp.OutputOpts.Type = CSV } if exp.OutputOpts.Type == "" { // special error for an empty type value return fmt.Errorf("--type cannot be empty") } if exp.OutputOpts.Type != CSV && exp.OutputOpts.Type != JSON { return fmt.Errorf("invalid output type '%v', choose 'json' or 'csv'", exp.OutputOpts.Type) } if exp.InputOpts.Query != "" && exp.InputOpts.ForceTableScan { return fmt.Errorf("cannot use --forceTableScan when specifying --query") } if exp.InputOpts.Query != "" && exp.InputOpts.QueryFile != "" { return fmt.Errorf("either --query or --queryFile can be specified as a query option") } if exp.InputOpts != nil && exp.InputOpts.HasQuery() { content, err := exp.InputOpts.GetQuery() if err != nil { return err } _, err2 := getObjectFromByteArg(content) if err2 != nil { return err2 } } if exp.InputOpts != nil && exp.InputOpts.Sort != "" { _, err := getSortFromArg(exp.InputOpts.Sort) if err != nil { return err } } return nil }
// ParseAndValidateOptions returns a non-nil error if user-supplied options are invalid. func (restore *MongoRestore) ParseAndValidateOptions() error { // Can't use option pkg defaults for --objcheck because it's two separate flags, // and we need to be able to see if they're both being used. We default to // true here and then see if noobjcheck is enabled. log.Logv(log.DebugHigh, "checking options") if restore.InputOptions.Objcheck { restore.objCheck = true log.Logv(log.DebugHigh, "\tdumping with object check enabled") } else { log.Logv(log.DebugHigh, "\tdumping with object check disabled") } if restore.NSOptions.DB == "" && restore.NSOptions.Collection != "" { return fmt.Errorf("cannot restore a collection without a specified database") } if restore.NSOptions.DB != "" { if err := util.ValidateDBName(restore.NSOptions.DB); err != nil { return fmt.Errorf("invalid db name: %v", err) } } if restore.NSOptions.Collection != "" { if err := util.ValidateCollectionGrammar(restore.NSOptions.Collection); err != nil { return fmt.Errorf("invalid collection name: %v", err) } } if restore.InputOptions.RestoreDBUsersAndRoles && restore.NSOptions.DB == "" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles without a specified database") } if restore.InputOptions.RestoreDBUsersAndRoles && restore.NSOptions.DB == "admin" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles with the admin database") } var err error restore.isMongos, err = restore.SessionProvider.IsMongos() if err != nil { return err } if restore.isMongos { log.Logv(log.DebugLow, "restoring to a sharded system") } if restore.InputOptions.OplogLimit != "" { if !restore.InputOptions.OplogReplay { return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled") } restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit) if err != nil { return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err) } } if restore.InputOptions.OplogFile != "" { if !restore.InputOptions.OplogReplay { return fmt.Errorf("cannot use --oplogFile without --oplogReplay enabled") } if restore.InputOptions.Archive != "" { return fmt.Errorf("cannot use --oplogFile with --archive specified") } } // check if we are using a replica set and fall back to w=1 if we aren't (for <= 2.4) nodeType, err := restore.SessionProvider.GetNodeType() if err != nil { return fmt.Errorf("error determining type of connected node: %v", err) } log.Logvf(log.DebugLow, "connected to node type: %v", nodeType) restore.safety, err = db.BuildWriteConcern(restore.OutputOptions.WriteConcern, nodeType) if err != nil { return fmt.Errorf("error parsing write concern: %v", err) } // deprecations with --nsInclude --nsExclude if restore.NSOptions.DB != "" || restore.NSOptions.Collection != "" { // these are only okay if restoring from a bson file _, fileType := restore.getInfoFromFilename(restore.TargetDirectory) if fileType != BSONFileType { log.Logvf(log.Always, "the --db and --collection args should only be used when "+ "restoring from a BSON file. Other uses are deprecated and will not exist "+ "in the future; use --nsInclude instead") } } if len(restore.NSOptions.ExcludedCollections) > 0 || len(restore.NSOptions.ExcludedCollectionPrefixes) > 0 { log.Logvf(log.Always, "the --excludeCollections and --excludeCollectionPrefixes options "+ "are deprecated and will not exist in the future; use --nsExclude instead") } if restore.InputOptions.OplogReplay { if len(restore.NSOptions.NSInclude) > 0 || restore.NSOptions.DB != "" { return fmt.Errorf("cannot use --oplogReplay with includes specified") } if len(restore.NSOptions.NSExclude) > 0 || len(restore.NSOptions.ExcludedCollections) > 0 || len(restore.NSOptions.ExcludedCollectionPrefixes) > 0 { return fmt.Errorf("cannot use --oplogReplay with excludes specified") } if len(restore.NSOptions.NSFrom) > 0 { return fmt.Errorf("cannot use --oplogReplay with namespace renames specified") } } includes := restore.NSOptions.NSInclude if restore.NSOptions.DB != "" && restore.NSOptions.Collection != "" { includes = append(includes, ns.Escape(restore.NSOptions.DB)+"."+ restore.NSOptions.Collection) } else if restore.NSOptions.DB != "" { includes = append(includes, ns.Escape(restore.NSOptions.DB)+".*") } if len(includes) == 0 { includes = []string{"*"} } restore.includer, err = ns.NewMatcher(includes) if err != nil { return fmt.Errorf("invalid includes: %v", err) } if len(restore.NSOptions.ExcludedCollections) > 0 && restore.NSOptions.Collection != "" { return fmt.Errorf("--collection is not allowed when --excludeCollection is specified") } if len(restore.NSOptions.ExcludedCollectionPrefixes) > 0 && restore.NSOptions.Collection != "" { return fmt.Errorf("--collection is not allowed when --excludeCollectionsWithPrefix is specified") } excludes := restore.NSOptions.NSExclude for _, col := range restore.NSOptions.ExcludedCollections { excludes = append(excludes, "*."+ns.Escape(col)) } for _, colPrefix := range restore.NSOptions.ExcludedCollectionPrefixes { excludes = append(excludes, "*."+ns.Escape(colPrefix)+"*") } restore.excluder, err = ns.NewMatcher(excludes) if err != nil { return fmt.Errorf("invalid excludes: %v", err) } if len(restore.NSOptions.NSFrom) != len(restore.NSOptions.NSTo) { return fmt.Errorf("--nsFrom and --nsTo arguments must be specified an equal number of times") } restore.renamer, err = ns.NewRenamer(restore.NSOptions.NSFrom, restore.NSOptions.NSTo) if err != nil { return fmt.Errorf("invalid renames: %v", err) } if restore.OutputOptions.NumInsertionWorkers < 0 { return fmt.Errorf( "cannot specify a negative number of insertion workers per collection") } // a single dash signals reading from stdin if restore.TargetDirectory == "-" { if restore.InputOptions.Archive != "" { return fmt.Errorf( "cannot restore from \"-\" when --archive is specified") } if restore.NSOptions.Collection == "" { return fmt.Errorf("cannot restore from stdin without a specified collection") } } if restore.stdin == nil { restore.stdin = os.Stdin } return nil }
// ValidateSettings ensures that the tool specific options supplied for // MongoImport are valid. func (imp *MongoImport) ValidateSettings(args []string) error { // namespace must have a valid database; if none is specified, use 'test' if imp.ToolOptions.DB == "" { imp.ToolOptions.DB = "test" } err := util.ValidateDBName(imp.ToolOptions.DB) if err != nil { return fmt.Errorf("invalid database name: %v", err) } imp.InputOptions.Type = strings.ToLower(imp.InputOptions.Type) // use JSON as default input type if imp.InputOptions.Type == "" { imp.InputOptions.Type = JSON } else { if !(imp.InputOptions.Type == TSV || imp.InputOptions.Type == JSON || imp.InputOptions.Type == CSV) { return fmt.Errorf("unknown type %v", imp.InputOptions.Type) } } // ensure headers are supplied for CSV/TSV if imp.InputOptions.Type == CSV || imp.InputOptions.Type == TSV { if !imp.InputOptions.HeaderLine { if imp.InputOptions.Fields == nil && imp.InputOptions.FieldFile == nil { return fmt.Errorf("must specify --fields, --fieldFile or --headerline to import this file type") } if imp.InputOptions.FieldFile != nil && *imp.InputOptions.FieldFile == "" { return fmt.Errorf("--fieldFile can not be empty string") } if imp.InputOptions.Fields != nil && imp.InputOptions.FieldFile != nil { return fmt.Errorf("incompatible options: --fields and --fieldFile") } } else { if imp.InputOptions.Fields != nil { return fmt.Errorf("incompatible options: --fields and --headerline") } if imp.InputOptions.FieldFile != nil { return fmt.Errorf("incompatible options: --fieldFile and --headerline") } } if _, err := ValidatePG(imp.InputOptions.ParseGrace); err != nil { return err } } else { // input type is JSON if imp.InputOptions.HeaderLine { return fmt.Errorf("can not use --headerline when input type is JSON") } if imp.InputOptions.Fields != nil { return fmt.Errorf("can not use --fields when input type is JSON") } if imp.InputOptions.FieldFile != nil { return fmt.Errorf("can not use --fieldFile when input type is JSON") } if imp.IngestOptions.IgnoreBlanks { return fmt.Errorf("can not use --ignoreBlanks when input type is JSON") } if imp.InputOptions.ColumnsHaveTypes { return fmt.Errorf("can not use --columnsHaveTypes when input type is JSON") } } // deprecated if imp.IngestOptions.Upsert == true { imp.IngestOptions.Mode = modeUpsert } // parse UpsertFields, may set default mode to modeUpsert if imp.IngestOptions.UpsertFields != "" { if imp.IngestOptions.Mode == "" { imp.IngestOptions.Mode = modeUpsert } else if imp.IngestOptions.Mode == modeInsert { return fmt.Errorf("can not use --upsertFields with --mode=insert") } imp.upsertFields = strings.Split(imp.IngestOptions.UpsertFields, ",") if err := validateFields(imp.upsertFields); err != nil { return fmt.Errorf("invalid --upsertFields argument: %v", err) } } else if imp.IngestOptions.Mode != modeInsert { imp.upsertFields = []string{"_id"} } // set default mode, must be after parsing UpsertFields if imp.IngestOptions.Mode == "" { imp.IngestOptions.Mode = modeInsert } // double-check mode choices if !(imp.IngestOptions.Mode == modeInsert || imp.IngestOptions.Mode == modeUpsert || imp.IngestOptions.Mode == modeMerge) { return fmt.Errorf("invalid --mode argument: %v", imp.IngestOptions.Mode) } if imp.IngestOptions.Mode != modeInsert { imp.IngestOptions.MaintainInsertionOrder = true log.Logvf(log.Info, "using upsert fields: %v", imp.upsertFields) } // set the number of decoding workers to use for imports if imp.IngestOptions.NumDecodingWorkers <= 0 { imp.IngestOptions.NumDecodingWorkers = imp.ToolOptions.MaxProcs } log.Logvf(log.DebugLow, "using %v decoding workers", imp.IngestOptions.NumDecodingWorkers) // set the number of insertion workers to use for imports if imp.IngestOptions.NumInsertionWorkers <= 0 { imp.IngestOptions.NumInsertionWorkers = 1 } log.Logvf(log.DebugLow, "using %v insert workers", imp.IngestOptions.NumInsertionWorkers) // if --maintainInsertionOrder is set, we can only allow 1 insertion worker if imp.IngestOptions.MaintainInsertionOrder { imp.IngestOptions.NumInsertionWorkers = 1 } // get the number of documents per batch if imp.IngestOptions.BulkBufferSize <= 0 || imp.IngestOptions.BulkBufferSize > 1000 { imp.IngestOptions.BulkBufferSize = 1000 } // ensure no more than one positional argument is supplied if len(args) > 1 { return fmt.Errorf("only one positional argument is allowed") } // ensure either a positional argument is supplied or an argument is passed // to the --file flag - and not both if imp.InputOptions.File != "" && len(args) != 0 { return fmt.Errorf("incompatible options: --file and positional argument(s)") } if imp.InputOptions.File == "" { if len(args) != 0 { // if --file is not supplied, use the positional argument supplied imp.InputOptions.File = args[0] } } // ensure we have a valid string to use for the collection if imp.ToolOptions.Collection == "" { log.Logvf(log.Always, "no collection specified") fileBaseName := filepath.Base(imp.InputOptions.File) lastDotIndex := strings.LastIndex(fileBaseName, ".") if lastDotIndex != -1 { fileBaseName = fileBaseName[0:lastDotIndex] } log.Logvf(log.Always, "using filename '%v' as collection", fileBaseName) imp.ToolOptions.Collection = fileBaseName } err = util.ValidateCollectionName(imp.ToolOptions.Collection) if err != nil { return fmt.Errorf("invalid collection name: %v", err) } return nil }
// ParseAndValidateOptions returns a non-nil error if user-supplied options are invalid. func (restore *MongoRestore) ParseAndValidateOptions() error { // Can't use option pkg defaults for --objcheck because it's two separate flags, // and we need to be able to see if they're both being used. We default to // true here and then see if noobjcheck is enabled. log.Log(log.DebugHigh, "checking options") if restore.InputOptions.Objcheck { restore.objCheck = true log.Log(log.DebugHigh, "\tdumping with object check enabled") } else { log.Log(log.DebugHigh, "\tdumping with object check disabled") } if restore.ToolOptions.DB == "" && restore.ToolOptions.Collection != "" { return fmt.Errorf("cannot dump a collection without a specified database") } if restore.ToolOptions.DB != "" { if err := util.ValidateDBName(restore.ToolOptions.DB); err != nil { return fmt.Errorf("invalid db name: %v", err) } } if restore.ToolOptions.Collection != "" { if err := util.ValidateCollectionGrammar(restore.ToolOptions.Collection); err != nil { return fmt.Errorf("invalid collection name: %v", err) } } if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles without a specified database") } if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "admin" { return fmt.Errorf("cannot use --restoreDbUsersAndRoles with the admin database") } var err error restore.isMongos, err = restore.SessionProvider.IsMongos() if err != nil { return err } if restore.isMongos { log.Log(log.DebugLow, "restoring to a sharded system") } if restore.InputOptions.OplogLimit != "" { if !restore.InputOptions.OplogReplay { return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled") } restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit) if err != nil { return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err) } } // check if we are using a replica set and fall back to w=1 if we aren't (for <= 2.4) nodeType, err := restore.SessionProvider.GetNodeType() if err != nil { return fmt.Errorf("error determining type of connected node: %v", err) } log.Logf(log.DebugLow, "connected to node type: %v", nodeType) restore.safety, err = db.BuildWriteConcern(restore.OutputOptions.WriteConcern, nodeType) if err != nil { return fmt.Errorf("error parsing write concern: %v", err) } // handle the hidden auth collection flags if restore.ToolOptions.HiddenOptions.TempUsersColl == nil { restore.tempUsersCol = "tempusers" } else { restore.tempUsersCol = *restore.ToolOptions.HiddenOptions.TempUsersColl } if restore.ToolOptions.HiddenOptions.TempRolesColl == nil { restore.tempRolesCol = "temproles" } else { restore.tempRolesCol = *restore.ToolOptions.HiddenOptions.TempRolesColl } if restore.OutputOptions.NumInsertionWorkers < 0 { return fmt.Errorf( "cannot specify a negative number of insertion workers per collection") } // a single dash signals reading from stdin if restore.TargetDirectory == "-" { restore.useStdin = true if restore.InputOptions.Archive != "" { return fmt.Errorf( "cannot restore from \"-\" when --archive is specified") } if restore.ToolOptions.Collection == "" { return fmt.Errorf("cannot restore from stdin without a specified collection") } } return nil }
// CreateAllIntents drills down into a dump folder, creating intents for all of // the databases and collections it finds. func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike, filterDB string, filterCollection string) error { log.Logf(log.DebugHigh, "using %v as dump root directory", dir.Path()) foundOplog := false entries, err := dir.ReadDir() if err != nil { return fmt.Errorf("error reading root dump folder: %v", err) } for _, entry := range entries { if entry.IsDir() { if err = util.ValidateDBName(entry.Name()); err != nil { return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err) } if filterDB == "" || entry.Name() == filterDB { err = restore.CreateIntentsForDB(entry.Name(), filterCollection, entry, false) } else { err = restore.CreateIntentsForDB(entry.Name(), "", entry, true) } if err != nil { return err } } else { if entry.Name() == "oplog.bson" { if restore.InputOptions.OplogReplay { log.Log(log.DebugLow, "found oplog.bson file to replay") } foundOplog = true oplogIntent := &intents.Intent{ C: "oplog", Size: entry.Size(), Location: entry.Path(), } // filterDB is used to mimic CreateIntentsForDB, and since CreateIntentsForDB wouldn't // apply the oplog, even when asked, we don't either. if filterDB != "" || !restore.InputOptions.OplogReplay { if restore.InputOptions.Archive == "" { continue } else { mutedOut := &archive.MutedCollection{ Intent: oplogIntent, Demux: restore.archive.Demux, } restore.archive.Demux.Open( oplogIntent.Namespace(), mutedOut, ) continue } } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { oplogIntent.Location = "archive on stdin" } else { oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } // no need to check that we want to cache here oplogIntent.BSONFile = &archive.RegularCollectionReceiver{ Intent: oplogIntent, Demux: restore.archive.Demux, } } else { oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip} } restore.manager.Put(oplogIntent) } else { log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } if restore.InputOptions.OplogReplay && !foundOplog { return fmt.Errorf("no %v/oplog.bson file to replay; make sure you run mongodump with --oplog", dir.Path()) } return nil }
// CreateAllIntents drills down into a dump folder, creating intents for all of // the databases and collections it finds. func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike) error { log.Logvf(log.DebugHigh, "using %v as dump root directory", dir.Path()) entries, err := dir.ReadDir() if err != nil { return fmt.Errorf("error reading root dump folder: %v", err) } for _, entry := range entries { if entry.IsDir() { if err = util.ValidateDBName(entry.Name()); err != nil { return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err) } err = restore.CreateIntentsForDB(entry.Name(), entry) if err != nil { return err } } else { if entry.Name() == "oplog.bson" { if restore.InputOptions.OplogReplay { log.Logv(log.DebugLow, "found oplog.bson file to replay") } oplogIntent := &intents.Intent{ C: "oplog", Size: entry.Size(), Location: entry.Path(), } if !restore.InputOptions.OplogReplay { if restore.InputOptions.Archive != "" { mutedOut := &archive.MutedCollection{ Intent: oplogIntent, Demux: restore.archive.Demux, } restore.archive.Demux.Open( oplogIntent.Namespace(), mutedOut, ) } continue } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { oplogIntent.Location = "archive on stdin" } else { oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } // no need to check that we want to cache here oplogIntent.BSONFile = &archive.RegularCollectionReceiver{ Intent: oplogIntent, Origin: oplogIntent.Namespace(), Demux: restore.archive.Demux, } } else { oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip} } restore.manager.Put(oplogIntent) } else { log.Logvf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } return nil }