// CreateAllIntents drills down into a dump folder, creating intents for all of // the databases and collections it finds. func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike, filterDB string, filterCollection string) error { log.Logf(log.DebugHigh, "using %v as dump root directory", dir.Path()) foundOplog := false entries, err := dir.ReadDir() if err != nil { return fmt.Errorf("error reading root dump folder: %v", err) } for _, entry := range entries { if entry.IsDir() { if err = util.ValidateDBName(entry.Name()); err != nil { return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err) } if filterDB == "" || entry.Name() == filterDB { err = restore.CreateIntentsForDB(entry.Name(), filterCollection, entry, false) } else { err = restore.CreateIntentsForDB(entry.Name(), "", entry, true) } if err != nil { return err } } else { if entry.Name() == "oplog.bson" { if restore.InputOptions.OplogReplay { log.Log(log.DebugLow, "found oplog.bson file to replay") } foundOplog = true oplogIntent := &intents.Intent{ C: "oplog", Size: entry.Size(), Location: entry.Path(), } // filterDB is used to mimic CreateIntentsForDB, and since CreateIntentsForDB wouldn't // apply the oplog, even when asked, we don't either. if filterDB != "" || !restore.InputOptions.OplogReplay { if restore.InputOptions.Archive == "" { continue } else { mutedOut := &archive.MutedCollection{ Intent: oplogIntent, Demux: restore.archive.Demux, } restore.archive.Demux.Open( oplogIntent.Namespace(), mutedOut, ) continue } } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { oplogIntent.Location = "archive on stdin" } else { oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } // no need to check that we want to cache here oplogIntent.BSONFile = &archive.RegularCollectionReceiver{ Intent: oplogIntent, Demux: restore.archive.Demux, } } else { oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip} } restore.manager.Put(oplogIntent) } else { log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } if restore.InputOptions.OplogReplay && !foundOplog { return fmt.Errorf("no %v/oplog.bson file to replay; make sure you run mongodump with --oplog", dir.Path()) } return nil }
// CreateIntentsForDB drills down into the dir folder, creating intents // for all of the collection dump files it finds for the db database. func (restore *MongoRestore) CreateIntentsForDB(db string, filterCollection string, dir archive.DirLike, mute bool) (err error) { var entries []archive.DirLike log.Logf(log.DebugHigh, "reading collections for database %v in %v", db, dir.Name()) entries, err = dir.ReadDir() if err != nil { return fmt.Errorf("error reading db folder %v: %v", db, err) } usesMetadataFiles := hasMetadataFiles(entries) for _, entry := range entries { if entry.IsDir() { log.Logf(log.Always, `don't know what to do with subdirectory "%v", skipping...`, filepath.Join(dir.Name(), entry.Name())) } else { collection, fileType := restore.getInfoFromFilename(entry.Name()) switch fileType { case BSONFileType: var skip = mute // Dumps of a single database (i.e. with the -d flag) may contain special // db-specific collections that start with a "$" (for example, $admin.system.users // holds the users for a database that was dumped with --dumpDbUsersAndRoles enabled). // If these special files manage to be included in a dump directory during a full // (multi-db) restore, we should ignore them. if restore.ToolOptions.DB == "" && strings.HasPrefix(collection, "$") { log.Logf(log.DebugLow, "not restoring special collection %v.%v", db, collection) skip = true } // TOOLS-717: disallow restoring to the system.profile collection. // Server versions >= 3.0.3 disallow user inserts to system.profile so // it would likely fail anyway. if collection == "system.profile" { log.Logf(log.DebugLow, "skipping restore of system.profile collection", db) skip = true } // skip restoring the indexes collection if we are using metadata // files to store index information, to eliminate redundancy if collection == "system.indexes" && usesMetadataFiles { log.Logf(log.DebugLow, "not restoring system.indexes collection because database %v "+ "has .metadata.json files", db) skip = true } // TOOLS-976: skip restoring the collections should be excluded if filterCollection == "" && restore.shouldSkipCollection(collection) { log.Logf(log.DebugLow, "skipping restoring %v.%v, it is excluded", db, collection) skip = true } if filterCollection != "" && filterCollection != collection { skip = true } intent := &intents.Intent{ DB: db, C: collection, Size: entry.Size(), } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { intent.Location = "archive on stdin" } else { intent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } if skip { // adding the DemuxOut to the demux, but not adding the intent to the manager mutedOut := &archive.MutedCollection{Intent: intent, Demux: restore.archive.Demux} restore.archive.Demux.Open(intent.Namespace(), mutedOut) continue } else { if intent.IsSpecialCollection() { intent.BSONFile = &archive.SpecialCollectionCache{Intent: intent, Demux: restore.archive.Demux} restore.archive.Demux.Open(intent.Namespace(), intent.BSONFile) } else { intent.BSONFile = &archive.RegularCollectionReceiver{Intent: intent, Demux: restore.archive.Demux} } } } else { if skip { continue } intent.Location = entry.Path() intent.BSONFile = &realBSONFile{path: entry.Path(), intent: intent, gzip: restore.InputOptions.Gzip} } log.Logf(log.Info, "found collection %v bson to restore", intent.Namespace()) restore.manager.Put(intent) case MetadataFileType: // TOOLS-976: skip restoring the collections should be excluded if filterCollection == "" && restore.shouldSkipCollection(collection) { log.Logf(log.DebugLow, "skipping restoring %v.%v metadata, it is excluded", db, collection) continue } usesMetadataFiles = true intent := &intents.Intent{ DB: db, C: collection, } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { intent.MetadataLocation = "archive on stdin" } else { intent.MetadataLocation = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } intent.MetadataFile = &archive.MetadataPreludeFile{Intent: intent, Prelude: restore.archive.Prelude} } else { intent.MetadataLocation = entry.Path() intent.MetadataFile = &realMetadataFile{path: entry.Path(), intent: intent, gzip: restore.InputOptions.Gzip} } log.Logf(log.Info, "found collection %v metadata to restore", intent.Namespace()) restore.manager.Put(intent) default: log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } return nil }
// CreateAllIntents drills down into a dump folder, creating intents for all of // the databases and collections it finds. func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike) error { log.Logvf(log.DebugHigh, "using %v as dump root directory", dir.Path()) entries, err := dir.ReadDir() if err != nil { return fmt.Errorf("error reading root dump folder: %v", err) } for _, entry := range entries { if entry.IsDir() { if err = util.ValidateDBName(entry.Name()); err != nil { return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err) } err = restore.CreateIntentsForDB(entry.Name(), entry) if err != nil { return err } } else { if entry.Name() == "oplog.bson" { if restore.InputOptions.OplogReplay { log.Logv(log.DebugLow, "found oplog.bson file to replay") } oplogIntent := &intents.Intent{ C: "oplog", Size: entry.Size(), Location: entry.Path(), } if !restore.InputOptions.OplogReplay { if restore.InputOptions.Archive != "" { mutedOut := &archive.MutedCollection{ Intent: oplogIntent, Demux: restore.archive.Demux, } restore.archive.Demux.Open( oplogIntent.Namespace(), mutedOut, ) } continue } if restore.InputOptions.Archive != "" { if restore.InputOptions.Archive == "-" { oplogIntent.Location = "archive on stdin" } else { oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive) } // no need to check that we want to cache here oplogIntent.BSONFile = &archive.RegularCollectionReceiver{ Intent: oplogIntent, Origin: oplogIntent.Namespace(), Demux: restore.archive.Demux, } } else { oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip} } restore.manager.Put(oplogIntent) } else { log.Logvf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path()) } } } return nil }