// Restore runs the mongorestore program.
func (restore *MongoRestore) Restore() error {
var target archive.DirLike
err := restore.ParseAndValidateOptions()
if err != nil {
log.Logvf(log.DebugLow, "got error from options parsing: %v", err)
return err
}
// Build up all intents to be restored
restore.manager = intents.NewIntentManager()
if restore.InputOptions.Archive == "" && restore.InputOptions.OplogReplay {
restore.manager.SetSmartPickOplog(true)
}
if restore.InputOptions.Archive != "" {
archiveReader, err := restore.getArchiveReader()
if err != nil {
return err
}
restore.archive = &archive.Reader{
In: archiveReader,
Prelude: &archive.Prelude{},
}
err = restore.archive.Prelude.Read(restore.archive.In)
if err != nil {
return err
}
log.Logvf(log.DebugLow, `archive format version "%v"`, restore.archive.Prelude.Header.FormatVersion)
log.Logvf(log.DebugLow, `archive server version "%v"`, restore.archive.Prelude.Header.ServerVersion)
log.Logvf(log.DebugLow, `archive tool version "%v"`, restore.archive.Prelude.Header.ToolVersion)
target, err = restore.archive.Prelude.NewPreludeExplorer()
if err != nil {
return err
}
} else if restore.TargetDirectory != "-" {
var usedDefaultTarget bool
if restore.TargetDirectory == "" {
restore.TargetDirectory = "dump"
log.Logv(log.Always, "using default 'dump' directory")
usedDefaultTarget = true
}
target, err = newActualPath(restore.TargetDirectory)
if err != nil {
if usedDefaultTarget {
log.Logv(log.Always, "see mongorestore --help for usage information")
}
return fmt.Errorf("mongorestore target '%v' invalid: %v", restore.TargetDirectory, err)
}
// handle cases where the user passes in a file instead of a directory
if !target.IsDir() {
log.Logv(log.DebugLow, "mongorestore target is a file, not a directory")
err = restore.handleBSONInsteadOfDirectory(restore.TargetDirectory)
if err != nil {
return err
}
} else {
log.Logv(log.DebugLow, "mongorestore target is a directory, not a file")
}
}
if restore.NSOptions.Collection != "" &&
restore.OutputOptions.NumParallelCollections > 1 &&
restore.OutputOptions.NumInsertionWorkers == 1 {
// handle special parallelization case when we are only restoring one collection
// by mapping -j to insertion workers rather than parallel collections
log.Logvf(log.DebugHigh,
"setting number of insertions workers to number of parallel collections (%v)",
restore.OutputOptions.NumParallelCollections)
restore.OutputOptions.NumInsertionWorkers = restore.OutputOptions.NumParallelCollections
}
if restore.InputOptions.Archive != "" {
if int(restore.archive.Prelude.Header.ConcurrentCollections) > restore.OutputOptions.NumParallelCollections {
restore.OutputOptions.NumParallelCollections = int(restore.archive.Prelude.Header.ConcurrentCollections)
restore.OutputOptions.NumInsertionWorkers = int(restore.archive.Prelude.Header.ConcurrentCollections)
log.Logvf(log.Always,
"setting number of parallel collections to number of parallel collections in archive (%v)",
restore.archive.Prelude.Header.ConcurrentCollections,
)
}
}
// Create the demux before intent creation, because muted archive intents need
// to register themselves with the demux directly
if restore.InputOptions.Archive != "" {
restore.archive.Demux = &archive.Demultiplexer{
In: restore.archive.In,
}
}
switch {
case restore.InputOptions.Archive != "":
log.Logvf(log.Always, "preparing collections to restore from")
err = restore.CreateAllIntents(target)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection == "":
log.Logvf(log.Always,
"building a list of collections to restore from %v dir",
target.Path())
err = restore.CreateIntentsForDB(
restore.NSOptions.DB,
target,
)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "" && restore.TargetDirectory == "-":
log.Logvf(log.Always, "setting up a collection to be read from standard input")
err = restore.CreateStdinIntentForCollection(
restore.NSOptions.DB,
restore.NSOptions.Collection,
)
case restore.NSOptions.DB != "" && restore.NSOptions.Collection != "":
log.Logvf(log.Always, "checking for collection data in %v", target.Path())
err = restore.CreateIntentForCollection(
restore.NSOptions.DB,
restore.NSOptions.Collection,
target,
)
default:
log.Logvf(log.Always, "preparing collections to restore from")
err = restore.CreateAllIntents(target)
}
if err != nil {
return fmt.Errorf("error scanning filesystem: %v", err)
}
if restore.isMongos && restore.manager.HasConfigDBIntent() && restore.NSOptions.DB == "" {
return fmt.Errorf("cannot do a full restore on a sharded system - " +
"remove the 'config' directory from the dump directory first")
}
if restore.InputOptions.OplogFile != "" {
err = restore.CreateIntentForOplog()
if err != nil {
return fmt.Errorf("error reading oplog file: %v", err)
}
}
if restore.InputOptions.OplogReplay && restore.manager.Oplog() == nil {
return fmt.Errorf("no oplog file to replay; make sure you run mongodump with --oplog")
}
if restore.manager.GetOplogConflict() {
return fmt.Errorf("cannot provide both an oplog.bson file and an oplog file with --oplogFile, " +
"nor can you provide both a local/oplog.rs.bson and a local/oplog.$main.bson file.")
}
conflicts := restore.manager.GetDestinationConflicts()
if len(conflicts) > 0 {
for _, conflict := range conflicts {
log.Logvf(log.Always, "%s", conflict.Error())
}
return fmt.Errorf("cannot restore with conflicting namespace destinations")
}
if restore.OutputOptions.DryRun {
log.Logvf(log.Always, "dry run completed")
return nil
}
if restore.InputOptions.Archive != "" {
namespaceChan := make(chan string, 1)
namespaceErrorChan := make(chan error)
restore.archive.Demux.NamespaceChan = namespaceChan
restore.archive.Demux.NamespaceErrorChan = namespaceErrorChan
go restore.archive.Demux.Run()
// consume the new namespace announcement from the demux for all of the special collections
// that get cached when being read out of the archive.
// The first regular collection found gets pushed back on to the namespaceChan
// consume the new namespace announcement from the demux for all of the collections that get cached
for {
ns, ok := <-namespaceChan
// the archive can have only special collections. In that case we keep reading until
// the namespaces are exhausted, indicated by the namespaceChan being closed.
if !ok {
break
}
intent := restore.manager.IntentForNamespace(ns)
if intent == nil {
return fmt.Errorf("no intent for collection in archive: %v", ns)
}
if intent.IsSystemIndexes() ||
intent.IsUsers() ||
intent.IsRoles() ||
intent.IsAuthVersion() {
log.Logvf(log.DebugLow, "special collection %v found", ns)
namespaceErrorChan <- nil
} else {
// Put the ns back on the announcement chan so that the
// demultiplexer can start correctly
log.Logvf(log.DebugLow, "first non special collection %v found."+
" The demultiplexer will handle it and the remainder", ns)
namespaceChan <- ns
break
}
}
}
// If restoring users and roles, make sure we validate auth versions
if restore.ShouldRestoreUsersAndRoles() {
log.Logv(log.Info, "comparing auth version of the dump directory and target server")
restore.authVersions.Dump, err = restore.GetDumpAuthVersion()
if err != nil {
return fmt.Errorf("error getting auth version from dump: %v", err)
}
restore.authVersions.Server, err = auth.GetAuthVersion(restore.SessionProvider)
if err != nil {
return fmt.Errorf("error getting auth version of server: %v", err)
}
err = restore.ValidateAuthVersions()
if err != nil {
return fmt.Errorf(
"the users and roles collections in the dump have an incompatible auth version with target server: %v",
err)
}
}
err = restore.LoadIndexesFromBSON()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
// Restore the regular collections
if restore.InputOptions.Archive != "" {
restore.manager.UsePrioritizer(restore.archive.Demux.NewPrioritizer(restore.manager))
} else if restore.OutputOptions.NumParallelCollections > 1 {
restore.manager.Finalize(intents.MultiDatabaseLTF)
} else {
// use legacy restoration order if we are single-threaded
restore.manager.Finalize(intents.Legacy)
}
restore.termChan = make(chan struct{})
if err := restore.RestoreIntents(); err != nil {
return err
}
// Restore users/roles
if restore.ShouldRestoreUsersAndRoles() {
err = restore.RestoreUsersOrRoles(restore.manager.Users(), restore.manager.Roles())
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
// Restore oplog
if restore.InputOptions.OplogReplay {
err = restore.RestoreOplog()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
log.Logv(log.Always, "done")
return nil
}
// CreateIntentForCollection builds an intent for the given database and collection name
// along with a path to a .bson collection file. It searches the file's parent directory
// for a matching metadata file.
//
// This method is not called by CreateIntentsForDB,
// it is only used in the case where --db and --collection flags are set.
func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, dir archive.DirLike) error {
log.Logf(log.DebugLow, "reading collection %v for database %v from %v",
collection, db, dir.Path())
// first make sure the bson file exists and is valid
_, err := dir.Stat()
if err != nil {
return err
}
if dir.IsDir() {
return fmt.Errorf("file %v is a directory, not a bson file", dir.Path())
}
baseName, fileType := restore.getInfoFromFilename(dir.Name())
if fileType != BSONFileType {
return fmt.Errorf("file %v does not have .bson extension", dir.Path())
}
// then create its intent
intent := &intents.Intent{
DB: db,
C: collection,
Size: dir.Size(),
Location: dir.Path(),
}
intent.BSONFile = &realBSONFile{path: dir.Path(), intent: intent, gzip: restore.InputOptions.Gzip}
// finally, check if it has a .metadata.json file in its folder
log.Logf(log.DebugLow, "scanning directory %v for metadata", dir.Name())
entries, err := dir.Parent().ReadDir()
if err != nil {
// try and carry on if we can
log.Logf(log.Info, "error attempting to locate metadata for file: %v", err)
log.Log(log.Info, "restoring collection without metadata")
restore.manager.Put(intent)
return nil
}
metadataName := baseName + ".metadata.json"
for _, entry := range entries {
if entry.Name() == metadataName {
metadataPath := entry.Path()
log.Logf(log.Info, "found metadata for collection at %v", metadataPath)
intent.MetadataLocation = metadataPath
intent.MetadataFile = &realMetadataFile{path: metadataPath, intent: intent, gzip: restore.InputOptions.Gzip}
break
}
}
if intent.MetadataFile == nil {
log.Log(log.Info, "restoring collection without metadata")
}
restore.manager.Put(intent)
return nil
}
// CreateAllIntents drills down into a dump folder, creating intents for all of
// the databases and collections it finds.
func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike, filterDB string, filterCollection string) error {
log.Logf(log.DebugHigh, "using %v as dump root directory", dir.Path())
foundOplog := false
entries, err := dir.ReadDir()
if err != nil {
return fmt.Errorf("error reading root dump folder: %v", err)
}
for _, entry := range entries {
if entry.IsDir() {
if err = util.ValidateDBName(entry.Name()); err != nil {
return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err)
}
if filterDB == "" || entry.Name() == filterDB {
err = restore.CreateIntentsForDB(entry.Name(), filterCollection, entry, false)
} else {
err = restore.CreateIntentsForDB(entry.Name(), "", entry, true)
}
if err != nil {
return err
}
} else {
if entry.Name() == "oplog.bson" {
if restore.InputOptions.OplogReplay {
log.Log(log.DebugLow, "found oplog.bson file to replay")
}
foundOplog = true
oplogIntent := &intents.Intent{
C: "oplog",
Size: entry.Size(),
Location: entry.Path(),
}
// filterDB is used to mimic CreateIntentsForDB, and since CreateIntentsForDB wouldn't
// apply the oplog, even when asked, we don't either.
if filterDB != "" || !restore.InputOptions.OplogReplay {
if restore.InputOptions.Archive == "" {
continue
} else {
mutedOut := &archive.MutedCollection{
Intent: oplogIntent,
Demux: restore.archive.Demux,
}
restore.archive.Demux.Open(
oplogIntent.Namespace(),
mutedOut,
)
continue
}
}
if restore.InputOptions.Archive != "" {
if restore.InputOptions.Archive == "-" {
oplogIntent.Location = "archive on stdin"
} else {
oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive)
}
// no need to check that we want to cache here
oplogIntent.BSONFile =
&archive.RegularCollectionReceiver{
Intent: oplogIntent,
Demux: restore.archive.Demux,
}
} else {
oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip}
}
restore.manager.Put(oplogIntent)
} else {
log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path())
}
}
}
if restore.InputOptions.OplogReplay && !foundOplog {
return fmt.Errorf("no %v/oplog.bson file to replay; make sure you run mongodump with --oplog", dir.Path())
}
return nil
}
// CreateIntentsForDB drills down into the dir folder, creating intents
// for all of the collection dump files it finds for the db database.
func (restore *MongoRestore) CreateIntentsForDB(db string, filterCollection string, dir archive.DirLike, mute bool) (err error) {
var entries []archive.DirLike
log.Logf(log.DebugHigh, "reading collections for database %v in %v", db, dir.Name())
entries, err = dir.ReadDir()
if err != nil {
return fmt.Errorf("error reading db folder %v: %v", db, err)
}
usesMetadataFiles := hasMetadataFiles(entries)
for _, entry := range entries {
if entry.IsDir() {
log.Logf(log.Always, `don't know what to do with subdirectory "%v", skipping...`,
filepath.Join(dir.Name(), entry.Name()))
} else {
collection, fileType := restore.getInfoFromFilename(entry.Name())
switch fileType {
case BSONFileType:
var skip = mute
// Dumps of a single database (i.e. with the -d flag) may contain special
// db-specific collections that start with a "$" (for example, $admin.system.users
// holds the users for a database that was dumped with --dumpDbUsersAndRoles enabled).
// If these special files manage to be included in a dump directory during a full
// (multi-db) restore, we should ignore them.
if restore.ToolOptions.DB == "" && strings.HasPrefix(collection, "$") {
log.Logf(log.DebugLow, "not restoring special collection %v.%v", db, collection)
skip = true
}
// TOOLS-717: disallow restoring to the system.profile collection.
// Server versions >= 3.0.3 disallow user inserts to system.profile so
// it would likely fail anyway.
if collection == "system.profile" {
log.Logf(log.DebugLow, "skipping restore of system.profile collection", db)
skip = true
}
// skip restoring the indexes collection if we are using metadata
// files to store index information, to eliminate redundancy
if collection == "system.indexes" && usesMetadataFiles {
log.Logf(log.DebugLow,
"not restoring system.indexes collection because database %v "+
"has .metadata.json files", db)
skip = true
}
// TOOLS-976: skip restoring the collections should be excluded
if filterCollection == "" && restore.shouldSkipCollection(collection) {
log.Logf(log.DebugLow, "skipping restoring %v.%v, it is excluded", db, collection)
skip = true
}
if filterCollection != "" && filterCollection != collection {
skip = true
}
intent := &intents.Intent{
DB: db,
C: collection,
Size: entry.Size(),
}
if restore.InputOptions.Archive != "" {
if restore.InputOptions.Archive == "-" {
intent.Location = "archive on stdin"
} else {
intent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive)
}
if skip {
// adding the DemuxOut to the demux, but not adding the intent to the manager
mutedOut := &archive.MutedCollection{Intent: intent, Demux: restore.archive.Demux}
restore.archive.Demux.Open(intent.Namespace(), mutedOut)
continue
} else {
if intent.IsSpecialCollection() {
intent.BSONFile = &archive.SpecialCollectionCache{Intent: intent, Demux: restore.archive.Demux}
restore.archive.Demux.Open(intent.Namespace(), intent.BSONFile)
} else {
intent.BSONFile = &archive.RegularCollectionReceiver{Intent: intent, Demux: restore.archive.Demux}
}
}
} else {
if skip {
continue
}
intent.Location = entry.Path()
intent.BSONFile = &realBSONFile{path: entry.Path(), intent: intent, gzip: restore.InputOptions.Gzip}
}
log.Logf(log.Info, "found collection %v bson to restore", intent.Namespace())
restore.manager.Put(intent)
case MetadataFileType:
// TOOLS-976: skip restoring the collections should be excluded
if filterCollection == "" && restore.shouldSkipCollection(collection) {
log.Logf(log.DebugLow, "skipping restoring %v.%v metadata, it is excluded", db, collection)
continue
}
usesMetadataFiles = true
intent := &intents.Intent{
DB: db,
C: collection,
}
if restore.InputOptions.Archive != "" {
if restore.InputOptions.Archive == "-" {
intent.MetadataLocation = "archive on stdin"
} else {
intent.MetadataLocation = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive)
}
intent.MetadataFile = &archive.MetadataPreludeFile{Intent: intent, Prelude: restore.archive.Prelude}
} else {
intent.MetadataLocation = entry.Path()
intent.MetadataFile = &realMetadataFile{path: entry.Path(), intent: intent, gzip: restore.InputOptions.Gzip}
}
log.Logf(log.Info, "found collection %v metadata to restore", intent.Namespace())
restore.manager.Put(intent)
default:
log.Logf(log.Always, `don't know what to do with file "%v", skipping...`,
entry.Path())
}
}
}
return nil
}
// CreateAllIntents drills down into a dump folder, creating intents for all of
// the databases and collections it finds.
func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike) error {
log.Logvf(log.DebugHigh, "using %v as dump root directory", dir.Path())
entries, err := dir.ReadDir()
if err != nil {
return fmt.Errorf("error reading root dump folder: %v", err)
}
for _, entry := range entries {
if entry.IsDir() {
if err = util.ValidateDBName(entry.Name()); err != nil {
return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err)
}
err = restore.CreateIntentsForDB(entry.Name(), entry)
if err != nil {
return err
}
} else {
if entry.Name() == "oplog.bson" {
if restore.InputOptions.OplogReplay {
log.Logv(log.DebugLow, "found oplog.bson file to replay")
}
oplogIntent := &intents.Intent{
C: "oplog",
Size: entry.Size(),
Location: entry.Path(),
}
if !restore.InputOptions.OplogReplay {
if restore.InputOptions.Archive != "" {
mutedOut := &archive.MutedCollection{
Intent: oplogIntent,
Demux: restore.archive.Demux,
}
restore.archive.Demux.Open(
oplogIntent.Namespace(),
mutedOut,
)
}
continue
}
if restore.InputOptions.Archive != "" {
if restore.InputOptions.Archive == "-" {
oplogIntent.Location = "archive on stdin"
} else {
oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive)
}
// no need to check that we want to cache here
oplogIntent.BSONFile = &archive.RegularCollectionReceiver{
Intent: oplogIntent,
Origin: oplogIntent.Namespace(),
Demux: restore.archive.Demux,
}
} else {
oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip}
}
restore.manager.Put(oplogIntent)
} else {
log.Logvf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path())
}
}
}
return nil
}