// CreateIntentForCollection builds an intent for the given database and collection name
// along with a path to a .bson collection file. It searches the file's parent directory
// for a matching metadata file.
//
// This method is not called by CreateIntentsForDB,
// it is only used in the case where --db and --collection flags are set.
func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, dir archive.DirLike) error {
log.Logf(log.DebugLow, "reading collection %v for database %v from %v",
collection, db, dir.Path())
// first make sure the bson file exists and is valid
_, err := dir.Stat()
if err != nil {
return err
}
if dir.IsDir() {
return fmt.Errorf("file %v is a directory, not a bson file", dir.Path())
}
baseName, fileType := restore.getInfoFromFilename(dir.Name())
if fileType != BSONFileType {
return fmt.Errorf("file %v does not have .bson extension", dir.Path())
}
// then create its intent
intent := &intents.Intent{
DB: db,
C: collection,
Size: dir.Size(),
Location: dir.Path(),
}
intent.BSONFile = &realBSONFile{path: dir.Path(), intent: intent, gzip: restore.InputOptions.Gzip}
// finally, check if it has a .metadata.json file in its folder
log.Logf(log.DebugLow, "scanning directory %v for metadata", dir.Name())
entries, err := dir.Parent().ReadDir()
if err != nil {
// try and carry on if we can
log.Logf(log.Info, "error attempting to locate metadata for file: %v", err)
log.Log(log.Info, "restoring collection without metadata")
restore.manager.Put(intent)
return nil
}
metadataName := baseName + ".metadata.json"
if restore.InputOptions.Gzip {
metadataName += ".gz"
}
for _, entry := range entries {
if entry.Name() == metadataName {
metadataPath := entry.Path()
log.Logf(log.Info, "found metadata for collection at %v", metadataPath)
intent.MetadataLocation = metadataPath
intent.MetadataFile = &realMetadataFile{path: metadataPath, intent: intent, gzip: restore.InputOptions.Gzip}
break
}
}
if intent.MetadataFile == nil {
log.Log(log.Info, "restoring collection without metadata")
}
restore.manager.Put(intent)
return nil
}
// HandleSignals listens for either SIGTERM, SIGINT or the
// SIGHUP signal. It ends restore reads for all goroutines
// as soon as any of those signals is received.
func HandleSignals(dump *MongoDump) {
log.Log(log.DebugLow, "will listen for SIGTERM, SIGINT and SIGHUP")
sigChan := make(chan os.Signal, 2)
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
// first signal cleanly terminates dump writes
<-sigChan
log.Log(log.Always, "ending dump writes")
close(dump.termChan)
// second signal exits immediately
<-sigChan
log.Log(log.Always, "forcefully terminating mongodump")
os.Exit(util.ExitKill)
}
// handleSignals listens for either SIGTERM, SIGINT or the
// SIGHUP signal. It ends restore reads for all goroutines
// as soon as any of those signals is received.
func (restore *MongoRestore) handleSignals() {
log.Log(log.DebugLow, "will listen for SIGTERM, SIGINT and SIGHUP")
sigChan := make(chan os.Signal, 2)
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
// first signal cleanly terminates restore reads
<-sigChan
log.Log(log.Always, "ending restore reads")
close(restore.termChan)
// second signal exits immediately
<-sigChan
log.Log(log.Always, "forcefully terminating mongorestore")
os.Exit(util.ExitKill)
}
// Prompt displays a prompt asking for the password and returns the
// password the user enters as a string.
func Prompt() string {
var pass string
if IsTerminal() {
log.Log(log.DebugLow, "standard input is a terminal; reading password from terminal")
fmt.Fprintf(os.Stderr, "Enter password:"******"reading password from standard input")
fmt.Fprintf(os.Stderr, "Enter password:")
pass = readPassFromStdin()
}
fmt.Fprintln(os.Stderr)
return pass
}
// getTargetDirFromArgs handles the logic and error cases of figuring out
// the target restore directory.
func getTargetDirFromArgs(extraArgs []string, dirFlag string) (string, error) {
// This logic is in a switch statement so that the rules are understandable.
// We start by handling error cases, and then handle the different ways the target
// directory can be legally set.
switch {
case len(extraArgs) > 1:
// error on cases when there are too many positional arguments
return "", fmt.Errorf("too many positional arguments")
case dirFlag != "" && len(extraArgs) > 0:
// error when positional arguments and --dir are used
return "", fmt.Errorf(
"cannot use both --dir and a positional argument to set the target directory")
case len(extraArgs) == 1:
// a nice, simple case where one argument is given, so we use it
return extraArgs[0], nil
case dirFlag != "":
// if we have no extra args and a --dir flag, use the --dir flag
log.Log(log.Info, "using --dir flag instead of arguments")
return dirFlag, nil
default:
return "", nil
}
}
// Finalize processes the intents for prioritization. Currently only two
// kinds of prioritizers are supported. No more "Put" operations may be done
// after finalize is called.
func (manager *Manager) Finalize(pType PriorityType) {
switch pType {
case Legacy:
log.Log(log.DebugHigh, "finalizing intent manager with legacy prioritizer")
manager.prioritizer = NewLegacyPrioritizer(manager.intentsByDiscoveryOrder)
case LongestTaskFirst:
log.Log(log.DebugHigh, "finalizing intent manager with longest task first prioritizer")
manager.prioritizer = NewLongestTaskFirstPrioritizer(manager.intentsByDiscoveryOrder)
case MultiDatabaseLTF:
log.Log(log.DebugHigh, "finalizing intent manager with multi-database longest task first prioritizer")
manager.prioritizer = NewMultiDatabaseLTFPrioritizer(manager.intentsByDiscoveryOrder)
default:
panic("cannot initialize IntentPrioritizer with unknown type")
}
// release these for the garbage collector and to ensure code correctness
manager.intents = nil
manager.intentsByDiscoveryOrder = nil
}
// ValidateSettings returns an error if any settings specified on the command line
// were invalid, or nil if they are valid.
func (exp *MongoExport) ValidateSettings() error {
// Namespace must have a valid database if none is specified,
// use 'test'
if exp.ToolOptions.Namespace.DB == "" {
exp.ToolOptions.Namespace.DB = "test"
}
if exp.ToolOptions.Namespace.Collection == "" {
return fmt.Errorf("must specify a collection")
}
if err := util.ValidateCollectionName(exp.ToolOptions.Namespace.Collection); err != nil {
return err
}
exp.OutputOpts.Type = strings.ToLower(exp.OutputOpts.Type)
if exp.ToolOptions.HiddenOptions.CSVOutputType {
log.Log(log.Always, "csv flag is deprecated; please use --type=csv instead")
exp.OutputOpts.Type = CSV
}
if exp.OutputOpts.Type == "" {
// special error for an empty type value
return fmt.Errorf("--type cannot be empty")
}
if exp.OutputOpts.Type != CSV && exp.OutputOpts.Type != JSON {
return fmt.Errorf("invalid output type '%v', choose 'json' or 'csv'", exp.OutputOpts.Type)
}
if exp.InputOpts.Query != "" && exp.InputOpts.ForceTableScan {
return fmt.Errorf("cannot use --forceTableScan when specifying --query")
}
if exp.InputOpts.Query != "" && exp.InputOpts.QueryFile != "" {
return fmt.Errorf("either --query or --queryFile can be specified as a query option")
}
if exp.InputOpts != nil && exp.InputOpts.HasQuery() {
content, err := exp.InputOpts.GetQuery()
if err != nil {
return err
}
_, err2 := getObjectFromByteArg(content)
if err2 != nil {
return err2
}
}
if exp.InputOpts != nil && exp.InputOpts.Sort != "" {
_, err := getSortFromArg(exp.InputOpts.Sort)
if err != nil {
return err
}
}
return nil
}
// CreateIndexes takes in an intent and an array of index documents and
// attempts to create them using the createIndexes command. If that command
// fails, we fall back to individual index creation.
func (restore *MongoRestore) CreateIndexes(intent *intents.Intent, indexes []IndexDocument) error {
// first, sanitize the indexes
for _, index := range indexes {
// update the namespace of the index before inserting
index.Options["ns"] = intent.Namespace()
// check for length violations before building the command
fullIndexName := fmt.Sprintf("%v.$%v", index.Options["ns"], index.Options["name"])
if len(fullIndexName) > 127 {
return fmt.Errorf(
"cannot restore index with namespace '%v': "+
"namespace is too long (max size is 127 bytes)", fullIndexName)
}
// remove the index version, forcing an update,
// unless we specifically want to keep it
if !restore.OutputOptions.KeepIndexVersion {
delete(index.Options, "v")
}
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
session.SetSafe(&mgo.Safe{})
defer session.Close()
// then attempt the createIndexes command
rawCommand := bson.D{
{"createIndexes", intent.C},
{"indexes", indexes},
}
results := bson.M{}
err = session.DB(intent.DB).Run(rawCommand, &results)
if err == nil {
return nil
}
if err.Error() != "no such cmd: createIndexes" {
return fmt.Errorf("createIndex error: %v", err)
}
// if we're here, the connected server does not support the command, so we fall back
log.Log(log.Info, "\tcreateIndexes command not supported, attemping legacy index insertion")
for _, idx := range indexes {
log.Logf(log.Info, "\tmanually creating index %v", idx.Options["name"])
err = restore.LegacyInsertIndex(intent, idx)
if err != nil {
return fmt.Errorf("error creating index %v: %v", idx.Options["name"], err)
}
}
return nil
}
// GetDumpAuthVersion reads the admin.system.version collection in the dump directory
// to determine the authentication version of the files in the dump. If that collection is not
// present in the dump, we try to infer the authentication version based on its absence.
// Returns the authentication version number and any errors that occur.
func (restore *MongoRestore) GetDumpAuthVersion() (int, error) {
// first handle the case where we have no auth version
intent := restore.manager.AuthVersion()
if intent == nil {
if restore.InputOptions.RestoreDBUsersAndRoles {
// If we are using --restoreDbUsersAndRoles, we cannot guarantee an
// $admin.system.version collection from a 2.6 server,
// so we can assume up to version 3.
log.Logf(log.Always, "no system.version bson file found in '%v' database dump", restore.ToolOptions.DB)
log.Log(log.Always, "warning: assuming users and roles collections are of auth version 3")
log.Log(log.Always, "if users are from an earlier version of MongoDB, they may not restore properly")
return 3, nil
}
log.Log(log.Info, "no system.version bson file found in dump")
log.Log(log.Always, "assuming users in the dump directory are from <= 2.4 (auth version 1)")
return 1, nil
}
err := intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
versionDoc := struct {
CurrentVersion int `bson:"currentVersion"`
}{}
bsonSource.Next(&versionDoc)
if err := bsonSource.Err(); err != nil {
return 0, fmt.Errorf("error reading version bson file %v: %v", intent.BSONPath, err)
}
authVersion := versionDoc.CurrentVersion
if authVersion == 0 {
// 0 is not a possible valid version number, so this can only indicate bad input
return 0, fmt.Errorf("system.version bson file does not have 'currentVersion' field")
}
return authVersion, nil
}
// ValidateAuthVersions compares the authentication version of the dump files and the
// authentication version of the target server, and returns an error if the versions
// are incompatible.
func (restore *MongoRestore) ValidateAuthVersions() error {
if restore.authVersions.Dump == 2 || restore.authVersions.Dump == 4 {
return fmt.Errorf(
"cannot restore users and roles from a dump file with auth version %v; "+
"finish the upgrade or roll it back", restore.authVersions.Dump)
}
if restore.authVersions.Server == 2 || restore.authVersions.Server == 4 {
return fmt.Errorf(
"cannot restore users and roles to a server with auth version %v; "+
"finish the upgrade or roll it back", restore.authVersions.Server)
}
switch restore.authVersions {
case authVersionPair{3, 5}:
log.Log(log.Info,
"restoring users and roles of auth version 3 to a server of auth version 5")
case authVersionPair{5, 5}:
log.Log(log.Info,
"restoring users and roles of auth version 5 to a server of auth version 5")
case authVersionPair{3, 3}:
log.Log(log.Info,
"restoring users and roles of auth version 3 to a server of auth version 3")
case authVersionPair{1, 1}:
log.Log(log.Info,
"restoring users and roles of auth version 1 to a server of auth version 1")
case authVersionPair{1, 5}:
return fmt.Errorf("cannot restore users of auth version 1 to a server of auth version 5")
case authVersionPair{5, 3}:
return fmt.Errorf("cannot restore users of auth version 5 to a server of auth version 3")
case authVersionPair{1, 3}:
log.Log(log.Info,
"restoring users and roles of auth version 1 to a server of auth version 3")
log.Log(log.Always,
"users and roles will have to be updated with the authSchemaUpgrade command")
case authVersionPair{5, 1}:
fallthrough
case authVersionPair{3, 1}:
return fmt.Errorf(
"cannot restore users and roles dump file >= auth version 3 to a server of auth version 1")
default:
return fmt.Errorf("invalid auth pair: dump=%v, server=%v",
restore.authVersions.Dump, restore.authVersions.Server)
}
return nil
}
func main() {
go signals.Handle()
// initialize command-line opts
opts := options.New("bsondump", bsondump.Usage, options.EnabledOptions{})
bsonDumpOpts := &bsondump.BSONDumpOptions{}
opts.AddOptions(bsonDumpOpts)
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'bsondump --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
// pull out the filename
if len(args) == 0 {
log.Logf(log.Always, "must provide a filename")
log.Logf(log.Always, "try 'bsondump --help' for more information")
os.Exit(util.ExitBadOptions)
} else if len(args) > 1 {
log.Logf(log.Always, "too many positional arguments: %v", args)
log.Logf(log.Always, "try 'bsondump --help' for more information")
os.Exit(util.ExitBadOptions)
}
dumper := bsondump.BSONDump{
ToolOptions: opts,
BSONDumpOptions: bsonDumpOpts,
FileName: args[0],
Out: os.Stdout,
}
log.Logf(log.DebugLow, "running bsondump with --objcheck: %v", bsonDumpOpts.ObjCheck)
if len(bsonDumpOpts.Type) != 0 && bsonDumpOpts.Type != "debug" && bsonDumpOpts.Type != "json" {
log.Logf(log.Always, "Unsupported output type '%v'. Must be either 'debug' or 'json'", bsonDumpOpts.Type)
os.Exit(util.ExitBadOptions)
}
err = dumper.Open()
if err != nil {
log.Logf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
var numFound int
if bsonDumpOpts.Type == "debug" {
numFound, err = dumper.Debug()
} else {
numFound, err = dumper.JSON()
}
log.Logf(log.Always, "%v objects found", numFound)
if err != nil {
log.Log(log.Always, err.Error())
os.Exit(util.ExitError)
}
}
// Dump handles some final options checking and executes MongoDump.
func (dump *MongoDump) Dump() (err error) {
if dump.InputOptions.HasQuery() {
// parse JSON then convert extended JSON values
var asJSON interface{}
content, err := dump.InputOptions.GetQuery()
if err != nil {
return err
}
err = json.Unmarshal(content, &asJSON)
if err != nil {
return fmt.Errorf("error parsing query as json: %v", err)
}
convertedJSON, err := bsonutil.ConvertJSONValueToBSON(asJSON)
if err != nil {
return fmt.Errorf("error converting query to bson: %v", err)
}
asMap, ok := convertedJSON.(map[string]interface{})
if !ok {
// unlikely to be reached
return fmt.Errorf("query is not in proper format")
}
dump.query = bson.M(asMap)
}
if dump.OutputOptions.DumpDBUsersAndRoles {
// first make sure this is possible with the connected database
dump.authVersion, err = auth.GetAuthVersion(dump.sessionProvider)
if err != nil {
return fmt.Errorf("error getting auth schema version for dumpDbUsersAndRoles: %v", err)
}
log.Logf(log.DebugLow, "using auth schema version %v", dump.authVersion)
if dump.authVersion < 3 {
return fmt.Errorf("backing up users and roles is only supported for "+
"deployments with auth schema versions >= 3, found: %v", dump.authVersion)
}
}
if dump.OutputOptions.Archive != "" {
//getArchiveOut gives us a WriteCloser to which we should write the archive
var archiveOut io.WriteCloser
archiveOut, err = dump.getArchiveOut()
if err != nil {
return err
}
dump.archive = &archive.Writer{
// The archive.Writer needs its own copy of archiveOut because things
// like the prelude are not written by the multiplexer.
Out: archiveOut,
Mux: archive.NewMultiplexer(archiveOut),
}
go dump.archive.Mux.Run()
defer func() {
// The Mux runs until its Control is closed
close(dump.archive.Mux.Control)
muxErr := <-dump.archive.Mux.Completed
archiveOut.Close()
if muxErr != nil {
if err != nil {
err = fmt.Errorf("%v && %v", err, muxErr)
} else {
err = muxErr
}
log.Logf(log.DebugLow, "mux returned an error: %v", err)
} else {
log.Logf(log.DebugLow, "mux completed successfully")
}
}()
}
// switch on what kind of execution to do
switch {
case dump.ToolOptions.DB == "" && dump.ToolOptions.Collection == "":
err = dump.CreateAllIntents()
case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection == "":
err = dump.CreateIntentsForDatabase(dump.ToolOptions.DB)
case dump.ToolOptions.DB != "" && dump.ToolOptions.Collection != "":
err = dump.CreateCollectionIntent(dump.ToolOptions.DB, dump.ToolOptions.Collection)
}
if err != nil {
return err
}
if dump.OutputOptions.Oplog {
err = dump.CreateOplogIntents()
if err != nil {
return err
}
}
if dump.OutputOptions.DumpDBUsersAndRoles && dump.ToolOptions.DB != "admin" {
err = dump.CreateUsersRolesVersionIntentsForDB(dump.ToolOptions.DB)
if err != nil {
return err
}
}
// verify we can use repair cursors
if dump.OutputOptions.Repair {
log.Log(log.DebugLow, "verifying that the connected server supports repairCursor")
if dump.isMongos {
return fmt.Errorf("cannot use --repair on mongos")
}
exampleIntent := dump.manager.Peek()
if exampleIntent != nil {
supported, err := dump.sessionProvider.SupportsRepairCursor(
exampleIntent.DB, exampleIntent.C)
if !supported {
return err // no extra context needed
}
}
}
// IO Phase I
// metadata, users, roles, and versions
// TODO, either remove this debug or improve the language
log.Logf(log.DebugHigh, "dump phase I: metadata, indexes, users, roles, version")
err = dump.DumpMetadata()
if err != nil {
return fmt.Errorf("error dumping metadata: %v", err)
}
if dump.OutputOptions.Archive != "" {
dump.archive.Prelude, err = archive.NewPrelude(dump.manager, dump.ToolOptions.HiddenOptions.MaxProcs)
if err != nil {
return fmt.Errorf("creating archive prelude: %v", err)
}
err = dump.archive.Prelude.Write(dump.archive.Out)
if err != nil {
return fmt.Errorf("error writing metadata into archive: %v", err)
}
}
err = dump.DumpSystemIndexes()
if err != nil {
return fmt.Errorf("error dumping system indexes: %v", err)
}
if dump.ToolOptions.DB == "admin" || dump.ToolOptions.DB == "" {
err = dump.DumpUsersAndRoles()
if err != nil {
return fmt.Errorf("error dumping users and roles: %v", err)
}
}
if dump.OutputOptions.DumpDBUsersAndRoles {
log.Logf(log.Always, "dumping users and roles for %v", dump.ToolOptions.DB)
if dump.ToolOptions.DB == "admin" {
log.Logf(log.Always, "skipping users/roles dump, already dumped admin database")
} else {
err = dump.DumpUsersAndRolesForDB(dump.ToolOptions.DB)
if err != nil {
return fmt.Errorf("error dumping users and roles for db: %v", err)
}
}
}
// If oplog capturing is enabled, we first check the most recent
// oplog entry and save its timestamp, this will let us later
// copy all oplog entries that occurred while dumping, creating
// what is effectively a point-in-time snapshot.
if dump.OutputOptions.Oplog {
err := dump.determineOplogCollectionName()
if err != nil {
return fmt.Errorf("error finding oplog: %v", err)
}
log.Logf(log.Info, "getting most recent oplog timestamp")
dump.oplogStart, err = dump.getOplogStartTime()
if err != nil {
return fmt.Errorf("error getting oplog start: %v", err)
}
}
// IO Phase II
// regular collections
// TODO, either remove this debug or improve the language
log.Logf(log.DebugHigh, "dump phase II: regular collections")
// kick off the progress bar manager and begin dumping intents
if dump.ProgressManager != nil {
dump.ProgressManager.Start()
defer dump.ProgressManager.Stop()
}
dump.termChan = make(chan struct{})
if dump.HandleSignals != nil {
go dump.HandleSignals(dump)
}
if err := dump.DumpIntents(); err != nil {
return err
}
// IO Phase III
// oplog
// TODO, either remove this debug or improve the language
log.Logf(log.DebugLow, "dump phase III: the oplog")
// If we are capturing the oplog, we dump all oplog entries that occurred
// while dumping the database. Before and after dumping the oplog,
// we check to see if the oplog has rolled over (i.e. the most recent entry when
// we started still exist, so we know we haven't lost data)
if dump.OutputOptions.Oplog {
log.Logf(log.DebugLow, "checking if oplog entry %v still exists", dump.oplogStart)
exists, err := dump.checkOplogTimestampExists(dump.oplogStart)
if !exists {
return fmt.Errorf(
"oplog overflow: mongodump was unable to capture all new oplog entries during execution")
}
if err != nil {
return fmt.Errorf("unable to check oplog for overflow: %v", err)
}
log.Logf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart)
log.Logf(log.Always, "writing captured oplog to %v", dump.manager.Oplog().BSONPath)
err = dump.DumpOplogAfterTimestamp(dump.oplogStart)
if err != nil {
return fmt.Errorf("error dumping oplog: %v", err)
}
// check the oplog for a rollover one last time, to avoid a race condition
// wherein the oplog rolls over in the time after our first check, but before
// we copy it.
log.Logf(log.DebugLow, "checking again if oplog entry %v still exists", dump.oplogStart)
exists, err = dump.checkOplogTimestampExists(dump.oplogStart)
if !exists {
return fmt.Errorf(
"oplog overflow: mongodump was unable to capture all new oplog entries during execution")
}
if err != nil {
return fmt.Errorf("unable to check oplog for overflow: %v", err)
}
log.Logf(log.DebugHigh, "oplog entry %v still exists", dump.oplogStart)
}
log.Logf(log.Info, "done")
return err
}
// CreateAllIntents drills down into a dump folder, creating intents for all of
// the databases and collections it finds.
func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike, filterDB string, filterCollection string) error {
log.Logf(log.DebugHigh, "using %v as dump root directory", dir.Path())
foundOplog := false
entries, err := dir.ReadDir()
if err != nil {
return fmt.Errorf("error reading root dump folder: %v", err)
}
for _, entry := range entries {
if entry.IsDir() {
if err = util.ValidateDBName(entry.Name()); err != nil {
return fmt.Errorf("invalid database name '%v': %v", entry.Name(), err)
}
if filterDB == "" || entry.Name() == filterDB {
err = restore.CreateIntentsForDB(entry.Name(), filterCollection, entry, false)
} else {
err = restore.CreateIntentsForDB(entry.Name(), "", entry, true)
}
if err != nil {
return err
}
} else {
if entry.Name() == "oplog.bson" {
if restore.InputOptions.OplogReplay {
log.Log(log.DebugLow, "found oplog.bson file to replay")
}
foundOplog = true
oplogIntent := &intents.Intent{
C: "oplog",
Size: entry.Size(),
Location: entry.Path(),
}
// filterDB is used to mimic CreateIntentsForDB, and since CreateIntentsForDB wouldn't
// apply the oplog, even when asked, we don't either.
if filterDB != "" || !restore.InputOptions.OplogReplay {
if restore.InputOptions.Archive == "" {
continue
} else {
mutedOut := &archive.MutedCollection{
Intent: oplogIntent,
Demux: restore.archive.Demux,
}
restore.archive.Demux.Open(
oplogIntent.Namespace(),
mutedOut,
)
continue
}
}
if restore.InputOptions.Archive != "" {
if restore.InputOptions.Archive == "-" {
oplogIntent.Location = "archive on stdin"
} else {
oplogIntent.Location = fmt.Sprintf("archive '%v'", restore.InputOptions.Archive)
}
// no need to check that we want to cache here
oplogIntent.BSONFile =
&archive.RegularCollectionReceiver{
Intent: oplogIntent,
Demux: restore.archive.Demux,
}
} else {
oplogIntent.BSONFile = &realBSONFile{path: entry.Path(), intent: oplogIntent, gzip: restore.InputOptions.Gzip}
}
restore.manager.Put(oplogIntent)
} else {
log.Logf(log.Always, `don't know what to do with file "%v", skipping...`, entry.Path())
}
}
}
if restore.InputOptions.OplogReplay && !foundOplog {
return fmt.Errorf("no %v/oplog.bson file to replay; make sure you run mongodump with --oplog", dir.Path())
}
return nil
}
// RestoreOplog attempts to restore a MongoDB oplog.
func (restore *MongoRestore) RestoreOplog() error {
log.Log(log.Always, "replaying oplog")
intent := restore.manager.Oplog()
if intent == nil {
// this should not be reached
log.Log(log.Always, "no oplog.bson file in root of the dump directory, skipping oplog application")
return nil
}
if err := intent.BSONFile.Open(); err != nil {
return err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
entryArray := make([]interface{}, 0, 1024)
rawOplogEntry := &bson.Raw{}
var totalOps int64
var entrySize, bufferedBytes int
oplogProgressor := progress.NewCounter(intent.BSONSize)
bar := progress.Bar{
Name: "oplog",
Watching: oplogProgressor,
WaitTime: 3 * time.Second,
Writer: log.Writer(0),
BarLength: progressBarLength,
IsBytes: true,
}
bar.Start()
defer bar.Stop()
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
// To restore the oplog, we iterate over the oplog entries,
// filling up a buffer. Once the buffer reaches max document size,
// apply the current buffered ops and reset the buffer.
for bsonSource.Next(rawOplogEntry) {
entrySize = len(rawOplogEntry.Data)
if bufferedBytes+entrySize > oplogMaxCommandSize {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
entryArray = make([]interface{}, 0, 1024)
bufferedBytes = 0
}
entryAsOplog := db.Oplog{}
err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog)
if err != nil {
return fmt.Errorf("error reading oplog: %v", err)
}
if entryAsOplog.Operation == "n" {
//skip no-ops
continue
}
if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) {
log.Logf(
log.DebugLow,
"timestamp %v is not below limit of %v; ending oplog restoration",
entryAsOplog.Timestamp,
restore.oplogLimit,
)
break
}
totalOps++
bufferedBytes += entrySize
oplogProgressor.Inc(int64(entrySize))
entryArray = append(entryArray, entryAsOplog)
}
// finally, flush the remaining entries
if len(entryArray) > 0 {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
}
log.Logf(log.Info, "applied %v ops", totalOps)
return nil
}
// RestoreIntent attempts to restore a given intent into MongoDB.
func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error {
collectionExists, err := restore.CollectionExists(intent)
if err != nil {
return fmt.Errorf("error reading database: %v", err)
}
if restore.safety == nil && !restore.OutputOptions.Drop && collectionExists {
log.Logf(log.Always, "restoring to existing collection %v without dropping", intent.Namespace())
log.Log(log.Always, "Important: restored data will be inserted without raising errors; check your server log")
}
if restore.OutputOptions.Drop {
if collectionExists {
if strings.HasPrefix(intent.C, "system.") {
log.Logf(log.Always, "cannot drop system collection %v, skipping", intent.Namespace())
} else {
log.Logf(log.Info, "dropping collection %v before restoring", intent.Namespace())
err = restore.DropCollection(intent)
if err != nil {
return err // no context needed
}
collectionExists = false
}
} else {
log.Logf(log.DebugLow, "collection %v doesn't exist, skipping drop command", intent.Namespace())
}
}
var options bson.D
var indexes []IndexDocument
// get indexes from system.indexes dump if we have it but don't have metadata files
if intent.MetadataPath == "" {
if _, ok := restore.dbCollectionIndexes[intent.DB]; ok {
if indexes, ok = restore.dbCollectionIndexes[intent.DB][intent.C]; ok {
log.Logf(log.Always, "no metadata; falling back to system.indexes")
}
}
}
// first create the collection with options from the metadata file
if intent.MetadataPath != "" {
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
log.Logf(log.Always, "reading metadata for %v from %v", intent.Namespace(), intent.Location)
metadata, err := ioutil.ReadAll(intent.MetadataFile)
if err != nil {
return fmt.Errorf("error reading metadata from %v: %v", intent.Location, err)
}
options, indexes, err = restore.MetadataFromJSON(metadata)
if err != nil {
return fmt.Errorf("error parsing metadata from %v: %v", intent.Location, err)
}
if !restore.OutputOptions.NoOptionsRestore {
if options != nil {
if !collectionExists {
log.Logf(log.Info, "creating collection %v using options from metadata", intent.Namespace())
err = restore.CreateCollection(intent, options)
if err != nil {
return fmt.Errorf("error creating collection %v: %v", intent.Namespace(), err)
}
} else {
log.Logf(log.Info, "collection %v already exists", intent.Namespace())
}
} else {
log.Log(log.Info, "no collection options to restore")
}
} else {
log.Log(log.Info, "skipping options restoration")
}
}
if intent.BSONPath != "" {
err = intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
log.Logf(log.Always, "restoring %v from %v", intent.Namespace(), intent.Location)
var size int64
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
err = restore.RestoreCollectionToDB(intent.DB, intent.C, bsonSource, size)
if err != nil {
return fmt.Errorf("error restoring from %v: %v", intent.BSONPath, err)
}
}
// finally, add indexes
if len(indexes) > 0 && !restore.OutputOptions.NoIndexRestore {
log.Logf(log.Always, "restoring indexes for collection %v from metadata", intent.Namespace())
err = restore.CreateIndexes(intent, indexes)
if err != nil {
return fmt.Errorf("error creating indexes for %v: %v", intent.Namespace(), err)
}
} else {
log.Log(log.Always, "no indexes to restore")
}
log.Logf(log.Always, "finished restoring %v", intent.Namespace())
return nil
}
func main() {
go signals.Handle()
// initialize command-line opts
opts := options.New("bsondump", bsondump.Usage, options.EnabledOptions{})
bsonDumpOpts := &bsondump.BSONDumpOptions{}
opts.AddOptions(bsonDumpOpts)
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'bsondump --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
if len(args) > 1 {
log.Logf(log.Always, "too many positional arguments: %v", args)
log.Logf(log.Always, "try 'bsondump --help' for more information")
os.Exit(util.ExitBadOptions)
}
// If the user specified a bson input file
if len(args) == 1 {
if bsonDumpOpts.BSONFileName != "" {
log.Logf(log.Always, "Cannot specify both a positional argument and --bsonFile")
os.Exit(util.ExitBadOptions)
}
bsonDumpOpts.BSONFileName = args[0]
}
dumper := bsondump.BSONDump{
ToolOptions: opts,
BSONDumpOptions: bsonDumpOpts,
}
reader, err := bsonDumpOpts.GetBSONReader()
if err != nil {
log.Logf(log.Always, "Getting BSON Reader Failed: %v", err)
os.Exit(util.ExitError)
}
dumper.BSONSource = db.NewBSONSource(reader)
defer dumper.BSONSource.Close()
writer, err := bsonDumpOpts.GetWriter()
if err != nil {
log.Logf(log.Always, "Getting Writer Failed: %v", err)
os.Exit(util.ExitError)
}
dumper.Out = writer
defer dumper.Out.Close()
log.Logf(log.DebugLow, "running bsondump with --objcheck: %v", bsonDumpOpts.ObjCheck)
if len(bsonDumpOpts.Type) != 0 && bsonDumpOpts.Type != "debug" && bsonDumpOpts.Type != "json" {
log.Logf(log.Always, "Unsupported output type '%v'. Must be either 'debug' or 'json'", bsonDumpOpts.Type)
os.Exit(util.ExitBadOptions)
}
var numFound int
if bsonDumpOpts.Type == "debug" {
numFound, err = dumper.Debug()
} else {
numFound, err = dumper.JSON()
}
log.Logf(log.Always, "%v objects found", numFound)
if err != nil {
log.Log(log.Always, err.Error())
os.Exit(util.ExitError)
}
}
// Restore runs the mongorestore program.
func (restore *MongoRestore) Restore() error {
var target archive.DirLike
err := restore.ParseAndValidateOptions()
if err != nil {
log.Logf(log.DebugLow, "got error from options parsing: %v", err)
return err
}
// Build up all intents to be restored
restore.manager = intents.NewIntentManager()
if restore.InputOptions.Archive != "" {
archiveReader, err := restore.getArchiveReader()
if err != nil {
return err
}
restore.archive = &archive.Reader{
In: archiveReader,
Prelude: &archive.Prelude{},
}
err = restore.archive.Prelude.Read(restore.archive.In)
if err != nil {
return err
}
target, err = restore.archive.Prelude.NewPreludeExplorer()
if err != nil {
return err
}
} else {
if restore.TargetDirectory == "" {
restore.TargetDirectory = "dump"
log.Log(log.Always, "using default 'dump' directory")
}
target, err = newActualPath(restore.TargetDirectory)
if err != nil {
return err
}
// handle cases where the user passes in a file instead of a directory
if !target.IsDir() {
log.Log(log.DebugLow, "mongorestore target is a file, not a directory")
err = restore.handleBSONInsteadOfDirectory(restore.TargetDirectory)
if err != nil {
return err
}
} else {
log.Log(log.DebugLow, "mongorestore target is a directory, not a file")
}
}
if restore.ToolOptions.Collection != "" &&
restore.OutputOptions.NumParallelCollections > 1 &&
restore.OutputOptions.NumInsertionWorkers == 1 {
// handle special parallelization case when we are only restoring one collection
// by mapping -j to insertion workers rather than parallel collections
log.Logf(log.DebugHigh,
"setting number of insertions workers to number of parallel collections (%v)",
restore.OutputOptions.NumParallelCollections)
restore.OutputOptions.NumInsertionWorkers = restore.OutputOptions.NumParallelCollections
}
if restore.InputOptions.Archive != "" {
if int(restore.archive.Prelude.Header.ConcurrentCollections) > restore.OutputOptions.NumParallelCollections {
restore.OutputOptions.NumParallelCollections = int(restore.archive.Prelude.Header.ConcurrentCollections)
restore.OutputOptions.NumInsertionWorkers = int(restore.archive.Prelude.Header.ConcurrentCollections)
log.Logf(log.Always,
"setting number of parallel collections to number of parallel collections in archive (%v)",
restore.archive.Prelude.Header.ConcurrentCollections,
)
}
}
// Create the demux before intent creation, because muted archive intents need
// to register themselves with the demux directly
if restore.InputOptions.Archive != "" {
restore.archive.Demux = &archive.Demultiplexer{
In: restore.archive.In,
}
}
switch {
case restore.InputOptions.Archive != "":
log.Logf(log.Always,
"creating intents for archive")
err = restore.CreateAllIntents(target, restore.ToolOptions.DB, restore.ToolOptions.Collection)
case restore.ToolOptions.DB == "" && restore.ToolOptions.Collection == "":
log.Logf(log.Always,
"building a list of dbs and collections to restore from %v dir",
target.Path())
err = restore.CreateAllIntents(target, "", "")
case restore.ToolOptions.DB != "" && restore.ToolOptions.Collection == "":
log.Logf(log.Always,
"building a list of collections to restore from %v dir",
target.Path())
err = restore.CreateIntentsForDB(
restore.ToolOptions.DB,
"",
target,
false,
)
case restore.ToolOptions.DB != "" && restore.ToolOptions.Collection != "":
log.Logf(log.Always, "checking for collection data in %v", target.Path())
err = restore.CreateIntentForCollection(
restore.ToolOptions.DB,
restore.ToolOptions.Collection,
target,
)
}
if err != nil {
return fmt.Errorf("error scanning filesystem: %v", err)
}
if restore.isMongos && restore.manager.HasConfigDBIntent() && restore.ToolOptions.DB == "" {
return fmt.Errorf("cannot do a full restore on a sharded system - " +
"remove the 'config' directory from the dump directory first")
}
if restore.InputOptions.Archive != "" {
namespaceChan := make(chan string, 1)
namespaceErrorChan := make(chan error)
restore.archive.Demux.NamespaceChan = namespaceChan
restore.archive.Demux.NamespaceErrorChan = namespaceErrorChan
go restore.archive.Demux.Run()
// consume the new namespace announcement from the demux for all of the special collections
// that get cached when being read out of the archive.
// The first regular collection found gets pushed back on to the namespaceChan
// consume the new namespace announcement from the demux for all of the collections that get cached
for {
ns, ok := <-namespaceChan
// the archive can have only special collections. In that case we keep reading until
// the namespaces are exhausted, indicated by the namespaceChan being closed.
if !ok {
break
}
intent := restore.manager.IntentForNamespace(ns)
if intent == nil {
return fmt.Errorf("no intent for collection in archive: %v", ns)
}
if intent.IsSystemIndexes() ||
intent.IsUsers() ||
intent.IsRoles() ||
intent.IsAuthVersion() {
log.Logf(log.DebugLow, "special collection %v found", ns)
namespaceErrorChan <- nil
} else {
// Put the ns back on the announcement chan so that the
// demultiplexer can start correctly
log.Logf(log.DebugLow, "first non special collection %v found."+
" The demultiplexer will handle it and the remainder", ns)
namespaceChan <- ns
break
}
}
}
// If restoring users and roles, make sure we validate auth versions
if restore.ShouldRestoreUsersAndRoles() {
log.Log(log.Info, "comparing auth version of the dump directory and target server")
restore.authVersions.Dump, err = restore.GetDumpAuthVersion()
if err != nil {
return fmt.Errorf("error getting auth version from dump: %v", err)
}
restore.authVersions.Server, err = auth.GetAuthVersion(restore.SessionProvider)
if err != nil {
return fmt.Errorf("error getting auth version of server: %v", err)
}
err = restore.ValidateAuthVersions()
if err != nil {
return fmt.Errorf(
"the users and roles collections in the dump have an incompatible auth version with target server: %v",
err)
}
}
err = restore.LoadIndexesFromBSON()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
// Restore the regular collections
if restore.InputOptions.Archive != "" {
restore.manager.UsePrioritizer(restore.archive.Demux.NewPrioritizer(restore.manager))
} else if restore.OutputOptions.NumParallelCollections > 1 {
restore.manager.Finalize(intents.MultiDatabaseLTF)
} else {
// use legacy restoration order if we are single-threaded
restore.manager.Finalize(intents.Legacy)
}
restore.termChan = make(chan struct{})
go restore.handleSignals()
if err := restore.RestoreIntents(); err != nil {
return err
}
// Restore users/roles
if restore.ShouldRestoreUsersAndRoles() {
if restore.manager.Users() != nil {
err = restore.RestoreUsersOrRoles(Users, restore.manager.Users())
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
if restore.manager.Roles() != nil {
err = restore.RestoreUsersOrRoles(Roles, restore.manager.Roles())
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
}
// Restore oplog
if restore.InputOptions.OplogReplay {
err = restore.RestoreOplog()
if err != nil {
return fmt.Errorf("restore error: %v", err)
}
}
log.Log(log.Always, "done")
return nil
}
// ParseAndValidateOptions returns a non-nil error if user-supplied options are invalid.
func (restore *MongoRestore) ParseAndValidateOptions() error {
// Can't use option pkg defaults for --objcheck because it's two separate flags,
// and we need to be able to see if they're both being used. We default to
// true here and then see if noobjcheck is enabled.
log.Log(log.DebugHigh, "checking options")
if restore.InputOptions.Objcheck {
restore.objCheck = true
log.Log(log.DebugHigh, "\tdumping with object check enabled")
} else {
log.Log(log.DebugHigh, "\tdumping with object check disabled")
}
if restore.ToolOptions.DB == "" && restore.ToolOptions.Collection != "" {
return fmt.Errorf("cannot dump a collection without a specified database")
}
if restore.ToolOptions.DB != "" {
if err := util.ValidateDBName(restore.ToolOptions.DB); err != nil {
return fmt.Errorf("invalid db name: %v", err)
}
}
if restore.ToolOptions.Collection != "" {
if err := util.ValidateCollectionGrammar(restore.ToolOptions.Collection); err != nil {
return fmt.Errorf("invalid collection name: %v", err)
}
}
if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "" {
return fmt.Errorf("cannot use --restoreDbUsersAndRoles without a specified database")
}
if restore.InputOptions.RestoreDBUsersAndRoles && restore.ToolOptions.DB == "admin" {
return fmt.Errorf("cannot use --restoreDbUsersAndRoles with the admin database")
}
var err error
restore.isMongos, err = restore.SessionProvider.IsMongos()
if err != nil {
return err
}
if restore.isMongos {
log.Log(log.DebugLow, "restoring to a sharded system")
}
if restore.InputOptions.OplogLimit != "" {
if !restore.InputOptions.OplogReplay {
return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled")
}
restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit)
if err != nil {
return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err)
}
}
// check if we are using a replica set and fall back to w=1 if we aren't (for <= 2.4)
nodeType, err := restore.SessionProvider.GetNodeType()
if err != nil {
return fmt.Errorf("error determining type of connected node: %v", err)
}
log.Logf(log.DebugLow, "connected to node type: %v", nodeType)
restore.safety, err = db.BuildWriteConcern(restore.OutputOptions.WriteConcern, nodeType)
if err != nil {
return fmt.Errorf("error parsing write concern: %v", err)
}
// handle the hidden auth collection flags
if restore.ToolOptions.HiddenOptions.TempUsersColl == nil {
restore.tempUsersCol = "tempusers"
} else {
restore.tempUsersCol = *restore.ToolOptions.HiddenOptions.TempUsersColl
}
if restore.ToolOptions.HiddenOptions.TempRolesColl == nil {
restore.tempRolesCol = "temproles"
} else {
restore.tempRolesCol = *restore.ToolOptions.HiddenOptions.TempRolesColl
}
if restore.OutputOptions.NumInsertionWorkers < 0 {
return fmt.Errorf(
"cannot specify a negative number of insertion workers per collection")
}
// a single dash signals reading from stdin
if restore.TargetDirectory == "-" {
restore.useStdin = true
if restore.InputOptions.Archive != "" {
return fmt.Errorf(
"cannot restore from \"-\" when --archive is specified")
}
if restore.ToolOptions.Collection == "" {
return fmt.Errorf("cannot restore from stdin without a specified collection")
}
}
return nil
}
// Run executes the mongooplog program.
func (mo *MongoOplog) Run() error {
// split up the oplog namespace we are using
oplogDB, oplogColl, err :=
util.SplitAndValidateNamespace(mo.SourceOptions.OplogNS)
if err != nil {
return err
}
// the full oplog namespace needs to be specified
if oplogColl == "" {
return fmt.Errorf("the oplog namespace must specify a collection")
}
log.Logf(log.DebugLow, "using oplog namespace `%v.%v`", oplogDB, oplogColl)
// connect to the destination server
toSession, err := mo.SessionProviderTo.GetSession()
if err != nil {
return fmt.Errorf("error connecting to destination db: %v", err)
}
defer toSession.Close()
toSession.SetSocketTimeout(0)
// purely for logging
destServerStr := mo.ToolOptions.Host
if mo.ToolOptions.Port != "" {
destServerStr = destServerStr + ":" + mo.ToolOptions.Port
}
log.Logf(log.DebugLow, "successfully connected to destination server `%v`", destServerStr)
// connect to the source server
fromSession, err := mo.SessionProviderFrom.GetSession()
if err != nil {
return fmt.Errorf("error connecting to source db: %v", err)
}
defer fromSession.Close()
fromSession.SetSocketTimeout(0)
log.Logf(log.DebugLow, "successfully connected to source server `%v`", mo.SourceOptions.From)
// set slave ok
fromSession.SetMode(mgo.Eventual, true)
// get the tailing cursor for the source server's oplog
tail := buildTailingCursor(fromSession.DB(oplogDB).C(oplogColl),
mo.SourceOptions)
defer tail.Close()
// read the cursor dry, applying ops to the destination
// server in the process
oplogEntry := &db.Oplog{}
res := &db.ApplyOpsResponse{}
log.Log(log.DebugLow, "applying oplog entries...")
for tail.Next(oplogEntry) {
// skip noops
if oplogEntry.Operation == "n" {
log.Logf(log.DebugHigh, "skipping no-op for namespace `%v`", oplogEntry.Namespace)
continue
}
// prepare the op to be applied
opsToApply := []db.Oplog{*oplogEntry}
// apply the operation
err := toSession.Run(bson.M{"applyOps": opsToApply}, res)
if err != nil {
return fmt.Errorf("error applying ops: %v", err)
}
// check the server's response for an issue
if !res.Ok {
return fmt.Errorf("server gave error applying ops: %v", res.ErrMsg)
}
}
// make sure there was no tailing error
if err := tail.Err(); err != nil {
return fmt.Errorf("error querying oplog: %v", err)
}
log.Log(log.DebugLow, "done applying oplog entries")
return nil
}