// LoadIndexesFromBSON reads indexes from the index BSON files and
// caches them in the MongoRestore object.
func (restore *MongoRestore) LoadIndexesFromBSON() error {
dbCollectionIndexes := make(map[string]collectionIndexes)
for _, dbname := range restore.manager.SystemIndexDBs() {
dbCollectionIndexes[dbname] = make(collectionIndexes)
intent := restore.manager.SystemIndexes(dbname)
err := intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
// iterate over stored indexes, saving all that match the collection
indexDocument := &IndexDocument{}
for bsonSource.Next(indexDocument) {
namespace := indexDocument.Options["ns"].(string)
dbCollectionIndexes[dbname][stripDBFromNS(namespace)] =
append(dbCollectionIndexes[dbname][stripDBFromNS(namespace)], *indexDocument)
}
if err := bsonSource.Err(); err != nil {
return fmt.Errorf("error scanning system.indexes: %v", err)
}
}
restore.dbCollectionIndexes = dbCollectionIndexes
return nil
}
// JSON iterates through the BSON file and for each document it finds,
// recursively descends into objects and arrays and prints the human readable
// JSON representation.
// It returns the number of documents processed and a non-nil error if one is
// encountered before the end of the file is reached.
func (bd *BSONDump) JSON() (int, error) {
numFound := 0
if bd.BSONSource == nil {
panic("Tried to call JSON() before opening file")
}
decodedStream := db.NewDecodedBSONSource(bd.BSONSource)
var result bson.Raw
for decodedStream.Next(&result) {
if err := printJSON(&result, bd.Out, bd.BSONDumpOptions.Pretty); err != nil {
log.Logvf(log.Always, "unable to dump document %v: %v", numFound+1, err)
//if objcheck is turned on, stop now. otherwise keep on dumpin'
if bd.BSONDumpOptions.ObjCheck {
return numFound, err
}
} else {
_, err := bd.Out.Write([]byte("\n"))
if err != nil {
return numFound, err
}
}
numFound++
}
if err := decodedStream.Err(); err != nil {
return numFound, err
}
return numFound, nil
}
// read all the database bson documents from dir and put it into another DB
// ignore the inddexes for now
func readBSONIntoDatabase(dir, restoreDBName string) error {
if ok := fileDirExists(dir); !ok {
return fmt.Errorf("error finding '%v' on local FS", dir)
}
session, err := getBareSession()
if err != nil {
return err
}
defer session.Close()
fileInfos, err := ioutil.ReadDir(dir)
if err != nil {
return err
}
for _, fileInfo := range fileInfos {
fileName := fileInfo.Name()
if !strings.HasSuffix(fileName, ".bson") || fileName == "system.indexes.bson" {
continue
}
collectionName := fileName[:strings.LastIndex(fileName, ".bson")]
collection := session.DB(restoreDBName).C(collectionName)
file, err := os.Open(fmt.Sprintf("%s/%s", dir, fileName))
if err != nil {
return err
}
defer file.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(file))
defer bsonSource.Close()
var result bson.M
for bsonSource.Next(&result) {
err = collection.Insert(result)
if err != nil {
return err
}
}
if err = bsonSource.Err(); err != nil {
return err
}
}
return nil
}
func NewByLineOpsReader(reader io.ReadCloser, logger *Logger, opFilter string) (error, *ByLineOpsReader) {
opFilters := make([]OpType, 0)
if opFilter != "" {
filterList := strings.Split(opFilter, ",")
for _, filter := range filterList {
opFilters = append(opFilters, OpType(filter))
}
}
return nil, &ByLineOpsReader{
src: db.NewDecodedBSONSource(db.NewBSONSource(reader)),
err: nil,
opsRead: 0,
logger: logger,
opFilters: opFilters,
}
}
// GetDumpAuthVersion reads the admin.system.version collection in the dump directory
// to determine the authentication version of the files in the dump. If that collection is not
// present in the dump, we try to infer the authentication version based on its absence.
// Returns the authentication version number and any errors that occur.
func (restore *MongoRestore) GetDumpAuthVersion() (int, error) {
// first handle the case where we have no auth version
intent := restore.manager.AuthVersion()
if intent == nil {
if restore.InputOptions.RestoreDBUsersAndRoles {
// If we are using --restoreDbUsersAndRoles, we cannot guarantee an
// $admin.system.version collection from a 2.6 server,
// so we can assume up to version 3.
log.Logvf(log.Always, "no system.version bson file found in '%v' database dump", restore.NSOptions.DB)
log.Logv(log.Always, "warning: assuming users and roles collections are of auth version 3")
log.Logv(log.Always, "if users are from an earlier version of MongoDB, they may not restore properly")
return 3, nil
}
log.Logv(log.Info, "no system.version bson file found in dump")
log.Logv(log.Always, "assuming users in the dump directory are from <= 2.4 (auth version 1)")
return 1, nil
}
err := intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
versionDoc := bson.M{}
for bsonSource.Next(&versionDoc) {
id, ok := versionDoc["_id"].(string)
if ok && id == "authSchema" {
authVersion, ok := versionDoc["currentVersion"].(int)
if ok {
return authVersion, nil
}
return 0, fmt.Errorf("can't unmarshal system.version curentVersion as an int")
}
log.Logvf(log.DebugLow, "system.version document is not an authSchema %v", versionDoc["_id"])
}
err = bsonSource.Err()
if err != nil {
log.Logvf(log.Info, "can't unmarshal system.version document: %v", err)
}
return 0, fmt.Errorf("system.version bson file does not have authSchema document")
}
// GetDumpAuthVersion reads the admin.system.version collection in the dump directory
// to determine the authentication version of the files in the dump. If that collection is not
// present in the dump, we try to infer the authentication version based on its absence.
// Returns the authentication version number and any errors that occur.
func (restore *MongoRestore) GetDumpAuthVersion() (int, error) {
// first handle the case where we have no auth version
intent := restore.manager.AuthVersion()
if intent == nil {
if restore.InputOptions.RestoreDBUsersAndRoles {
// If we are using --restoreDbUsersAndRoles, we cannot guarantee an
// $admin.system.version collection from a 2.6 server,
// so we can assume up to version 3.
log.Logf(log.Always, "no system.version bson file found in '%v' database dump", restore.ToolOptions.DB)
log.Log(log.Always, "warning: assuming users and roles collections are of auth version 3")
log.Log(log.Always, "if users are from an earlier version of MongoDB, they may not restore properly")
return 3, nil
}
log.Log(log.Info, "no system.version bson file found in dump")
log.Log(log.Always, "assuming users in the dump directory are from <= 2.4 (auth version 1)")
return 1, nil
}
err := intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
versionDoc := struct {
CurrentVersion int `bson:"currentVersion"`
}{}
bsonSource.Next(&versionDoc)
if err := bsonSource.Err(); err != nil {
return 0, fmt.Errorf("error reading version bson file %v: %v", intent.BSONPath, err)
}
authVersion := versionDoc.CurrentVersion
if authVersion == 0 {
// 0 is not a possible valid version number, so this can only indicate bad input
return 0, fmt.Errorf("system.version bson file does not have 'currentVersion' field")
}
return authVersion, nil
}
func (bd *BSONDump) Dump() error {
stream, err := bd.init()
if err != nil {
return err
}
decodedStream := db.NewDecodedBSONSource(stream)
defer decodedStream.Close()
var result bson.M
for decodedStream.Next(&result) {
if err := dumpDoc(&result, bd.Out); err != nil {
return err
}
_, err := bd.Out.Write([]byte("\n"))
if err != nil {
return err
}
}
if err := decodedStream.Err(); err != nil {
return err
}
return nil
}
func TestMongoDumpBSON(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
log.SetWriter(ioutil.Discard)
Convey("With a MongoDump instance", t, func() {
err := setUpMongoDumpTestData()
So(err, ShouldBeNil)
Convey("testing that using MongoDump WITHOUT giving a query dumps everything in the database and/or collection", func() {
md := simpleMongoDumpInstance()
md.InputOptions.Query = ""
Convey("and that for a particular collection", func() {
md.ToolOptions.Namespace.Collection = testCollectionNames[0]
err = md.Init()
So(err, ShouldBeNil)
Convey("it dumps to the default output directory", func() {
// we don't have to set this manually if parsing options via command line
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
err = readBSONIntoDatabase(dumpDBDir, testRestoreDB)
So(err, ShouldBeNil)
session, err := getBareSession()
So(err, ShouldBeNil)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
collOriginal := session.DB(testDB).C(testCollectionNames[0])
collRestore := session.DB(testRestoreDB).C(testCollectionNames[0])
Convey("with the correct number of documents", func() {
numDocsOrig, err := collOriginal.Count()
So(err, ShouldBeNil)
numDocsRestore, err := collRestore.Count()
So(err, ShouldBeNil)
So(numDocsOrig, ShouldEqual, numDocsRestore)
})
Convey("that are the same as the documents in the test database", func() {
iter := collOriginal.Find(nil).Iter()
var result bson.M
for iter.Next(&result) {
restoredCount, err := collRestore.Find(result).Count()
So(err, ShouldBeNil)
So(restoredCount, ShouldNotEqual, 0)
}
So(iter.Close(), ShouldBeNil)
})
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to a user-specified output directory", func() {
md.OutputOptions.Out = "dump_user"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump_user"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, 1)
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("it dumps to standard output", func() {
md.OutputOptions.Out = "-"
stdoutBuf := &bytes.Buffer{}
md.stdout = stdoutBuf
err = md.Dump()
So(err, ShouldBeNil)
var count int
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(ioutil.NopCloser(stdoutBuf)))
defer bsonSource.Close()
var result bson.Raw
for bsonSource.Next(&result) {
count++
}
So(bsonSource.Err(), ShouldBeNil)
So(count, ShouldEqual, 10) //The 0th collection has 10 documents
Reset(func() {
})
})
})
Convey("for an entire database", func() {
md.ToolOptions.Namespace.Collection = ""
err = md.Init()
So(err, ShouldBeNil)
Convey("that exists. The dumped directory should contain the necessary bson files", func() {
md.OutputOptions.Out = "dump"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, testDB))
So(fileDirExists(dumpDir), ShouldBeTrue)
So(fileDirExists(dumpDBDir), ShouldBeTrue)
countColls, err := countNonIndexBSONFiles(dumpDBDir)
So(err, ShouldBeNil)
So(countColls, ShouldEqual, len(testCollectionNames))
Reset(func() {
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("that does not exist. The dumped directory shouldn't be created", func() {
md.OutputOptions.Out = "dump"
md.ToolOptions.Namespace.DB = "nottestdb"
err = md.Dump()
So(err, ShouldBeNil)
path, err := os.Getwd()
So(err, ShouldBeNil)
dumpDir := util.ToUniversalPath(filepath.Join(path, "dump"))
dumpDBDir := util.ToUniversalPath(filepath.Join(dumpDir, "nottestdb"))
So(fileDirExists(dumpDir), ShouldBeFalse)
So(fileDirExists(dumpDBDir), ShouldBeFalse)
})
})
})
Convey("testing that using MongoDump WITH a query dumps a subset of documents in a database and/or collection", func() {
session, err := getBareSession()
So(err, ShouldBeNil)
md := simpleMongoDumpInstance()
// expect 10 documents per collection
bsonQuery := bson.M{"age": bson.M{"$lt": 10}}
jsonQuery, err := bsonutil.ConvertBSONValueToJSON(bsonQuery)
So(err, ShouldBeNil)
jsonQueryBytes, err := json.Marshal(jsonQuery)
So(err, ShouldBeNil)
Convey("using --query for all the collections in the database", func() {
md.InputOptions.Query = string(jsonQueryBytes)
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
})
})
Convey("using --queryFile for all the collections in the database", func() {
ioutil.WriteFile("example.json", jsonQueryBytes, 0777)
md.InputOptions.QueryFile = "example.json"
md.ToolOptions.Namespace.DB = testDB
md.OutputOptions.Out = "dump"
dumpDir := testQuery(md, session)
Reset(func() {
So(session.DB(testRestoreDB).DropDatabase(), ShouldBeNil)
So(os.RemoveAll(dumpDir), ShouldBeNil)
So(os.Remove("example.json"), ShouldBeNil)
})
})
})
Reset(func() {
So(tearDownMongoDumpTestData(), ShouldBeNil)
})
})
}
// RestoreIntent attempts to restore a given intent into MongoDB.
func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error {
collectionExists, err := restore.CollectionExists(intent)
if err != nil {
return fmt.Errorf("error reading database: %v", err)
}
if restore.safety == nil && !restore.OutputOptions.Drop && collectionExists {
log.Logvf(log.Always, "restoring to existing collection %v without dropping", intent.Namespace())
log.Logv(log.Always, "Important: restored data will be inserted without raising errors; check your server log")
}
if restore.OutputOptions.Drop {
if collectionExists {
if strings.HasPrefix(intent.C, "system.") {
log.Logvf(log.Always, "cannot drop system collection %v, skipping", intent.Namespace())
} else {
log.Logvf(log.Info, "dropping collection %v before restoring", intent.Namespace())
err = restore.DropCollection(intent)
if err != nil {
return err // no context needed
}
collectionExists = false
}
} else {
log.Logvf(log.DebugLow, "collection %v doesn't exist, skipping drop command", intent.Namespace())
}
}
var options bson.D
var indexes []IndexDocument
// get indexes from system.indexes dump if we have it but don't have metadata files
if intent.MetadataFile == nil {
if _, ok := restore.dbCollectionIndexes[intent.DB]; ok {
if indexes, ok = restore.dbCollectionIndexes[intent.DB][intent.C]; ok {
log.Logvf(log.Always, "no metadata; falling back to system.indexes")
}
}
}
logMessageSuffix := "with no metadata"
// first create the collection with options from the metadata file
if intent.MetadataFile != nil {
logMessageSuffix = "using options from metadata"
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
log.Logvf(log.Always, "reading metadata for %v from %v", intent.Namespace(), intent.MetadataLocation)
metadata, err := ioutil.ReadAll(intent.MetadataFile)
if err != nil {
return fmt.Errorf("error reading metadata from %v: %v", intent.MetadataLocation, err)
}
options, indexes, err = restore.MetadataFromJSON(metadata)
if err != nil {
return fmt.Errorf("error parsing metadata from %v: %v", intent.MetadataLocation, err)
}
if restore.OutputOptions.NoOptionsRestore {
log.Logv(log.Info, "not restoring collection options")
logMessageSuffix = "with no collection options"
options = nil
}
}
if !collectionExists {
log.Logvf(log.Info, "creating collection %v %s", intent.Namespace(), logMessageSuffix)
log.Logvf(log.DebugHigh, "using collection options: %#v", options)
err = restore.CreateCollection(intent, options)
if err != nil {
return fmt.Errorf("error creating collection %v: %v", intent.Namespace(), err)
}
} else {
log.Logvf(log.Info, "collection %v already exists - skipping collection create", intent.Namespace())
}
var documentCount int64
if intent.BSONFile != nil {
err = intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
log.Logvf(log.Always, "restoring %v from %v", intent.Namespace(), intent.Location)
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
documentCount, err = restore.RestoreCollectionToDB(intent.DB, intent.C, bsonSource, intent.BSONFile, intent.Size)
if err != nil {
return fmt.Errorf("error restoring from %v: %v", intent.Location, err)
}
}
// finally, add indexes
if len(indexes) > 0 && !restore.OutputOptions.NoIndexRestore {
log.Logvf(log.Always, "restoring indexes for collection %v from metadata", intent.Namespace())
err = restore.CreateIndexes(intent, indexes)
if err != nil {
return fmt.Errorf("error creating indexes for %v: %v", intent.Namespace(), err)
}
} else {
log.Logv(log.Always, "no indexes to restore")
}
log.Logvf(log.Always, "finished restoring %v (%v %v)",
intent.Namespace(), documentCount, util.Pluralize(int(documentCount), "document", "documents"))
return nil
}
func getDocSource(exp MongoExport) (db.DocSource, error) {
if exp.ToolOptions.Namespace.DBPath != "" {
shimPath, err := db.LocateShim()
if err != nil {
return nil, err
}
bsonTool := db.StorageShim{
DBPath: exp.ToolOptions.Namespace.DBPath,
Database: exp.ToolOptions.Namespace.DB,
Collection: exp.ToolOptions.Namespace.Collection,
Query: exp.InputOpts.Query,
Skip: exp.InputOpts.Skip,
Limit: exp.InputOpts.Limit,
ShimPath: shimPath,
}
iter, _, err := bsonTool.Open()
if err != nil {
return nil, err
}
return db.NewDecodedBSONSource(iter), nil
}
sessionProvider, err := db.InitSessionProvider(exp.ToolOptions)
if err != nil {
return nil, err
}
session, err := sessionProvider.GetSession()
if err != nil {
return nil, err
}
collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)
query := map[string]interface{}{}
if exp.InputOpts != nil && exp.InputOpts.Query != "" {
var err error
query, err = getObjectFromArg(exp.InputOpts.Query)
if err != nil {
return nil, err
}
}
q := collection.Find(query)
if exp.InputOpts != nil && exp.InputOpts.Skip > 0 {
q = q.Skip(exp.InputOpts.Skip)
}
if exp.InputOpts != nil && exp.InputOpts.Limit > 0 {
q = q.Limit(exp.InputOpts.Limit)
}
if exp.InputOpts != nil && exp.InputOpts.Sort != "" {
sortD, err := getSortFromArg(exp.InputOpts.Sort)
if err != nil {
return nil, err
}
sortFields, err := bsonutil.MakeSortString(sortD)
if err != nil {
return nil, err
}
q = q.Sort(sortFields...)
}
if len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.ForceTableScan != true && exp.InputOpts.Sort == "" {
q = q.Snapshot()
}
cursor := q.Iter()
return &db.CursorDocSource{cursor, session}, nil
}
// RestoreUsersOrRoles accepts a users intent and a roles intent, and restores
// them via _mergeAuthzCollections. Either or both can be nil. In the latter case
// nothing is done.
func (restore *MongoRestore) RestoreUsersOrRoles(users, roles *intents.Intent) error {
type loopArg struct {
intent *intents.Intent
intentType string
mergeParamName string
tempCollectionName string
}
if users == nil && roles == nil {
return nil
}
if users != nil && roles != nil && users.DB != roles.DB {
return fmt.Errorf("can't restore users and roles to different databases, %v and %v", users.DB, roles.DB)
}
args := []loopArg{}
mergeArgs := bson.D{}
userTargetDB := ""
if users != nil {
args = append(args, loopArg{users, "users", "tempUsersCollection", restore.tempUsersCol})
}
if roles != nil {
args = append(args, loopArg{roles, "roles", "tempRolesCollection", restore.tempRolesCol})
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
// For each of the users and roles intents:
// build up the mergeArgs component of the _mergeAuthzCollections command
// upload the BSONFile to a temporary collection
for _, arg := range args {
if arg.intent.Size == 0 {
// MongoDB complains if we try and remove a non-existent collection, so we should
// just skip auth collections with empty .bson files to avoid gnarly logic later on.
log.Logf(log.Always, "%v file '%v' is empty; skipping %v restoration", arg.intentType, arg.intent.BSONPath, arg.intentType)
}
log.Logf(log.Always, "restoring %v from %v", arg.intentType, arg.intent.BSONPath)
mergeArgs = append(mergeArgs, bson.DocElem{arg.mergeParamName, "admin." + arg.tempCollectionName})
err := arg.intent.BSONFile.Open()
if err != nil {
return err
}
defer arg.intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(arg.intent.BSONFile))
defer bsonSource.Close()
tempCollectionNameExists, err := restore.CollectionExists(&intents.Intent{DB: "admin", C: arg.tempCollectionName})
if err != nil {
return err
}
if tempCollectionNameExists {
log.Logf(log.Info, "dropping preexisting temporary collection admin.%v", arg.tempCollectionName)
err = session.DB("admin").C(arg.tempCollectionName).DropCollection()
if err != nil {
return fmt.Errorf("error dropping preexisting temporary collection %v: %v", arg.tempCollectionName, err)
}
}
log.Logf(log.DebugLow, "restoring %v to temporary collection", arg.intentType)
if _, err = restore.RestoreCollectionToDB("admin", arg.tempCollectionName, bsonSource, 0); err != nil {
return fmt.Errorf("error restoring %v: %v", arg.intentType, err)
}
// make sure we always drop the temporary collection
defer func() {
session, e := restore.SessionProvider.GetSession()
if e != nil {
// logging errors here because this has no way of returning that doesn't mask other errors
log.Logf(log.Info, "error establishing connection to drop temporary collection admin.%v: %v", arg.tempCollectionName, e)
return
}
defer session.Close()
log.Logf(log.DebugHigh, "dropping temporary collection admin.%v", arg.tempCollectionName)
e = session.DB("admin").C(arg.tempCollectionName).DropCollection()
if e != nil {
log.Logf(log.Info, "error dropping temporary collection admin.%v: %v", arg.tempCollectionName, e)
}
}()
userTargetDB = arg.intent.DB
}
if userTargetDB == "admin" {
// _mergeAuthzCollections uses an empty db string as a sentinel for "all databases"
userTargetDB = ""
}
// we have to manually convert mgo's safety to a writeconcern object
writeConcern := bson.M{}
if restore.safety == nil {
writeConcern["w"] = 0
} else {
if restore.safety.WMode != "" {
writeConcern["w"] = restore.safety.WMode
} else {
writeConcern["w"] = restore.safety.W
}
}
command := bsonutil.MarshalD{}
command = append(command,
bson.DocElem{"_mergeAuthzCollections", 1})
command = append(command,
mergeArgs...)
command = append(command,
bson.DocElem{"drop", restore.OutputOptions.Drop},
bson.DocElem{"writeConcern", writeConcern},
bson.DocElem{"db", userTargetDB})
log.Logf(log.DebugLow, "merging users/roles from temp collections")
res := bson.M{}
err = session.Run(command, &res)
if err != nil {
return fmt.Errorf("error running merge command: %v", err)
}
if util.IsFalsy(res["ok"]) {
return fmt.Errorf("_mergeAuthzCollections command: %v", res["errmsg"])
}
return nil
}
// RestoreOplog attempts to restore a MongoDB oplog.
func (restore *MongoRestore) RestoreOplog() error {
log.Logv(log.Always, "replaying oplog")
intent := restore.manager.Oplog()
if intent == nil {
// this should not be reached
log.Logv(log.Always, "no oplog file provided, skipping oplog application")
return nil
}
if err := intent.BSONFile.Open(); err != nil {
return err
}
defer intent.BSONFile.Close()
// NewBufferlessBSONSource reads each bson document into its own buffer
// because bson.Unmarshal currently can't unmarshal binary types without
// them referencing the source buffer
bsonSource := db.NewDecodedBSONSource(db.NewBufferlessBSONSource(intent.BSONFile))
defer bsonSource.Close()
rawOplogEntry := &bson.Raw{}
var totalOps int64
var entrySize int
oplogProgressor := progress.NewCounter(intent.BSONSize)
if restore.ProgressManager != nil {
restore.ProgressManager.Attach("oplog", oplogProgressor)
defer restore.ProgressManager.Detach("oplog")
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
for bsonSource.Next(rawOplogEntry) {
entrySize = len(rawOplogEntry.Data)
entryAsOplog := db.Oplog{}
err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog)
if err != nil {
return fmt.Errorf("error reading oplog: %v", err)
}
if entryAsOplog.Operation == "n" {
//skip no-ops
continue
}
if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) {
log.Logvf(
log.DebugLow,
"timestamp %v is not below limit of %v; ending oplog restoration",
entryAsOplog.Timestamp,
restore.oplogLimit,
)
break
}
totalOps++
oplogProgressor.Inc(int64(entrySize))
err = restore.ApplyOps(session, []interface{}{entryAsOplog})
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
}
log.Logvf(log.Info, "applied %v ops", totalOps)
return nil
}
// RestoreUsersOrRoles accepts a collection type (Users or Roles) and restores the intent
// in the appropriate collection.
func (restore *MongoRestore) RestoreUsersOrRoles(collectionType string, intent *intents.Intent) error {
log.Logf(log.Always, "restoring %v from %v", collectionType, intent.BSONPath)
if intent.Size == 0 {
// MongoDB complains if we try and remove a non-existent collection, so we should
// just skip auth collections with empty .bson files to avoid gnarly logic later on.
log.Logf(log.Always, "%v file '%v' is empty; skipping %v restoration",
collectionType, intent.BSONPath, collectionType)
return nil
}
var tempCol, tempColCommandField string
switch collectionType {
case Users:
tempCol = restore.tempUsersCol
tempColCommandField = "tempUsersCollection"
case Roles:
tempCol = restore.tempRolesCol
tempColCommandField = "tempRolesCollection"
default:
return fmt.Errorf("cannot use %v as a collection type in RestoreUsersOrRoles", collectionType)
}
err := intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
tempColExists, err := restore.CollectionExists(&intents.Intent{DB: "admin", C: tempCol})
if err != nil {
return err
}
if tempColExists {
return fmt.Errorf("temporary collection admin.%v already exists. "+
"Drop it or specify new temporary collections with --tempUsersColl "+
"and --tempRolesColl", tempCol)
}
log.Logf(log.DebugLow, "restoring %v to temporary collection", collectionType)
err = restore.RestoreCollectionToDB("admin", tempCol, bsonSource, 0)
if err != nil {
return fmt.Errorf("error restoring %v: %v", collectionType, err)
}
// make sure we always drop the temporary collection
defer func() {
session, err := restore.SessionProvider.GetSession()
if err != nil {
// logging errors here because this has no way of returning that doesn't mask other errors
log.Logf(log.Always, "error establishing connection to drop temporary collection %v: %v", tempCol, err)
return
}
defer session.Close()
log.Logf(log.DebugHigh, "dropping temporary collection %v", tempCol)
err = session.DB("admin").C(tempCol).DropCollection()
if err != nil {
log.Logf(log.Always, "error dropping temporary collection %v: %v", tempCol, err)
}
}()
// If we are restoring a single database (--restoreDBUsersAndRoles), then the
// target database will be that database, and the _mergeAuthzCollections command
// will only restore users/roles of that database. If we are restoring the admin db or
// doing a full restore, we tell the command to merge users/roles of all databases.
userTargetDB := intent.DB
if userTargetDB == "admin" {
// _mergeAuthzCollections uses an empty db string as a sentinel for "all databases"
userTargetDB = ""
}
// we have to manually convert mgo's safety to a writeconcern object
writeConcern := bson.M{}
if restore.safety == nil {
writeConcern["w"] = 0
} else {
if restore.safety.WMode != "" {
writeConcern["w"] = restore.safety.WMode
} else {
writeConcern["w"] = restore.safety.W
}
}
command := bsonutil.MarshalD{
{"_mergeAuthzCollections", 1},
{tempColCommandField, "admin." + tempCol},
{"drop", restore.OutputOptions.Drop},
{"writeConcern", writeConcern},
{"db", userTargetDB},
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
log.Logf(log.DebugLow, "merging %v from temp collection '%v'", collectionType, tempCol)
res := bson.M{}
err = session.Run(command, &res)
if err != nil {
return fmt.Errorf("error running merge command: %v", err)
}
if util.IsFalsy(res["ok"]) {
return fmt.Errorf("_mergeAuthzCollections command: %v", res["errmsg"])
}
return nil
}
// RestoreOplog attempts to restore a MongoDB oplog.
func (restore *MongoRestore) RestoreOplog() error {
log.Log(log.Always, "replaying oplog")
intent := restore.manager.Oplog()
if intent == nil {
// this should not be reached
log.Log(log.Always, "no oplog.bson file in root of the dump directory, skipping oplog application")
return nil
}
if err := intent.BSONFile.Open(); err != nil {
return err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
entryArray := make([]interface{}, 0, 1024)
rawOplogEntry := &bson.Raw{}
var totalOps int64
var entrySize, bufferedBytes int
oplogProgressor := progress.NewCounter(intent.BSONSize)
bar := progress.Bar{
Name: "oplog",
Watching: oplogProgressor,
WaitTime: 3 * time.Second,
Writer: log.Writer(0),
BarLength: progressBarLength,
IsBytes: true,
}
bar.Start()
defer bar.Stop()
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
// To restore the oplog, we iterate over the oplog entries,
// filling up a buffer. Once the buffer reaches max document size,
// apply the current buffered ops and reset the buffer.
for bsonSource.Next(rawOplogEntry) {
entrySize = len(rawOplogEntry.Data)
if bufferedBytes+entrySize > oplogMaxCommandSize {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
entryArray = make([]interface{}, 0, 1024)
bufferedBytes = 0
}
entryAsOplog := db.Oplog{}
err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog)
if err != nil {
return fmt.Errorf("error reading oplog: %v", err)
}
if entryAsOplog.Operation == "n" {
//skip no-ops
continue
}
if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) {
log.Logf(
log.DebugLow,
"timestamp %v is not below limit of %v; ending oplog restoration",
entryAsOplog.Timestamp,
restore.oplogLimit,
)
break
}
totalOps++
bufferedBytes += entrySize
oplogProgressor.Inc(int64(entrySize))
entryArray = append(entryArray, entryAsOplog)
}
// finally, flush the remaining entries
if len(entryArray) > 0 {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
}
log.Logf(log.Info, "applied %v ops", totalOps)
return nil
}
// RestoreOplog attempts to restore a MongoDB oplog.
func (restore *MongoRestore) RestoreOplog() error {
log.Logv(log.Always, "replaying oplog")
intent := restore.manager.Oplog()
if intent == nil {
// this should not be reached
log.Logv(log.Always, "no oplog file provided, skipping oplog application")
return nil
}
if err := intent.BSONFile.Open(); err != nil {
return err
}
defer intent.BSONFile.Close()
// NewBufferlessBSONSource reads each bson document into its own buffer
// because bson.Unmarshal currently can't unmarshal binary types without
// them referencing the source buffer
bsonSource := db.NewDecodedBSONSource(db.NewBufferlessBSONSource(intent.BSONFile))
defer bsonSource.Close()
entryArray := make([]interface{}, 0, 1024)
rawOplogEntry := &bson.Raw{}
var totalOps int64
var entrySize, bufferedBytes int
oplogProgressor := progress.NewCounter(intent.BSONSize)
if restore.ProgressManager != nil {
restore.ProgressManager.Attach("oplog", oplogProgressor)
defer restore.ProgressManager.Detach("oplog")
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
defer session.Close()
// To restore the oplog, we iterate over the oplog entries,
// filling up a buffer. Once the buffer reaches max document size,
// apply the current buffered ops and reset the buffer.
for bsonSource.Next(rawOplogEntry) {
entrySize = len(rawOplogEntry.Data)
if bufferedBytes+entrySize > oplogMaxCommandSize {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
entryArray = make([]interface{}, 0, 1024)
bufferedBytes = 0
}
entryAsOplog := db.Oplog{}
err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog)
if err != nil {
return fmt.Errorf("error reading oplog: %v", err)
}
if entryAsOplog.Operation == "n" {
//skip no-ops
continue
}
if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) {
log.Logvf(
log.DebugLow,
"timestamp %v is not below limit of %v; ending oplog restoration",
entryAsOplog.Timestamp,
restore.oplogLimit,
)
break
}
totalOps++
bufferedBytes += entrySize
oplogProgressor.Inc(int64(entrySize))
entryArray = append(entryArray, entryAsOplog)
}
// finally, flush the remaining entries
if len(entryArray) > 0 {
err = restore.ApplyOps(session, entryArray)
if err != nil {
return fmt.Errorf("error applying oplog: %v", err)
}
}
log.Logvf(log.Info, "applied %v ops", totalOps)
return nil
}