// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (int64, error) {
total, err := query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logf(log.Info, "\tcounted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
dumpProgressor := progress.NewCounter(int64(total))
bar := &progress.Bar{
Name: intent.Namespace(),
Watching: dumpProgressor,
BarLength: progressBarLength,
}
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(bar)
defer dump.ProgressManager.Detach(bar)
}
err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
_, dumpCount := dumpProgressor.Progress()
return dumpCount, err
}
// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (int64, error) {
// don't dump any data for views being dumped as views
if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
return 0, nil
}
var total int
var err error
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
_, dumpCount := dumpProgressor.Progress()
return dumpCount, err
}
// CreateIndexes takes in an intent and an array of index documents and
// attempts to create them using the createIndexes command. If that command
// fails, we fall back to individual index creation.
func (restore *MongoRestore) CreateIndexes(intent *intents.Intent, indexes []IndexDocument) error {
// first, sanitize the indexes
for _, index := range indexes {
// update the namespace of the index before inserting
index.Options["ns"] = intent.Namespace()
// check for length violations before building the command
fullIndexName := fmt.Sprintf("%v.$%v", index.Options["ns"], index.Options["name"])
if len(fullIndexName) > 127 {
return fmt.Errorf(
"cannot restore index with namespace '%v': "+
"namespace is too long (max size is 127 bytes)", fullIndexName)
}
// remove the index version, forcing an update,
// unless we specifically want to keep it
if !restore.OutputOptions.KeepIndexVersion {
delete(index.Options, "v")
}
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
session.SetSafe(&mgo.Safe{})
defer session.Close()
// then attempt the createIndexes command
rawCommand := bson.D{
{"createIndexes", intent.C},
{"indexes", indexes},
}
results := bson.M{}
err = session.DB(intent.DB).Run(rawCommand, &results)
if err == nil {
return nil
}
if err.Error() != "no such cmd: createIndexes" {
return fmt.Errorf("createIndex error: %v", err)
}
// if we're here, the connected server does not support the command, so we fall back
log.Log(log.Info, "\tcreateIndexes command not supported, attemping legacy index insertion")
for _, idx := range indexes {
log.Logf(log.Info, "\tmanually creating index %v", idx.Options["name"])
err = restore.LegacyInsertIndex(intent, idx)
if err != nil {
return fmt.Errorf("error creating index %v: %v", idx.Options["name"], err)
}
}
return nil
}
// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (err error) {
total, err := query.Count()
if err != nil {
return fmt.Errorf("error reading from db: %v", err)
}
log.Logf(log.Info, "\t%v documents", total)
dumpProgressor := progress.NewCounter(int64(total))
bar := &progress.Bar{
Name: intent.Namespace(),
Watching: dumpProgressor,
BarLength: progressBarLength,
}
dump.progressManager.Attach(bar)
defer dump.progressManager.Detach(bar)
return dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
}
// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (int64, error) {
var total int
var err error
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
bar := &progress.Bar{
Name: intent.Namespace(),
Watching: dumpProgressor,
BarLength: progressBarLength,
}
dump.progressManager.Attach(bar)
defer dump.progressManager.Detach(bar)
err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
_, dumpCount := dumpProgressor.Progress()
return dumpCount, err
}
// RestoreIntent attempts to restore a given intent into MongoDB.
func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error {
collectionExists, err := restore.CollectionExists(intent)
if err != nil {
return fmt.Errorf("error reading database: %v", err)
}
if restore.safety == nil && !restore.OutputOptions.Drop && collectionExists {
log.Logvf(log.Always, "restoring to existing collection %v without dropping", intent.Namespace())
log.Logv(log.Always, "Important: restored data will be inserted without raising errors; check your server log")
}
if restore.OutputOptions.Drop {
if collectionExists {
if strings.HasPrefix(intent.C, "system.") {
log.Logvf(log.Always, "cannot drop system collection %v, skipping", intent.Namespace())
} else {
log.Logvf(log.Info, "dropping collection %v before restoring", intent.Namespace())
err = restore.DropCollection(intent)
if err != nil {
return err // no context needed
}
collectionExists = false
}
} else {
log.Logvf(log.DebugLow, "collection %v doesn't exist, skipping drop command", intent.Namespace())
}
}
var options bson.D
var indexes []IndexDocument
// get indexes from system.indexes dump if we have it but don't have metadata files
if intent.MetadataFile == nil {
if _, ok := restore.dbCollectionIndexes[intent.DB]; ok {
if indexes, ok = restore.dbCollectionIndexes[intent.DB][intent.C]; ok {
log.Logvf(log.Always, "no metadata; falling back to system.indexes")
}
}
}
logMessageSuffix := "with no metadata"
// first create the collection with options from the metadata file
if intent.MetadataFile != nil {
logMessageSuffix = "using options from metadata"
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
log.Logvf(log.Always, "reading metadata for %v from %v", intent.Namespace(), intent.MetadataLocation)
metadata, err := ioutil.ReadAll(intent.MetadataFile)
if err != nil {
return fmt.Errorf("error reading metadata from %v: %v", intent.MetadataLocation, err)
}
options, indexes, err = restore.MetadataFromJSON(metadata)
if err != nil {
return fmt.Errorf("error parsing metadata from %v: %v", intent.MetadataLocation, err)
}
if restore.OutputOptions.NoOptionsRestore {
log.Logv(log.Info, "not restoring collection options")
logMessageSuffix = "with no collection options"
options = nil
}
}
if !collectionExists {
log.Logvf(log.Info, "creating collection %v %s", intent.Namespace(), logMessageSuffix)
log.Logvf(log.DebugHigh, "using collection options: %#v", options)
err = restore.CreateCollection(intent, options)
if err != nil {
return fmt.Errorf("error creating collection %v: %v", intent.Namespace(), err)
}
} else {
log.Logvf(log.Info, "collection %v already exists - skipping collection create", intent.Namespace())
}
var documentCount int64
if intent.BSONFile != nil {
err = intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
log.Logvf(log.Always, "restoring %v from %v", intent.Namespace(), intent.Location)
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
documentCount, err = restore.RestoreCollectionToDB(intent.DB, intent.C, bsonSource, intent.BSONFile, intent.Size)
if err != nil {
return fmt.Errorf("error restoring from %v: %v", intent.Location, err)
}
}
// finally, add indexes
if len(indexes) > 0 && !restore.OutputOptions.NoIndexRestore {
log.Logvf(log.Always, "restoring indexes for collection %v from metadata", intent.Namespace())
err = restore.CreateIndexes(intent, indexes)
if err != nil {
return fmt.Errorf("error creating indexes for %v: %v", intent.Namespace(), err)
}
} else {
log.Logv(log.Always, "no indexes to restore")
}
log.Logvf(log.Always, "finished restoring %v (%v %v)",
intent.Namespace(), documentCount, util.Pluralize(int(documentCount), "document", "documents"))
return nil
}
// dumpMetadata gets the metadata for a collection and writes it
// in readable JSON format.
func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
var err error
nsID := fmt.Sprintf("%v.%v", intent.DB, intent.C)
meta := Metadata{
// We have to initialize Indexes to an empty slice, not nil, so that an empty
// array is marshalled into json instead of null. That is, {indexes:[]} is okay
// but {indexes:null} will cause assertions in our legacy C++ mongotools
Indexes: []interface{}{},
}
// The collection options were already gathered while building the list of intents.
// We convert them to JSON so that they can be written to the metadata json file as text.
if intent.Options != nil {
if meta.Options, err = bsonutil.ConvertBSONValueToJSON(*intent.Options); err != nil {
return fmt.Errorf("error converting collection options to JSON: %v", err)
}
} else {
meta.Options = nil
}
// Second, we read the collection's index information by either calling
// listIndexes (pre-2.7 systems) or querying system.indexes.
// We keep a running list of all the indexes
// for the current collection as we iterate over the cursor, and include
// that list as the "indexes" field of the metadata document.
log.Logvf(log.DebugHigh, "\treading indexes for `%v`", nsID)
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
// get the indexes
indexesIter, err := db.GetIndexes(session.DB(intent.DB).C(intent.C))
if err != nil {
return err
}
if indexesIter == nil {
log.Logvf(log.Always, "the collection %v appears to have been dropped after the dump started", intent.Namespace())
return nil
}
indexOpts := &bson.D{}
for indexesIter.Next(indexOpts) {
convertedIndex, err := bsonutil.ConvertBSONValueToJSON(*indexOpts)
if err != nil {
return fmt.Errorf("error converting index (%#v): %v", convertedIndex, err)
}
meta.Indexes = append(meta.Indexes, convertedIndex)
}
if err := indexesIter.Err(); err != nil {
return fmt.Errorf("error getting indexes for collection `%v`: %v", nsID, err)
}
// Finally, we send the results to the writer as JSON bytes
jsonBytes, err := json.Marshal(meta)
if err != nil {
return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", nsID, err)
}
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
// make a buffered writer for nicer disk i/o
w := bufio.NewWriter(intent.MetadataFile)
_, err = w.Write(jsonBytes)
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
err = w.Flush()
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
return nil
}
// DumpIntent dumps the specified database's collection.
func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
// in mgo, setting prefetch = 1.0 causes the driver to make requests for
// more results as soon as results are returned. This effectively
// duplicates the behavior of an exhaust cursor.
session.SetPrefetch(1.0)
err = intent.BSONFile.Open()
if err != nil {
return err
}
defer intent.BSONFile.Close()
var findQuery *mgo.Query
switch {
case len(dump.query) > 0:
findQuery = session.DB(intent.DB).C(intent.C).Find(dump.query)
case dump.InputOptions.TableScan:
// ---forceTablesScan runs the query without snapshot enabled
findQuery = session.DB(intent.DB).C(intent.C).Find(nil)
default:
findQuery = session.DB(intent.DB).C(intent.C).Find(nil).Snapshot()
}
var dumpCount int64
if dump.OutputOptions.Out == "-" {
log.Logvf(log.Always, "writing %v to stdout", intent.Namespace())
dumpCount, err = dump.dumpQueryToWriter(findQuery, intent)
if err == nil {
// on success, print the document count
log.Logvf(log.Always, "dumped %v %v", dumpCount, docPlural(dumpCount))
}
return err
}
// set where the intent will be written to
if dump.OutputOptions.Archive != "" {
if dump.OutputOptions.Archive == "-" {
intent.Location = "archive on stdout"
} else {
intent.Location = fmt.Sprintf("archive '%v'", dump.OutputOptions.Archive)
}
}
if !dump.OutputOptions.Repair {
log.Logvf(log.Always, "writing %v to %v", intent.Namespace(), intent.Location)
if dumpCount, err = dump.dumpQueryToWriter(findQuery, intent); err != nil {
return err
}
} else {
// handle repairs as a special case, since we cannot count them
log.Logvf(log.Always, "writing repair of %v to %v", intent.Namespace(), intent.Location)
repairIter := session.DB(intent.DB).C(intent.C).Repair()
repairCounter := progress.NewCounter(1) // this counter is ignored
if err := dump.dumpIterToWriter(repairIter, intent.BSONFile, repairCounter); err != nil {
return fmt.Errorf("repair error: %v", err)
}
_, repairCount := repairCounter.Progress()
log.Logvf(log.Always, "\trepair cursor found %v %v in %v",
repairCount, docPlural(repairCount), intent.Namespace())
}
log.Logvf(log.Always, "done dumping %v (%v %v)", intent.Namespace(), dumpCount, docPlural(dumpCount))
return nil
}
// dumpQueryToIntent takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToIntent(
query *mgo.Query, intent *intents.Intent, buffer resettableOutputBuffer) (dumpCount int64, err error) {
// restore of views from archives require an empty collection as the trigger to create the view
// so, we open here before the early return if IsView so that we write an empty collection to the archive
err = intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer func() {
closeErr := intent.BSONFile.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
// don't dump any data for views being dumped as views
if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
return 0, nil
}
var total int
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
var f io.Writer
f = intent.BSONFile
if buffer != nil {
buffer.Reset(f)
f = buffer
defer func() {
closeErr := buffer.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
}
err = dump.dumpIterToWriter(query.Iter(), f, dumpProgressor)
dumpCount, _ = dumpProgressor.Progress()
if err != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), err)
}
return
}