// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (int64, error) {
// don't dump any data for views being dumped as views
if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
return 0, nil
}
var total int
var err error
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
_, dumpCount := dumpProgressor.Progress()
return dumpCount, err
}
// dumpQueryToIntent takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToIntent(
query *mgo.Query, intent *intents.Intent, buffer resettableOutputBuffer) (dumpCount int64, err error) {
// restore of views from archives require an empty collection as the trigger to create the view
// so, we open here before the early return if IsView so that we write an empty collection to the archive
err = intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer func() {
closeErr := intent.BSONFile.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
// don't dump any data for views being dumped as views
if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
return 0, nil
}
var total int
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
var f io.Writer
f = intent.BSONFile
if buffer != nil {
buffer.Reset(f)
f = buffer
defer func() {
closeErr := buffer.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
}
err = dump.dumpIterToWriter(query.Iter(), f, dumpProgressor)
dumpCount, _ = dumpProgressor.Progress()
if err != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), err)
}
return
}
// dumpMetadata gets the metadata for a collection and writes it
// in readable JSON format.
func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
var err error
nsID := fmt.Sprintf("%v.%v", intent.DB, intent.C)
meta := Metadata{
// We have to initialize Indexes to an empty slice, not nil, so that an empty
// array is marshalled into json instead of null. That is, {indexes:[]} is okay
// but {indexes:null} will cause assertions in our legacy C++ mongotools
Indexes: []interface{}{},
}
// The collection options were already gathered while building the list of intents.
// We convert them to JSON so that they can be written to the metadata json file as text.
if intent.Options != nil {
if meta.Options, err = bsonutil.ConvertBSONValueToJSON(*intent.Options); err != nil {
return fmt.Errorf("error converting collection options to JSON: %v", err)
}
} else {
meta.Options = nil
}
// Second, we read the collection's index information by either calling
// listIndexes (pre-2.7 systems) or querying system.indexes.
// We keep a running list of all the indexes
// for the current collection as we iterate over the cursor, and include
// that list as the "indexes" field of the metadata document.
log.Logvf(log.DebugHigh, "\treading indexes for `%v`", nsID)
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
if intent.IsView() {
log.Logvf(log.DebugLow, "not dumping indexes metadata for '%v' because it is a view", intent.Namespace())
} else {
// get the indexes
indexesIter, err := db.GetIndexes(session.DB(intent.DB).C(intent.C))
if err != nil {
return err
}
if indexesIter == nil {
log.Logvf(log.Always, "the collection %v appears to have been dropped after the dump started", intent.Namespace())
return nil
}
indexOpts := &bson.D{}
for indexesIter.Next(indexOpts) {
convertedIndex, err := bsonutil.ConvertBSONValueToJSON(*indexOpts)
if err != nil {
return fmt.Errorf("error converting index (%#v): %v", convertedIndex, err)
}
meta.Indexes = append(meta.Indexes, convertedIndex)
}
if err := indexesIter.Err(); err != nil {
return fmt.Errorf("error getting indexes for collection `%v`: %v", nsID, err)
}
}
// Finally, we send the results to the writer as JSON bytes
jsonBytes, err := json.Marshal(meta)
if err != nil {
return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", nsID, err)
}
err = intent.MetadataFile.Open()
if err != nil {
return err
}
defer intent.MetadataFile.Close()
// make a buffered writer for nicer disk i/o
w := bufio.NewWriter(intent.MetadataFile)
_, err = w.Write(jsonBytes)
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
err = w.Flush()
if err != nil {
return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
}
return nil
}