// BuildWriteConcern takes a string and a NodeType indicating the type of node the write concern
// is intended to be used against, and converts the write concern string argument into an
// mgo.Safe object that's usable on sessions for that node type.
func BuildWriteConcern(writeConcern string, nodeType NodeType) (*mgo.Safe, error) {
sessionSafety, err := constructWCObject(writeConcern)
if err != nil {
return nil, err
}
if sessionSafety == nil {
log.Logvf(log.DebugLow, "using unacknowledged write concern")
return nil, nil
}
// for standalone mongods, set the default write concern to 1
if nodeType == Standalone {
log.Logvf(log.DebugLow, "standalone server: setting write concern %v to 1", w)
sessionSafety.W = 1
sessionSafety.WMode = ""
}
var writeConcernStr interface{}
if sessionSafety.WMode != "" {
writeConcernStr = sessionSafety.WMode
} else {
writeConcernStr = sessionSafety.W
}
log.Logvf(log.Info, "using write concern: %v='%v', %v=%v, %v=%v, %v=%v",
w, writeConcernStr,
j, sessionSafety.J,
fSync, sessionSafety.FSync,
wTimeout, sessionSafety.WTimeout,
)
return sessionSafety, nil
}
// handleBSONInsteadOfDirectory updates -d and -c settings based on
// the path to the BSON file passed to mongorestore. This is only
// applicable if the target path points to a .bson file.
//
// As an example, when the user passes 'dump/mydb/col.bson', this method
// will infer that 'mydb' is the database and 'col' is the collection name.
func (restore *MongoRestore) handleBSONInsteadOfDirectory(path string) error {
// we know we have been given a non-directory, so we should handle it
// like a bson file and infer as much as we can
if restore.NSOptions.Collection == "" {
// if the user did not set -c, use the file name for the collection
newCollectionName, fileType := restore.getInfoFromFilename(path)
if fileType != BSONFileType {
return fmt.Errorf("file %v does not have .bson extension", path)
}
restore.NSOptions.Collection = newCollectionName
log.Logvf(log.DebugLow, "inferred collection '%v' from file", restore.NSOptions.Collection)
}
if restore.NSOptions.DB == "" {
// if the user did not set -d, use the directory containing the target
// file as the db name (as it would be in a dump directory). If
// we cannot determine the directory name, use "test"
dirForFile := filepath.Base(filepath.Dir(path))
if dirForFile == "." || dirForFile == ".." {
dirForFile = "test"
}
restore.NSOptions.DB = dirForFile
log.Logvf(log.DebugLow, "inferred db '%v' from the file's directory", restore.NSOptions.DB)
}
return nil
}
// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToWriter(
query *mgo.Query, intent *intents.Intent) (int64, error) {
var total int
var err error
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
_, dumpCount := dumpProgressor.Progress()
return dumpCount, err
}
func (dump *MongoDump) createIntentFromOptions(dbName string, ci *collectionInfo) error {
if dump.shouldSkipCollection(ci.Name) {
log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, ci.Name)
return nil
}
if dump.OutputOptions.ViewsAsCollections && !ci.IsView() {
log.Logvf(log.DebugLow, "skipping dump of %v.%v because it is not a view", dbName, ci.Name)
return nil
}
intent, err := dump.NewIntent(dbName, ci.Name)
if err != nil {
return err
}
if dump.OutputOptions.ViewsAsCollections {
log.Logvf(log.DebugLow, "not dumping metadata for %v.%v because it is a view", dbName, ci.Name)
intent.MetadataFile = nil
} else if ci.IsView() {
log.Logvf(log.DebugLow, "not dumping data for %v.%v because it is a view", dbName, ci.Name)
// only write a bson file if using archive
if dump.OutputOptions.Archive == "" {
intent.BSONFile = nil
}
}
intent.Options = ci.Options
dump.manager.Put(intent)
log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
return nil
}
func handleSignals(finalizer func(), finishedChan chan struct{}) {
// explicitly ignore SIGPIPE; the tools should deal with write errors
noopChan := make(chan os.Signal)
signal.Notify(noopChan, syscall.SIGPIPE)
log.Logv(log.DebugLow, "will listen for SIGTERM, SIGINT, and SIGKILL")
sigChan := make(chan os.Signal, 2)
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
defer signal.Stop(sigChan)
if finalizer != nil {
select {
case sig := <-sigChan:
// first signal use finalizer to terminate cleanly
log.Logvf(log.Always, "signal '%s' received; attempting to shut down", sig)
finalizer()
case <-finishedChan:
return
}
}
select {
case sig := <-sigChan:
// second signal exits immediately
log.Logvf(log.Always, "signal '%s' received; forcefully terminating", sig)
os.Exit(util.ExitKill)
case <-finishedChan:
return
}
}
// CreateIntentForCollection builds an intent for the given database and collection name
// along with a path to a .bson collection file. It searches the file's parent directory
// for a matching metadata file.
//
// This method is not called by CreateIntentsForDB,
// it is only used in the case where --db and --collection flags are set.
func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, dir archive.DirLike) error {
log.Logvf(log.DebugLow, "reading collection %v for database %v from %v",
collection, db, dir.Path())
// first make sure the bson file exists and is valid
_, err := dir.Stat()
if err != nil {
return err
}
if dir.IsDir() {
return fmt.Errorf("file %v is a directory, not a bson file", dir.Path())
}
baseName, fileType := restore.getInfoFromFilename(dir.Name())
if fileType != BSONFileType {
return fmt.Errorf("file %v does not have .bson extension", dir.Path())
}
// then create its intent
intent := &intents.Intent{
DB: db,
C: collection,
Size: dir.Size(),
Location: dir.Path(),
}
intent.BSONFile = &realBSONFile{path: dir.Path(), intent: intent, gzip: restore.InputOptions.Gzip}
// finally, check if it has a .metadata.json file in its folder
log.Logvf(log.DebugLow, "scanning directory %v for metadata", dir.Name())
entries, err := dir.Parent().ReadDir()
if err != nil {
// try and carry on if we can
log.Logvf(log.Info, "error attempting to locate metadata for file: %v", err)
log.Logv(log.Info, "restoring collection without metadata")
restore.manager.Put(intent)
return nil
}
metadataName := baseName + ".metadata.json"
if restore.InputOptions.Gzip {
metadataName += ".gz"
}
for _, entry := range entries {
if entry.Name() == metadataName {
metadataPath := entry.Path()
log.Logvf(log.Info, "found metadata for collection at %v", metadataPath)
intent.MetadataLocation = metadataPath
intent.MetadataFile = &realMetadataFile{path: metadataPath, intent: intent, gzip: restore.InputOptions.Gzip}
break
}
}
if intent.MetadataFile == nil {
log.Logv(log.Info, "restoring collection without metadata")
}
restore.manager.Put(intent)
return nil
}
// Run creates and runs a parser with the Demultiplexer as a consumer
func (demux *Demultiplexer) Run() error {
parser := Parser{In: demux.In}
err := parser.ReadAllBlocks(demux)
if len(demux.outs) > 0 {
log.Logvf(log.Always, "demux finishing when there are still outs (%v)", len(demux.outs))
}
log.Logvf(log.DebugLow, "demux finishing (err:%v)", err)
return err
}
// Report collects the stat info for a single node and sends found hostnames on
// the "discover" channel if checkShards is true.
func (node *NodeMonitor) Poll(discover chan string, checkShards bool) (*status.ServerStatus, error) {
stat := &status.ServerStatus{}
log.Logvf(log.DebugHigh, "getting session on server: %v", node.host)
s, err := node.sessionProvider.GetSession()
if err != nil {
log.Logvf(log.DebugLow, "got error getting session to server %v", node.host)
return nil, err
}
log.Logvf(log.DebugHigh, "got session on server: %v", node.host)
// The read pref for the session must be set to 'secondary' to enable using
// the driver with 'direct' connections, which disables the built-in
// replset discovery mechanism since we do our own node discovery here.
s.SetMode(mgo.Eventual, true)
// Disable the socket timeout - otherwise if db.serverStatus() takes a long time on the server
// side, the client will close the connection early and report an error.
s.SetSocketTimeout(0)
defer s.Close()
err = s.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 0}}, stat)
if err != nil {
log.Logvf(log.DebugLow, "got error calling serverStatus against server %v", node.host)
return nil, err
}
statMap := make(map[string]interface{})
s.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 0}}, statMap)
stat.Flattened = status.Flatten(statMap)
node.Err = nil
stat.SampleTime = time.Now()
if stat.Repl != nil && discover != nil {
for _, host := range stat.Repl.Hosts {
discover <- host
}
for _, host := range stat.Repl.Passives {
discover <- host
}
}
node.alias = stat.Host
stat.Host = node.host
if discover != nil && stat != nil && status.IsMongos(stat) && checkShards {
log.Logvf(log.DebugLow, "checking config database to discover shards")
shardCursor := s.DB("config").C("shards").Find(bson.M{}).Iter()
shard := ConfigShard{}
for shardCursor.Next(&shard) {
shardHosts := strings.Split(shard.Host, ",")
for _, shardHost := range shardHosts {
discover <- shardHost
}
}
shardCursor.Close()
}
return stat, nil
}
// dumpQueryToIntent takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToIntent(
query *mgo.Query, intent *intents.Intent, buffer resettableOutputBuffer) (dumpCount int64, err error) {
// restore of views from archives require an empty collection as the trigger to create the view
// so, we open here before the early return if IsView so that we write an empty collection to the archive
err = intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer func() {
closeErr := intent.BSONFile.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
// don't dump any data for views being dumped as views
if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
return 0, nil
}
var total int
if len(dump.query) == 0 {
total, err = query.Count()
if err != nil {
return int64(0), fmt.Errorf("error reading from db: %v", err)
}
log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
} else {
log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
}
dumpProgressor := progress.NewCounter(int64(total))
if dump.ProgressManager != nil {
dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
defer dump.ProgressManager.Detach(intent.Namespace())
}
var f io.Writer
f = intent.BSONFile
if buffer != nil {
buffer.Reset(f)
f = buffer
defer func() {
closeErr := buffer.Close()
if err == nil && closeErr != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
}
}()
}
err = dump.dumpIterToWriter(query.Iter(), f, dumpProgressor)
dumpCount, _ = dumpProgressor.Progress()
if err != nil {
err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), err)
}
return
}
// HeaderBSON is part of the ParserConsumer interface and receives headers from parser.
// Its main role is to implement opens and EOFs of the embedded stream.
func (demux *Demultiplexer) HeaderBSON(buf []byte) error {
colHeader := NamespaceHeader{}
err := bson.Unmarshal(buf, &colHeader)
if err != nil {
return newWrappedError("header bson doesn't unmarshal as a collection header", err)
}
log.Logvf(log.DebugHigh, "demux namespaceHeader: %v", colHeader)
if colHeader.Collection == "" {
return newError("collection header is missing a Collection")
}
demux.currentNamespace = colHeader.Database + "." + colHeader.Collection
if _, ok := demux.outs[demux.currentNamespace]; !ok {
if demux.NamespaceChan != nil {
demux.NamespaceChan <- demux.currentNamespace
err := <-demux.NamespaceErrorChan
if err == io.EOF {
// if the Prioritizer sends us back an io.EOF then it's telling us that
// it's finishing and doesn't need any more namespace announcements.
close(demux.NamespaceChan)
demux.NamespaceChan = nil
return nil
}
if err != nil {
return newWrappedError("failed arranging a consumer for new namespace", err)
}
}
}
if colHeader.EOF {
demux.outs[demux.currentNamespace].Close()
length := int64(demux.lengths[demux.currentNamespace])
crcUInt64, ok := demux.outs[demux.currentNamespace].Sum64()
if ok {
crc := int64(crcUInt64)
if crc != colHeader.CRC {
return fmt.Errorf("CRC mismatch for namespace %v, %v!=%v",
demux.currentNamespace,
crc,
colHeader.CRC,
)
}
log.Logvf(log.DebugHigh,
"demux checksum for namespace %v is correct (%v), %v bytes",
demux.currentNamespace, crc, length)
} else {
log.Logvf(log.DebugHigh,
"demux checksum for namespace %v was not calculated.",
demux.currentNamespace)
}
delete(demux.outs, demux.currentNamespace)
delete(demux.lengths, demux.currentNamespace)
// in case we get a BSONBody with this block,
// we want to ensure that that causes an error
demux.currentNamespace = ""
}
return nil
}
// validateReaderFields is a helper to validate fields for input readers
func validateReaderFields(fields []string) error {
if err := validateFields(fields); err != nil {
return err
}
if len(fields) == 1 {
log.Logvf(log.Info, "using field: %v", fields[0])
} else {
log.Logvf(log.Info, "using fields: %v", strings.Join(fields, ","))
}
return nil
}
// handle logic for 'put' command.
func (mf *MongoFiles) handlePut(gfs *mgo.GridFS) (output string, err error) {
localFileName := mf.getLocalFileName(nil)
// check if --replace flag turned on
if mf.StorageOptions.Replace {
err := gfs.Remove(mf.FileName)
if err != nil {
return "", err
}
output = fmt.Sprintf("removed all instances of '%v' from GridFS\n", mf.FileName)
}
var localFile io.ReadCloser
if localFileName == "-" {
localFile = os.Stdin
} else {
localFile, err = os.Open(localFileName)
if err != nil {
return "", fmt.Errorf("error while opening local file '%v' : %v\n", localFileName, err)
}
defer localFile.Close()
log.Logvf(log.DebugLow, "creating GridFS file '%v' from local file '%v'", mf.FileName, localFileName)
}
gFile, err := gfs.Create(mf.FileName)
if err != nil {
return "", fmt.Errorf("error while creating '%v' in GridFS: %v\n", mf.FileName, err)
}
defer func() {
// GridFS files flush a buffer on Close(), so it's important we
// capture any errors that occur as this function exits and
// overwrite the error if earlier writes executed successfully
if closeErr := gFile.Close(); err == nil && closeErr != nil {
log.Logvf(log.DebugHigh, "error occurred while closing GridFS file handler")
err = fmt.Errorf("error while storing '%v' into GridFS: %v\n", localFileName, closeErr)
}
}()
// set optional mime type
if mf.StorageOptions.ContentType != "" {
gFile.SetContentType(mf.StorageOptions.ContentType)
}
n, err := io.Copy(gFile, localFile)
if err != nil {
return "", fmt.Errorf("error while storing '%v' into GridFS: %v\n", localFileName, err)
}
log.Logvf(log.DebugLow, "copied %v bytes to server", n)
output += fmt.Sprintf("added file: %v\n", gFile.Name())
return output, nil
}
// Convert implements the Converter interface for JSON input. It converts a
// JSONConverter struct to a BSON document.
func (c JSONConverter) Convert() (bson.D, error) {
document, err := json.UnmarshalBsonD(c.data)
if err != nil {
return nil, fmt.Errorf("error unmarshaling bytes on document #%v: %v", c.index, err)
}
log.Logvf(log.DebugHigh, "got line: %v", document)
bsonD, err := bsonutil.GetExtendedBsonD(document)
if err != nil {
return nil, fmt.Errorf("error getting extended BSON for document #%v: %v", c.index, err)
}
log.Logvf(log.DebugHigh, "got extended line: %#v", bsonD)
return bsonD, nil
}
func (dump *MongoDump) createIntentFromOptions(dbName string, ci *collectionInfo) error {
if dump.shouldSkipCollection(ci.Name) {
log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, ci.Name)
return nil
}
intent, err := dump.NewIntent(dbName, ci.Name)
if err != nil {
return err
}
intent.Options = ci.Options
dump.manager.Put(intent)
log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
return nil
}
// DumpIntents iterates through the previously-created intents and
// dumps all of the found collections.
func (dump *MongoDump) DumpIntents() error {
resultChan := make(chan error)
jobs := dump.OutputOptions.NumParallelCollections
if numIntents := len(dump.manager.Intents()); jobs > numIntents {
jobs = numIntents
}
if jobs > 1 {
dump.manager.Finalize(intents.LongestTaskFirst)
} else {
dump.manager.Finalize(intents.Legacy)
}
log.Logvf(log.Info, "dumping up to %v collections in parallel", jobs)
// start a goroutine for each job thread
for i := 0; i < jobs; i++ {
go func(id int) {
buffer := dump.getResettableOutputBuffer()
log.Logvf(log.DebugHigh, "starting dump routine with id=%v", id)
for {
intent := dump.manager.Pop()
if intent == nil {
log.Logvf(log.DebugHigh, "ending dump routine with id=%v, no more work to do", id)
resultChan <- nil
return
}
if intent.BSONFile != nil {
err := dump.DumpIntent(intent, buffer)
if err != nil {
resultChan <- err
return
}
}
dump.manager.Finish(intent)
}
}(i)
}
// wait until all goroutines are done or one of them errors out
for i := 0; i < jobs; i++ {
if err := <-resultChan; err != nil {
return err
}
}
return nil
}
// checkOplogTimestampExists checks to make sure the oplog hasn't rolled over
// since mongodump started. It does this by checking the oldest oplog entry
// still in the database and making sure it happened at or before the timestamp
// captured at the start of the dump.
func (dump *MongoDump) checkOplogTimestampExists(ts bson.MongoTimestamp) (bool, error) {
oldestOplogEntry := db.Oplog{}
err := dump.sessionProvider.FindOne("local", dump.oplogCollection, 0, nil, []string{"+$natural"}, &oldestOplogEntry, 0)
if err != nil {
return false, fmt.Errorf("unable to read entry from oplog: %v", err)
}
log.Logvf(log.DebugHigh, "oldest oplog entry has timestamp %v", oldestOplogEntry.Timestamp)
if oldestOplogEntry.Timestamp > ts {
log.Logvf(log.Info, "oldest oplog entry of timestamp %v is older than %v",
oldestOplogEntry.Timestamp, ts)
return false, nil
}
return true, nil
}
// Run executes the mongotop program.
func (mt *MongoTop) Run() error {
connURL := mt.Options.Host
if connURL == "" {
connURL = "127.0.0.1"
}
if mt.Options.Port != "" {
connURL = connURL + ":" + mt.Options.Port
}
hasData := false
numPrinted := 0
for {
if mt.OutputOptions.RowCount > 0 && numPrinted > mt.OutputOptions.RowCount {
return nil
}
numPrinted++
diff, err := mt.runDiff()
if err != nil {
// If this is the first time trying to poll the server and it fails,
// just stop now instead of trying over and over.
if !hasData {
return err
}
log.Logvf(log.Always, "Error: %v\n", err)
time.Sleep(mt.Sleeptime)
}
// if this is the first time and the connection is successful, print
// the connection message
if !hasData && !mt.OutputOptions.Json {
log.Logvf(log.Always, "connected to: %v\n", connURL)
}
hasData = true
if diff != nil {
if mt.OutputOptions.Json {
fmt.Println(diff.JSON())
} else {
fmt.Println(diff.Grid())
}
}
time.Sleep(mt.Sleeptime)
}
}
// AddNewNode adds a new host name to be monitored and spawns the necessary
// goroutine to collect data from it.
func (mstat *MongoStat) AddNewNode(fullhost string) error {
mstat.nodesLock.Lock()
defer mstat.nodesLock.Unlock()
// Remove the 'shardXX/' prefix from the hostname, if applicable
pieces := strings.Split(fullhost, "/")
fullhost = pieces[len(pieces)-1]
if _, hasKey := mstat.Nodes[fullhost]; hasKey {
return nil
}
for _, node := range mstat.Nodes {
if node.alias == fullhost {
return nil
}
}
log.Logvf(log.DebugLow, "adding new host to monitoring: %v", fullhost)
// Create a new node monitor for this host
node, err := NewNodeMonitor(*mstat.Options, fullhost)
if err != nil {
return err
}
mstat.Nodes[fullhost] = node
go node.Watch(mstat.SleepInterval, mstat.Discovered, mstat.Cluster)
return nil
}
func GetCollections(database *mgo.Database, name string) (*mgo.Iter, bool, error) {
var cmdResult struct {
Cursor struct {
FirstBatch []bson.Raw `bson:"firstBatch"`
NS string
Id int64
}
}
command := bson.D{{"listCollections", 1}, {"cursor", bson.M{}}}
if len(name) > 0 {
command = bson.D{{"listCollections", 1}, {"filter", bson.M{"name": name}}, {"cursor", bson.M{}}}
}
err := database.Run(command, &cmdResult)
switch {
case err == nil:
ns := strings.SplitN(cmdResult.Cursor.NS, ".", 2)
if len(ns) < 2 {
return nil, false, fmt.Errorf("server returned invalid cursor.ns `%v` on listCollections for `%v`: %v",
cmdResult.Cursor.NS, database.Name, err)
}
return database.Session.DB(ns[0]).C(ns[1]).NewIter(database.Session, cmdResult.Cursor.FirstBatch, cmdResult.Cursor.Id, nil), false, nil
case IsNoCmd(err):
log.Logvf(log.DebugLow, "No support for listCollections command, falling back to querying system.namespaces")
iter, err := getCollectionsPre28(database, name)
return iter, true, err
default:
return nil, false, fmt.Errorf("error running `listCollections`. Database: `%v` Err: %v",
database.Name, err)
}
}
// JSON iterates through the BSON file and for each document it finds,
// recursively descends into objects and arrays and prints the human readable
// JSON representation.
// It returns the number of documents processed and a non-nil error if one is
// encountered before the end of the file is reached.
func (bd *BSONDump) JSON() (int, error) {
numFound := 0
if bd.BSONSource == nil {
panic("Tried to call JSON() before opening file")
}
decodedStream := db.NewDecodedBSONSource(bd.BSONSource)
var result bson.Raw
for decodedStream.Next(&result) {
if err := printJSON(&result, bd.Out, bd.BSONDumpOptions.Pretty); err != nil {
log.Logvf(log.Always, "unable to dump document %v: %v", numFound+1, err)
//if objcheck is turned on, stop now. otherwise keep on dumpin'
if bd.BSONDumpOptions.ObjCheck {
return numFound, err
}
} else {
_, err := bd.Out.Write([]byte("\n"))
if err != nil {
return numFound, err
}
}
numFound++
}
if err := decodedStream.Err(); err != nil {
return numFound, err
}
return numFound, nil
}
// GetIndexes returns an iterator to thethe raw index info for a collection by
// using the listIndexes command if available, or by falling back to querying
// against system.indexes (pre-3.0 systems). nil is returned if the collection
// does not exist.
func GetIndexes(coll *mgo.Collection) (*mgo.Iter, error) {
var cmdResult struct {
Cursor struct {
FirstBatch []bson.Raw `bson:"firstBatch"`
NS string
Id int64
}
}
err := coll.Database.Run(bson.D{{"listIndexes", coll.Name}, {"cursor", bson.M{}}}, &cmdResult)
switch {
case err == nil:
ns := strings.SplitN(cmdResult.Cursor.NS, ".", 2)
if len(ns) < 2 {
return nil, fmt.Errorf("server returned invalid cursor.ns `%v` on listIndexes for `%v`: %v",
cmdResult.Cursor.NS, coll.FullName, err)
}
ses := coll.Database.Session
return ses.DB(ns[0]).C(ns[1]).NewIter(ses, cmdResult.Cursor.FirstBatch, cmdResult.Cursor.Id, nil), nil
case IsNoCmd(err):
log.Logvf(log.DebugLow, "No support for listIndexes command, falling back to querying system.indexes")
return getIndexesPre28(coll)
case IsNoCollection(err):
return nil, nil
default:
return nil, fmt.Errorf("error running `listIndexes`. Collection: `%v` Err: %v", coll.FullName, err)
}
}
// Init performs preliminary setup operations for MongoDump.
func (dump *MongoDump) Init() error {
err := dump.ValidateOptions()
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
if dump.stdout == nil {
dump.stdout = os.Stdout
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
return fmt.Errorf("can't create session: %v", err)
}
// temporarily allow secondary reads for the isMongos check
dump.sessionProvider.SetReadPreference(mgo.Nearest)
dump.isMongos, err = dump.sessionProvider.IsMongos()
if err != nil {
return err
}
if dump.isMongos && dump.OutputOptions.Oplog {
return fmt.Errorf("can't use --oplog option when dumping from a mongos")
}
var mode mgo.Mode
if dump.ToolOptions.ReplicaSetName != "" || dump.isMongos {
mode = mgo.Primary
} else {
mode = mgo.Nearest
}
var tags bson.D
if dump.InputOptions.ReadPreference != "" {
mode, tags, err = db.ParseReadPreference(dump.InputOptions.ReadPreference)
if err != nil {
return fmt.Errorf("error parsing --readPreference : %v", err)
}
if len(tags) > 0 {
dump.sessionProvider.SetTags(tags)
}
}
// warn if we are trying to dump from a secondary in a sharded cluster
if dump.isMongos && mode != mgo.Primary {
log.Logvf(log.Always, db.WarningNonPrimaryMongosConnection)
}
dump.sessionProvider.SetReadPreference(mode)
dump.sessionProvider.SetTags(tags)
dump.sessionProvider.SetFlags(db.DisableSocketTimeout)
// return a helpful error message for mongos --repair
if dump.OutputOptions.Repair && dump.isMongos {
return fmt.Errorf("--repair flag cannot be used on a mongos")
}
dump.manager = intents.NewIntentManager()
return nil
}
// Watch continuously collects and processes stats for a single node on a
// regular interval. At each interval, it triggers the node's Poll function
// with the 'discover' channel.
func (node *NodeMonitor) Watch(sleep time.Duration, discover chan string, cluster ClusterMonitor) {
var cycle uint64
for ticker := time.Tick(sleep); ; <-ticker {
log.Logvf(log.DebugHigh, "polling server: %v", node.host)
stat, err := node.Poll(discover, cycle%10 == 0)
if stat != nil {
log.Logvf(log.DebugHigh, "successfully got statline from host: %v", node.host)
}
var nodeError *status.NodeError
if err != nil {
nodeError = status.NewNodeError(node.host, err)
}
cluster.Update(stat, nodeError)
cycle++
}
}
// tokensToBSON reads in slice of records - along with ordered column names -
// and returns a BSON document for the record.
func tokensToBSON(colSpecs []ColumnSpec, tokens []string, numProcessed uint64, ignoreBlanks bool) (bson.D, error) {
log.Logvf(log.DebugHigh, "got line: %v", tokens)
var parsedValue interface{}
document := bson.D{}
for index, token := range tokens {
if token == "" && ignoreBlanks {
continue
}
if index < len(colSpecs) {
parsedValue, err := colSpecs[index].Parser.Parse(token)
if err != nil {
log.Logvf(log.DebugHigh, "parse failure in document #%d for column '%s',"+
"could not parse token '%s' to type %s",
numProcessed, colSpecs[index].Name, token, colSpecs[index].TypeName)
switch colSpecs[index].ParseGrace {
case pgAutoCast:
parsedValue = autoParse(token)
case pgSkipField:
continue
case pgSkipRow:
log.Logvf(log.Always, "skipping row #%d: %v", numProcessed, tokens)
return nil, coercionError{}
case pgStop:
return nil, fmt.Errorf("type coercion failure in document #%d for column '%s', "+
"could not parse token '%s' to type %s",
numProcessed, colSpecs[index].Name, token, colSpecs[index].TypeName)
}
}
if strings.Index(colSpecs[index].Name, ".") != -1 {
setNestedValue(colSpecs[index].Name, parsedValue, &document)
} else {
document = append(document, bson.DocElem{Name: colSpecs[index].Name, Value: parsedValue})
}
} else {
parsedValue = autoParse(token)
key := "field" + strconv.Itoa(index)
if util.StringSliceContains(ColumnNames(colSpecs), key) {
return nil, fmt.Errorf("duplicate field name - on %v - for token #%v ('%v') in document #%v",
key, index+1, parsedValue, numProcessed)
}
document = append(document, bson.DocElem{Name: key, Value: parsedValue})
}
}
return document, nil
}
// GetDumpAuthVersion reads the admin.system.version collection in the dump directory
// to determine the authentication version of the files in the dump. If that collection is not
// present in the dump, we try to infer the authentication version based on its absence.
// Returns the authentication version number and any errors that occur.
func (restore *MongoRestore) GetDumpAuthVersion() (int, error) {
// first handle the case where we have no auth version
intent := restore.manager.AuthVersion()
if intent == nil {
if restore.InputOptions.RestoreDBUsersAndRoles {
// If we are using --restoreDbUsersAndRoles, we cannot guarantee an
// $admin.system.version collection from a 2.6 server,
// so we can assume up to version 3.
log.Logvf(log.Always, "no system.version bson file found in '%v' database dump", restore.NSOptions.DB)
log.Logv(log.Always, "warning: assuming users and roles collections are of auth version 3")
log.Logv(log.Always, "if users are from an earlier version of MongoDB, they may not restore properly")
return 3, nil
}
log.Logv(log.Info, "no system.version bson file found in dump")
log.Logv(log.Always, "assuming users in the dump directory are from <= 2.4 (auth version 1)")
return 1, nil
}
err := intent.BSONFile.Open()
if err != nil {
return 0, err
}
defer intent.BSONFile.Close()
bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(intent.BSONFile))
defer bsonSource.Close()
versionDoc := bson.M{}
for bsonSource.Next(&versionDoc) {
id, ok := versionDoc["_id"].(string)
if ok && id == "authSchema" {
authVersion, ok := versionDoc["currentVersion"].(int)
if ok {
return authVersion, nil
}
return 0, fmt.Errorf("can't unmarshal system.version curentVersion as an int")
}
log.Logvf(log.DebugLow, "system.version document is not an authSchema %v", versionDoc["_id"])
}
err = bsonSource.Err()
if err != nil {
log.Logvf(log.Info, "can't unmarshal system.version document: %v", err)
}
return 0, fmt.Errorf("system.version bson file does not have authSchema document")
}
// getSourceReader returns an io.Reader to read from the input source. Also
// returns a progress.Progressor which can be used to track progress if the
// reader supports it.
func (imp *MongoImport) getSourceReader() (io.ReadCloser, int64, error) {
if imp.InputOptions.File != "" {
file, err := os.Open(util.ToUniversalPath(imp.InputOptions.File))
if err != nil {
return nil, -1, err
}
fileStat, err := file.Stat()
if err != nil {
return nil, -1, err
}
log.Logvf(log.Info, "filesize: %v bytes", fileStat.Size())
return file, int64(fileStat.Size()), err
}
log.Logvf(log.Info, "reading from stdin")
// Stdin has undefined max size, so return 0
return os.Stdin, 0, nil
}
// AddMetadata adds a metadata data structure to a prelude and does the required bookkeeping.
func (prelude *Prelude) AddMetadata(cm *CollectionMetadata) {
prelude.NamespaceMetadatas = append(prelude.NamespaceMetadatas, cm)
if prelude.NamespaceMetadatasByDB == nil {
prelude.NamespaceMetadatasByDB = make(map[string][]*CollectionMetadata)
}
_, ok := prelude.NamespaceMetadatasByDB[cm.Database]
if !ok {
prelude.DBS = append(prelude.DBS, cm.Database)
}
prelude.NamespaceMetadatasByDB[cm.Database] = append(prelude.NamespaceMetadatasByDB[cm.Database], cm)
log.Logvf(log.Info, "archive prelude %v.%v", cm.Database, cm.Collection)
}
// CreateIndexes takes in an intent and an array of index documents and
// attempts to create them using the createIndexes command. If that command
// fails, we fall back to individual index creation.
func (restore *MongoRestore) CreateIndexes(intent *intents.Intent, indexes []IndexDocument) error {
// first, sanitize the indexes
for _, index := range indexes {
// update the namespace of the index before inserting
index.Options["ns"] = intent.Namespace()
// check for length violations before building the command
fullIndexName := fmt.Sprintf("%v.$%v", index.Options["ns"], index.Options["name"])
if len(fullIndexName) > 127 {
return fmt.Errorf(
"cannot restore index with namespace '%v': "+
"namespace is too long (max size is 127 bytes)", fullIndexName)
}
// remove the index version, forcing an update,
// unless we specifically want to keep it
if !restore.OutputOptions.KeepIndexVersion {
delete(index.Options, "v")
}
}
session, err := restore.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error establishing connection: %v", err)
}
session.SetSafe(&mgo.Safe{})
defer session.Close()
// then attempt the createIndexes command
rawCommand := bson.D{
{"createIndexes", intent.C},
{"indexes", indexes},
}
results := bson.M{}
err = session.DB(intent.DB).Run(rawCommand, &results)
if err == nil {
return nil
}
if err.Error() != "no such cmd: createIndexes" {
return fmt.Errorf("createIndex error: %v", err)
}
// if we're here, the connected server does not support the command, so we fall back
log.Logv(log.Info, "\tcreateIndexes command not supported, attemping legacy index insertion")
for _, idx := range indexes {
log.Logvf(log.Info, "\tmanually creating index %v", idx.Options["name"])
err = restore.LegacyInsertIndex(intent, idx)
if err != nil {
return fmt.Errorf("error creating index %v: %v", idx.Options["name"], err)
}
}
return nil
}
// CreateStdinIntentForCollection builds an intent for the given database and collection name
// that is to be read from standard input
func (restore *MongoRestore) CreateStdinIntentForCollection(db string, collection string) error {
log.Logvf(log.DebugLow, "reading collection %v for database %v from standard input",
collection, db)
intent := &intents.Intent{
DB: db,
C: collection,
Location: "-",
}
intent.BSONFile = &stdinFile{Reader: restore.stdin}
restore.manager.Put(intent)
return nil
}
// runInsertionWorker is a helper to InsertDocuments - it reads document off
// the read channel and prepares then in batches for insertion into the databas
func (imp *MongoImport) runInsertionWorker(readDocs chan bson.D) (err error) {
session, err := imp.SessionProvider.GetSession()
if err != nil {
return fmt.Errorf("error connecting to mongod: %v", err)
}
defer session.Close()
if err = imp.configureSession(session); err != nil {
return fmt.Errorf("error configuring session: %v", err)
}
collection := session.DB(imp.ToolOptions.DB).C(imp.ToolOptions.Collection)
var inserter flushInserter
if imp.IngestOptions.Mode == modeInsert {
inserter = db.NewBufferedBulkInserter(collection, imp.IngestOptions.BulkBufferSize, !imp.IngestOptions.StopOnError)
if !imp.IngestOptions.MaintainInsertionOrder {
inserter.(*db.BufferedBulkInserter).Unordered()
}
} else {
inserter = imp.newUpserter(collection)
}
readLoop:
for {
select {
case document, alive := <-readDocs:
if !alive {
break readLoop
}
err = filterIngestError(imp.IngestOptions.StopOnError, inserter.Insert(document))
if err != nil {
return err
}
atomic.AddUint64(&imp.insertionCount, 1)
case <-imp.Dying():
return nil
}
}
err = inserter.Flush()
// TOOLS-349 correct import count for bulk operations
if bulkError, ok := err.(*mgo.BulkError); ok {
failedDocs := make(map[int]bool) // index of failures
for _, failure := range bulkError.Cases() {
failedDocs[failure.Index] = true
}
numFailures := len(failedDocs)
if numFailures > 0 {
log.Logvf(log.Always, "num failures: %d", numFailures)
atomic.AddUint64(&imp.insertionCount, ^uint64(numFailures-1))
}
}
return filterIngestError(imp.IngestOptions.StopOnError, err)
}
