func TestKerberos(t *testing.T) {
testutil.VerifyTestType(t, testutil.KerberosTestType)
Convey("Should be able to run mongoexport with Kerberos auth", t, func() {
opts, err := testutil.GetKerberosOptions()
So(err, ShouldBeNil)
sessionProvider, err := db.NewSessionProvider(*opts)
So(err, ShouldBeNil)
export := MongoExport{
ToolOptions: *opts,
OutputOpts: &OutputFormatOptions{},
InputOpts: &InputOptions{},
SessionProvider: sessionProvider,
}
var out bytes.Buffer
num, err := export.exportInternal(&out)
So(err, ShouldBeNil)
So(num, ShouldEqual, 1)
outputLines := strings.Split(strings.TrimSpace(out.String()), "\n")
So(len(outputLines), ShouldEqual, 1)
outMap := map[string]interface{}{}
So(json.Unmarshal([]byte(outputLines[0]), &outMap), ShouldBeNil)
So(outMap["kerberos"], ShouldEqual, true)
So(outMap["authenticated"], ShouldEqual, "yeah")
So(outMap["_id"].(map[string]interface{})["$oid"], ShouldEqual, "528fb35afb3a8030e2f643c3")
})
}
// put in some test data into GridFS
func setUpGridFSTestData() ([]interface{}, error) {
sessionProvider, err := db.NewSessionProvider(*toolOptions)
if err != nil {
return nil, err
}
session, err := sessionProvider.GetSession()
if err != nil {
return nil, err
}
defer session.Close()
bytesExpected := []interface{}{}
gfs := session.DB(testDB).GridFS("fs")
var testfile *mgo.GridFile
for i, item := range []string{"testfile1", "testfile2", "testfile3"} {
testfile, err = gfs.Create(item)
if err != nil {
return nil, err
}
defer testfile.Close()
n, err := testfile.Write([]byte(strings.Repeat("a", (i+1)*5)))
if err != nil {
return nil, err
}
bytesExpected = append(bytesExpected, n)
}
return bytesExpected, nil
}
// Init performs preliminary setup operations for MongoDump.
func (dump *MongoDump) Init() error {
err := dump.ValidateOptions()
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
if dump.stdout == nil {
dump.stdout = os.Stdout
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
return fmt.Errorf("can't create session: %v", err)
}
// temporarily allow secondary reads for the isMongos check
dump.sessionProvider.SetReadPreference(mgo.Nearest)
dump.isMongos, err = dump.sessionProvider.IsMongos()
if err != nil {
return err
}
if dump.isMongos && dump.OutputOptions.Oplog {
return fmt.Errorf("can't use --oplog option when dumping from a mongos")
}
var mode mgo.Mode
if dump.ToolOptions.ReplicaSetName != "" || dump.isMongos {
mode = mgo.Primary
} else {
mode = mgo.Nearest
}
var tags bson.D
if dump.InputOptions.ReadPreference != "" {
mode, tags, err = db.ParseReadPreference(dump.InputOptions.ReadPreference)
if err != nil {
return fmt.Errorf("error parsing --readPreference : %v", err)
}
if len(tags) > 0 {
dump.sessionProvider.SetTags(tags)
}
}
// warn if we are trying to dump from a secondary in a sharded cluster
if dump.isMongos && mode != mgo.Primary {
log.Logf(log.Always, db.WarningNonPrimaryMongosConnection)
}
dump.sessionProvider.SetReadPreference(mode)
dump.sessionProvider.SetTags(tags)
dump.sessionProvider.SetFlags(db.DisableSocketTimeout)
// return a helpful error message for mongos --repair
if dump.OutputOptions.Repair && dump.isMongos {
return fmt.Errorf("--repair flag cannot be used on a mongos")
}
dump.manager = intents.NewIntentManager()
dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime)
return nil
}
func TestKerberos(t *testing.T) {
testutil.VerifyTestType(t, testutil.KerberosTestType)
Convey("Should be able to run mongoexport with Kerberos auth", t, func() {
opts, err := testutil.GetKerberosOptions()
So(err, ShouldBeNil)
sessionProvider, err := db.NewSessionProvider(*opts)
So(err, ShouldBeNil)
export := MongoExport{
ToolOptions: *opts,
OutputOpts: &OutputFormatOptions{},
InputOpts: &InputOptions{},
SessionProvider: sessionProvider,
}
var out bytes.Buffer
num, err := export.exportInternal(&out)
So(err, ShouldBeNil)
So(num, ShouldEqual, 1)
outputLines := strings.Split(strings.TrimSpace(out.String()), "\n")
So(len(outputLines), ShouldEqual, 1)
So(outputLines[0], ShouldEqual,
"{\"_id\":{\"$oid\":\"528fb35afb3a8030e2f643c3\"},"+
"\"authenticated\":\"yeah\",\"kerberos\":true}")
})
}
// Init performs preliminary setup operations for MongoDump.
func (dump *MongoDump) Init() error {
err := dump.ValidateOptions()
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
return fmt.Errorf("can't create session: %v", err)
}
// allow secondary reads for the isMongos check
dump.sessionProvider.SetFlags(db.Monotonic)
dump.isMongos, err = dump.sessionProvider.IsMongos()
if err != nil {
return err
}
// ensure we allow secondary reads on mongods and disable TCP timeouts
flags := db.DisableSocketTimeout
if dump.isMongos {
log.Logf(log.Info, "connecting to mongos; secondary reads disabled")
} else {
flags |= db.Monotonic
}
dump.sessionProvider.SetFlags(flags)
// return a helpful error message for mongos --repair
if dump.OutputOptions.Repair && dump.isMongos {
return fmt.Errorf("--repair flag cannot be used on a mongos")
}
dump.manager = intents.NewIntentManager()
dump.progressManager = progress.NewProgressBarManager(log.Writer(0), progressBarWaitTime)
return nil
}
func main() {
go signals.Handle()
// initialize command-line opts
opts := options.New("mongofiles", mongofiles.Usage, options.EnabledOptions{Auth: true, Connection: true, Namespace: false})
storageOpts := &mongofiles.StorageOptions{}
opts.AddOptions(storageOpts)
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'mongofiles --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
// add the specified database to the namespace options struct
opts.Namespace.DB = storageOpts.DB
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
// create a session provider to connect to the db
provider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
mf := mongofiles.MongoFiles{
ToolOptions: opts,
StorageOptions: storageOpts,
SessionProvider: provider,
}
if err := mf.ValidateCommand(args); err != nil {
log.Logf(log.Always, "%v", err)
log.Logf(log.Always, "try 'mongofiles --help' for more information")
os.Exit(util.ExitBadOptions)
}
output, err := mf.Run(true)
if err != nil {
log.Logf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
fmt.Printf("%s", output)
}
//Utility constructor for NodeMonitor that copies the same connection settings
//from an instance of ToolOptions, but for a different host name.
func NewNodeMonitor(opts commonopts.ToolOptions, fullHost string) *NodeMonitor {
optsCopy := opts
host, port := parseHostPort(fullHost)
optsCopy.Connection = &commonopts.Connection{host, port}
optsCopy.Direct = true
sessionProvider := db.NewSessionProvider(optsCopy)
return &NodeMonitor{
host: fullHost,
sessionProvider: sessionProvider,
LastStatus: nil,
LastUpdate: time.Now(),
Err: nil,
}
}
func NewMongoImport() (*MongoImport, error) {
toolOptions := getBasicToolOptions()
inputOptions := &InputOptions{}
ingestOptions := &IngestOptions{}
provider, err := db.NewSessionProvider(*toolOptions)
if err != nil {
return nil, err
}
return &MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: provider,
}, nil
}
func simpleMongoFilesInstance(args []string) (*MongoFiles, error) {
sessionProvider, err := db.NewSessionProvider(*toolOptions)
if err != nil {
return nil, err
}
mongofiles := MongoFiles{
ToolOptions: toolOptions,
StorageOptions: &StorageOptions{GridFSPrefix: "fs", DB: testDB},
SessionProvider: sessionProvider,
Command: args[0],
FileName: args[1],
}
return &mongofiles, nil
}
// remove test data from GridFS
func tearDownGridFSTestData() error {
sessionProvider, err := db.NewSessionProvider(*toolOptions)
if err != nil {
return err
}
session, err := sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
if err = session.DB(testDB).DropDatabase(); err != nil {
return err
}
return nil
}
// NewNodeMonitor copies the same connection settings from an instance of
// ToolOptions, but monitors fullHost.
func NewNodeMonitor(opts options.ToolOptions, fullHost string, all bool) (*NodeMonitor, error) {
optsCopy := opts
host, port := parseHostPort(fullHost)
optsCopy.Connection = &options.Connection{Host: host, Port: port}
optsCopy.Direct = true
sessionProvider, err := db.NewSessionProvider(optsCopy)
if err != nil {
return nil, err
}
return &NodeMonitor{
host: fullHost,
sessionProvider: sessionProvider,
LastStatus: nil,
LastUpdate: time.Now(),
All: all,
Err: nil,
}, nil
}
func getBareSession() (*mgo.Session, error) {
ssl := testutil.GetSSLOptions()
auth := testutil.GetAuthOptions()
sessionProvider, err := db.NewSessionProvider(options.ToolOptions{
Connection: &options.Connection{
Host: testServer,
Port: testPort,
},
Auth: &auth,
SSL: &ssl,
})
if err != nil {
return nil, err
}
session, err := sessionProvider.GetSession()
if err != nil {
return nil, err
}
return session, nil
}
func main() {
go signals.Handle()
// initialize command line options
opts := options.New("mongooplog", mongooplog.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: false})
// add the mongooplog-specific options
sourceOpts := &mongooplog.SourceOptions{}
opts.AddOptions(sourceOpts)
log.Logf(log.Always, "warning: mongooplog is deprecated, and will be removed completely in a future release")
// parse the command line options
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'mongooplog --help' for more information")
os.Exit(util.ExitBadOptions)
}
if len(args) != 0 {
log.Logf(log.Always, "positional arguments not allowed: %v", args)
log.Logf(log.Always, "try 'mongooplog --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// init logger
log.SetVerbosity(opts.Verbosity)
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
// validate the mongooplog options
if sourceOpts.From == "" {
log.Logf(log.Always, "command line error: need to specify --from")
os.Exit(util.ExitBadOptions)
}
// create a session provider for the destination server
sessionProviderTo, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to destination host: %v", err)
os.Exit(util.ExitError)
}
// create a session provider for the source server
opts.Connection.Host = sourceOpts.From
opts.Connection.Port = ""
sessionProviderFrom, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to source host: %v", err)
os.Exit(util.ExitError)
}
// initialize mongooplog
oplog := mongooplog.MongoOplog{
ToolOptions: opts,
SourceOptions: sourceOpts,
SessionProviderFrom: sessionProviderFrom,
SessionProviderTo: sessionProviderTo,
}
// kick it off
if err := oplog.Run(); err != nil {
log.Logf(log.Always, "error: %v", err)
os.Exit(util.ExitError)
}
}
func main() {
// initialize command-line opts
opts := options.New("mongoexport", mongoexport.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: true})
outputOpts := &mongoexport.OutputFormatOptions{}
opts.AddOptions(outputOpts)
inputOpts := &mongoexport.InputOptions{}
opts.AddOptions(inputOpts)
args, err := opts.Parse()
if err != nil {
log.Logvf(log.Always, "error parsing command line options: %v", err)
log.Logvf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
if len(args) != 0 {
log.Logvf(log.Always, "too many positional arguments: %v", args)
log.Logvf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
log.SetVerbosity(opts.Verbosity)
signals.Handle()
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
provider, err := db.NewSessionProvider(*opts)
// temporarily allow secondary reads for the isMongos check
provider.SetReadPreference(mgo.Nearest)
isMongos, err := provider.IsMongos()
if err != nil {
log.Logvf(log.Always, "%v", err)
os.Exit(util.ExitError)
}
provider.SetFlags(db.DisableSocketTimeout)
if inputOpts.SlaveOk {
if inputOpts.ReadPreference != "" {
log.Logvf(log.Always, "--slaveOk can't be specified when --readPreference is specified")
os.Exit(util.ExitBadOptions)
}
log.Logvf(log.Always, "--slaveOk is deprecated and being internally rewritten as --readPreference=nearest")
inputOpts.ReadPreference = "nearest"
}
var mode mgo.Mode
if opts.ReplicaSetName != "" || isMongos {
mode = mgo.Primary
} else {
mode = mgo.Nearest
}
var tags bson.D
if inputOpts.ReadPreference != "" {
mode, tags, err = db.ParseReadPreference(inputOpts.ReadPreference)
if err != nil {
log.Logvf(log.Always, "error parsing --ReadPreference: %v", err)
os.Exit(util.ExitBadOptions)
}
if len(tags) > 0 {
provider.SetTags(tags)
}
}
// warn if we are trying to export from a secondary in a sharded cluster
if isMongos && mode != mgo.Primary {
log.Logvf(log.Always, db.WarningNonPrimaryMongosConnection)
}
provider.SetReadPreference(mode)
if err != nil {
log.Logvf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
exporter := mongoexport.MongoExport{
ToolOptions: *opts,
OutputOpts: outputOpts,
InputOpts: inputOpts,
SessionProvider: provider,
}
err = exporter.ValidateSettings()
if err != nil {
log.Logvf(log.Always, "error validating settings: %v", err)
log.Logvf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
writer, err := exporter.GetOutputWriter()
if err != nil {
log.Logvf(log.Always, "error opening output stream: %v", err)
os.Exit(util.ExitError)
}
if writer == nil {
writer = os.Stdout
} else {
defer writer.Close()
}
numDocs, err := exporter.Export(writer)
if err != nil {
log.Logvf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
if numDocs == 1 {
log.Logvf(log.Always, "exported %v record", numDocs)
} else {
log.Logvf(log.Always, "exported %v records", numDocs)
}
}
func TestCollectionExists(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
Convey("With a test mongorestore", t, func() {
ssl := testutil.GetSSLOptions()
auth := testutil.GetAuthOptions()
sessionProvider, err := db.NewSessionProvider(commonOpts.ToolOptions{
Connection: &commonOpts.Connection{
Host: "localhost",
Port: db.DefaultTestPort,
},
Auth: &auth,
SSL: &ssl,
})
So(err, ShouldBeNil)
restore := &MongoRestore{
SessionProvider: sessionProvider,
}
Convey("and some test data in a server", func() {
session, err := restore.SessionProvider.GetSession()
So(err, ShouldBeNil)
So(session.DB(ExistsDB).C("one").Insert(bson.M{}), ShouldBeNil)
So(session.DB(ExistsDB).C("two").Insert(bson.M{}), ShouldBeNil)
So(session.DB(ExistsDB).C("three").Insert(bson.M{}), ShouldBeNil)
Convey("collections that exist should return true", func() {
exists, err := restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "one"})
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
exists, err = restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "two"})
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
exists, err = restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "three"})
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
Convey("and those that do not exist should return false", func() {
exists, err = restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "four"})
So(err, ShouldBeNil)
So(exists, ShouldBeFalse)
})
})
Reset(func() {
session.DB(ExistsDB).DropDatabase()
session.Close()
})
})
Convey("and a fake cache should be used instead of the server when it exists", func() {
restore.knownCollections = map[string][]string{
ExistsDB: []string{"cats", "dogs", "snakes"},
}
exists, err := restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "dogs"})
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
exists, err = restore.CollectionExists(&intents.Intent{DB: ExistsDB, C: "two"})
So(err, ShouldBeNil)
So(exists, ShouldBeFalse)
})
})
}
func TestMongorestore(t *testing.T) {
ssl := testutil.GetSSLOptions()
auth := testutil.GetAuthOptions()
testutil.VerifyTestType(t, testutil.IntegrationTestType)
toolOptions := &options.ToolOptions{
Connection: &options.Connection{
Host: testServer,
Port: testPort,
},
Auth: &auth,
SSL: &ssl,
}
inputOptions := &InputOptions{}
outputOptions := &OutputOptions{
NumParallelCollections: 1,
NumInsertionWorkers: 1,
WriteConcern: "majority",
}
nsOptions := &NSOptions{}
Convey("With a test MongoRestore", t, func() {
provider, err := db.NewSessionProvider(*toolOptions)
if err != nil {
log.Logvf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
restore := MongoRestore{
ToolOptions: toolOptions,
OutputOptions: outputOptions,
InputOptions: inputOptions,
NSOptions: nsOptions,
SessionProvider: provider,
}
session, _ := provider.GetSession()
c1 := session.DB("db1").C("c1")
c1.DropCollection()
Convey("and an explicit target restores from that dump directory", func() {
restore.TargetDirectory = "testdata/testdirs"
err = restore.Restore()
So(err, ShouldBeNil)
count, err := c1.Count()
So(err, ShouldBeNil)
So(count, ShouldEqual, 100)
})
Convey("and an target of '-' restores from standard input", func() {
bsonFile, err := os.Open("testdata/testdirs/db1/c1.bson")
restore.NSOptions.Collection = "c1"
restore.NSOptions.DB = "db1"
So(err, ShouldBeNil)
restore.stdin = bsonFile
restore.TargetDirectory = "-"
err = restore.Restore()
So(err, ShouldBeNil)
count, err := c1.Count()
So(err, ShouldBeNil)
So(count, ShouldEqual, 100)
})
})
}
func main() {
// initialize command-line opts
opts := options.New("mongorestore", mongorestore.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: true})
inputOpts := &mongorestore.InputOptions{}
opts.AddOptions(inputOpts)
outputOpts := &mongorestore.OutputOptions{}
opts.AddOptions(outputOpts)
extraArgs, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'mongorestore --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help or version info, if specified
if opts.PrintHelp(false) {
return
}
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
targetDir, err := getTargetDirFromArgs(extraArgs, inputOpts.Directory)
if err != nil {
log.Logf(log.Always, "%v", err)
log.Logf(log.Always, "try 'mongorestore --help' for more information")
os.Exit(util.ExitBadOptions)
}
targetDir = util.ToUniversalPath(targetDir)
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
provider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
provider.SetBypassDocumentValidation(outputOpts.BypassDocumentValidation)
// disable TCP timeouts for restore jobs
provider.SetFlags(db.DisableSocketTimeout)
restore := mongorestore.MongoRestore{
ToolOptions: opts,
OutputOptions: outputOpts,
InputOptions: inputOpts,
TargetDirectory: targetDir,
SessionProvider: provider,
}
if err = restore.Restore(); err != nil {
log.Logf(log.Always, "Failed: %v", err)
if err == util.ErrTerminated {
os.Exit(util.ExitKill)
}
os.Exit(util.ExitError)
}
}
func main() {
// initialize command-line opts
opts := options.New("mongorestore", mongorestore.Usage,
options.EnabledOptions{Auth: true, Connection: true})
nsOpts := &mongorestore.NSOptions{}
opts.AddOptions(nsOpts)
inputOpts := &mongorestore.InputOptions{}
opts.AddOptions(inputOpts)
outputOpts := &mongorestore.OutputOptions{}
opts.AddOptions(outputOpts)
extraArgs, err := opts.Parse()
if err != nil {
log.Logvf(log.Always, "error parsing command line options: %v", err)
log.Logvf(log.Always, "try 'mongorestore --help' for more information")
os.Exit(util.ExitBadOptions)
}
// Allow the db connector to fall back onto the current database when no
// auth database is given; the standard -d/-c options go into nsOpts now
opts.Namespace = &options.Namespace{DB: nsOpts.DB}
// print help or version info, if specified
if opts.PrintHelp(false) {
return
}
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
targetDir, err := getTargetDirFromArgs(extraArgs, inputOpts.Directory)
if err != nil {
log.Logvf(log.Always, "%v", err)
log.Logvf(log.Always, "try 'mongorestore --help' for more information")
os.Exit(util.ExitBadOptions)
}
targetDir = util.ToUniversalPath(targetDir)
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
provider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logvf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
provider.SetBypassDocumentValidation(outputOpts.BypassDocumentValidation)
// disable TCP timeouts for restore jobs
provider.SetFlags(db.DisableSocketTimeout)
restore := mongorestore.MongoRestore{
ToolOptions: opts,
OutputOptions: outputOpts,
InputOptions: inputOpts,
NSOptions: nsOpts,
TargetDirectory: targetDir,
SessionProvider: provider,
}
finishedChan := signals.HandleWithInterrupt(restore.HandleInterrupt)
defer close(finishedChan)
if err = restore.Restore(); err != nil {
log.Logvf(log.Always, "Failed: %v", err)
if err == util.ErrTerminated {
os.Exit(util.ExitKill)
}
os.Exit(util.ExitError)
}
}
func main() {
// initialize command-line opts
opts := options.New("mongotop", mongotop.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: false})
opts.UseReadOnlyHostDescription()
// add mongotop-specific options
outputOpts := &mongotop.Output{}
opts.AddOptions(outputOpts)
args, err := opts.Parse()
if err != nil {
log.Logvf(log.Always, "error parsing command line options: %v", err)
log.Logvf(log.Always, "try 'mongotop --help' for more information")
os.Exit(util.ExitBadOptions)
}
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
log.SetVerbosity(opts.Verbosity)
signals.Handle()
if len(args) > 1 {
log.Logvf(log.Always, "too many positional arguments")
log.Logvf(log.Always, "try 'mongotop --help' for more information")
os.Exit(util.ExitBadOptions)
}
sleeptime := 1 // default to 1 second sleep time
if len(args) > 0 {
sleeptime, err = strconv.Atoi(args[0])
if err != nil || sleeptime <= 0 {
log.Logvf(log.Always, "invalid sleep time: %v", args[0])
os.Exit(util.ExitBadOptions)
}
}
if outputOpts.RowCount < 0 {
log.Logvf(log.Always, "invalid value for --rowcount: %v", outputOpts.RowCount)
os.Exit(util.ExitBadOptions)
}
if opts.Auth.Username != "" && opts.Auth.Source == "" && !opts.Auth.RequiresExternalDB() {
log.Logvf(log.Always, "--authenticationDatabase is required when authenticating against a non $external database")
os.Exit(util.ExitBadOptions)
}
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
// create a session provider to connect to the db
sessionProvider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logvf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
if setName == "" {
sessionProvider.SetReadPreference(mgo.PrimaryPreferred)
}
// fail fast if connecting to a mongos
isMongos, err := sessionProvider.IsMongos()
if err != nil {
log.Logvf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
if isMongos {
log.Logvf(log.Always, "cannot run mongotop against a mongos")
os.Exit(util.ExitError)
}
// instantiate a mongotop instance
top := &mongotop.MongoTop{
Options: opts,
OutputOptions: outputOpts,
SessionProvider: sessionProvider,
Sleeptime: time.Duration(sleeptime) * time.Second,
}
// kick it off
if err := top.Run(); err != nil {
log.Logvf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
}
func TestBasicOps(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
var opts *options.ToolOptions
var sourceOpts *SourceOptions
Convey("When replicating operations", t, func() {
ssl := testutil.GetSSLOptions()
auth := testutil.GetAuthOptions()
// specify localhost:33333 as the destination host
opts = &options.ToolOptions{
Namespace: &options.Namespace{},
SSL: &ssl,
Auth: &auth,
Kerberos: &options.Kerberos{},
Connection: &options.Connection{
Host: "localhost",
Port: db.DefaultTestPort,
},
}
// specify localhost:33333 as the source host
sourceOpts = &SourceOptions{
Seconds: 84600, // the default
OplogNS: "local.oplog.rs", // the default
}
Convey("all operations should be applied correctly, without"+
" error", func() {
// set the "oplog" we will use
sourceOpts.OplogNS = "mongooplog_test.oplog"
// initialize a session provider for the source
sourceSP, err := db.NewSessionProvider(*opts)
So(err, ShouldBeNil)
// initialize a session provider for the destination
destSP, err := db.NewSessionProvider(*opts)
So(err, ShouldBeNil)
// clear out the oplog
sess, err := sourceSP.GetSession()
So(err, ShouldBeNil)
defer sess.Close()
oplogColl := sess.DB("mongooplog_test").C("oplog")
oplogColl.DropCollection()
// create the oplog as a capped collection, so it can be tailed
So(sess.DB("mongooplog_test").Run(
bson.D{{"create", "oplog"}, {"capped", true},
{"size", 10000}},
bson.M{}),
ShouldBeNil)
// create the collection we are testing against (ignore errors)
sess.DB("mongooplog_test").C("data").Create(&mgo.CollectionInfo{})
// clear out the collection we'll use for testing
testColl := sess.DB("mongooplog_test").C("data")
_, err = testColl.RemoveAll(bson.M{})
So(err, ShouldBeNil)
// insert some "ops" into the oplog to be found and applied
op1 := &db.Oplog{
Timestamp: bson.MongoTimestamp(1<<63 - 1), // years in the future
HistoryID: 100,
Version: 2,
Operation: "i",
Namespace: "mongooplog_test.data",
Object: bson.D{
bson.DocElem{
"_id", 3,
},
},
}
So(oplogColl.Insert(op1), ShouldBeNil)
op2 := &db.Oplog{
Timestamp: bson.MongoTimestamp(1<<63 - 1), // years in the future
HistoryID: 200,
Version: 2,
Operation: "i",
Namespace: "mongooplog_test.data",
Object: bson.D{
bson.DocElem{
"_id", 4,
},
},
}
So(oplogColl.Insert(op2), ShouldBeNil)
// this one should be filtered out, since it occurred before the
// threshold
op3 := &db.Oplog{
Timestamp: bson.MongoTimestamp(1<<62 - 1), // more than 1 day in the past
HistoryID: 300,
Version: 2,
Operation: "i",
Namespace: "mongooplog_test.data",
Object: bson.D{
bson.DocElem{
"_id", 3,
},
},
}
So(oplogColl.Insert(op3), ShouldBeNil)
// initialize the mongooplog
oplog := MongoOplog{
ToolOptions: opts,
SourceOptions: sourceOpts,
SessionProviderFrom: sourceSP,
SessionProviderTo: destSP,
}
// run it
So(oplog.Run(), ShouldBeNil)
// the operations should have been applied
var inserted []bson.M
So(testColl.Find(bson.M{}).Sort("_id").All(&inserted),
ShouldBeNil)
So(len(inserted), ShouldEqual, 2)
So(inserted[0]["_id"], ShouldEqual, 3)
So(inserted[1]["_id"], ShouldEqual, 4)
})
})
}
func main() {
go signals.Handle()
// initialize command-line opts
opts := options.New("mongoexport", mongoexport.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: true})
outputOpts := &mongoexport.OutputFormatOptions{}
opts.AddOptions(outputOpts)
inputOpts := &mongoexport.InputOptions{}
opts.AddOptions(inputOpts)
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
if len(args) != 0 {
log.Logf(log.Always, "too many positional arguments: %v", args)
log.Logf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
log.SetVerbosity(opts.Verbosity)
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
provider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
exporter := mongoexport.MongoExport{
ToolOptions: *opts,
OutputOpts: outputOpts,
InputOpts: inputOpts,
SessionProvider: provider,
}
err = exporter.ValidateSettings()
if err != nil {
log.Logf(log.Always, "error validating settings: %v", err)
log.Logf(log.Always, "try 'mongoexport --help' for more information")
os.Exit(util.ExitBadOptions)
}
writer, err := exporter.GetOutputWriter()
if err != nil {
log.Logf(log.Always, "error opening output stream: %v", err)
os.Exit(util.ExitError)
}
if writer == nil {
writer = os.Stdout
} else {
defer writer.Close()
}
numDocs, err := exporter.Export(writer)
if err != nil {
log.Logf(log.Always, "Failed: %v", err)
os.Exit(util.ExitError)
}
if numDocs == 1 {
log.Logf(log.Always, "exported %v record", numDocs)
} else {
log.Logf(log.Always, "exported %v records", numDocs)
}
}
func TestImportDocuments(t *testing.T) {
testutil.VerifyTestType(t, testutil.IntegrationTestType)
Convey("With a mongoimport instance", t, func() {
Reset(func() {
sessionProvider, err := db.NewSessionProvider(*getBasicToolOptions())
if err != nil {
t.Fatalf("error getting session provider session: %v", err)
}
session, err := sessionProvider.GetSession()
if err != nil {
t.Fatalf("error getting session: %v", err)
}
defer session.Close()
session.DB(testDb).C(testCollection).DropCollection()
})
Convey("no error should be thrown for CSV import on test data and all "+
"CSV data lines should be imported correctly", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "a,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.WriteConcern = "majority"
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
})
Convey("an error should be thrown for JSON import on test data that is "+
"JSON array", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.File = "testdata/test_array.json"
imp.IngestOptions.WriteConcern = "majority"
numImported, err := imp.ImportDocuments()
So(err, ShouldNotBeNil)
So(numImported, ShouldEqual, 0)
})
Convey("TOOLS-247: no error should be thrown for JSON import on test "+
"data and all documents should be imported correctly", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.File = "testdata/test_plain2.json"
imp.IngestOptions.WriteConcern = "majority"
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 10)
})
Convey("CSV import with --ignoreBlanks should import only non-blank fields", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_blanks.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.IgnoreBlanks = true
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2},
bson.M{"_id": 5, "c": "6e"},
bson.M{"_id": 7, "b": 8, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("CSV import without --ignoreBlanks should include blanks", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_blanks.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": ""},
bson.M{"_id": 5, "b": "", "c": "6e"},
bson.M{"_id": 7, "b": 8, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with --upsertFields", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.UpsertFields = "b,c"
imp.IngestOptions.MaintainInsertionOrder = true
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--stopOnError. Only documents before error should be imported", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.StopOnError = true
imp.IngestOptions.MaintainInsertionOrder = true
imp.IngestOptions.WriteConcern = "majority"
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("CSV import with duplicate _id's should not error if --stopOnError is not set", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_duplicate.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.StopOnError = false
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 5)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
bson.M{"_id": 8, "b": 6, "c": 6},
}
// all docs except the one with duplicate _id - should be imported
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with --drop", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.Drop = true
imp.IngestOptions.MaintainInsertionOrder = true
imp.IngestOptions.WriteConcern = "majority"
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("CSV import on test data with --headerLine should succeed", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.InputOptions.HeaderLine = true
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 2)
})
Convey("EOF should be thrown for CSV import with --headerLine if file is empty", func() {
csvFile, err := ioutil.TempFile("", "mongoimport_")
So(err, ShouldBeNil)
csvFile.Close()
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = csvFile.Name()
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.InputOptions.HeaderLine = true
numImported, err := imp.ImportDocuments()
So(err, ShouldEqual, io.EOF)
So(numImported, ShouldEqual, 0)
})
Convey("CSV import with --upsert and --upsertFields should succeed", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test.csv"
fields := "_id,c,b"
imp.InputOptions.Fields = &fields
imp.IngestOptions.UpsertFields = "_id"
imp.IngestOptions.MaintainInsertionOrder = true
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "c": 2, "b": 3},
bson.M{"_id": 3, "c": 5.4, "b": "string"},
bson.M{"_id": 5, "c": 6, "b": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("CSV import with --upsert/--upsertFields with duplicate id should succeed "+
"if stopOnError is not set", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_duplicate.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.Upsert = true
imp.upsertFields = []string{"_id"}
numImported, err := imp.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 5)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 9},
bson.M{"_id": 8, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("an error should be thrown for CSV import on test data with "+
"duplicate _id if --stopOnError is set", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_duplicate.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.StopOnError = true
imp.IngestOptions.WriteConcern = "1"
imp.IngestOptions.MaintainInsertionOrder = true
_, err = imp.ImportDocuments()
So(err, ShouldNotBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(*imp.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("an error should be thrown for JSON import on test data that "+
"is a JSON array without passing --jsonArray", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.File = "testdata/test_array.json"
imp.IngestOptions.WriteConcern = "1"
_, err = imp.ImportDocuments()
So(err, ShouldNotBeNil)
})
Convey("an error should be thrown if a plain JSON file is supplied", func() {
fileHandle, err := os.Open("testdata/test_plain.json")
So(err, ShouldBeNil)
jsonInputReader := NewJSONInputReader(true, fileHandle, 1)
docChan := make(chan bson.D, 1)
So(jsonInputReader.StreamDocument(true, docChan), ShouldNotBeNil)
})
Convey("an error should be thrown for invalid CSV import on test data", func() {
imp, err := NewMongoImport()
So(err, ShouldBeNil)
imp.InputOptions.Type = CSV
imp.InputOptions.File = "testdata/test_bad.csv"
fields := "_id,b,c"
imp.InputOptions.Fields = &fields
imp.IngestOptions.StopOnError = true
imp.IngestOptions.WriteConcern = "1"
imp.IngestOptions.MaintainInsertionOrder = true
_, err = imp.ImportDocuments()
So(err, ShouldNotBeNil)
})
})
}
func main() {
go signals.Handle()
// initialize command-line opts
opts := options.New("mongoimport", mongoimport.Usage,
options.EnabledOptions{Auth: true, Connection: true, Namespace: true})
inputOpts := &mongoimport.InputOptions{}
opts.AddOptions(inputOpts)
ingestOpts := &mongoimport.IngestOptions{}
opts.AddOptions(ingestOpts)
args, err := opts.Parse()
if err != nil {
log.Logf(log.Always, "error parsing command line options: %v", err)
log.Logf(log.Always, "try 'mongoimport --help' for more information")
os.Exit(util.ExitBadOptions)
}
log.SetVerbosity(opts.Verbosity)
// print help, if specified
if opts.PrintHelp(false) {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// connect directly, unless a replica set name is explicitly specified
_, setName := util.ParseConnectionString(opts.Host)
opts.Direct = (setName == "")
opts.ReplicaSetName = setName
// create a session provider to connect to the db
sessionProvider, err := db.NewSessionProvider(*opts)
if err != nil {
log.Logf(log.Always, "error connecting to host: %v", err)
os.Exit(util.ExitError)
}
m := mongoimport.MongoImport{
ToolOptions: opts,
InputOptions: inputOpts,
IngestOptions: ingestOpts,
SessionProvider: sessionProvider,
}
if err = m.ValidateSettings(args); err != nil {
log.Logf(log.Always, "error validating settings: %v", err)
log.Logf(log.Always, "try 'mongoimport --help' for more information")
os.Exit(util.ExitError)
}
numDocs, err := m.ImportDocuments()
if !opts.Quiet {
if err != nil {
log.Logf(log.Always, "Failed: %v", err)
}
message := fmt.Sprintf("imported 1 document")
if numDocs != 1 {
message = fmt.Sprintf("imported %v documents", numDocs)
}
log.Logf(log.Always, message)
}
if err != nil {
os.Exit(util.ExitError)
}
}