func main() {
// initialize command-line opts
usageStr := " --host myhost --db my_cms --collection docs < mydocfile." +
"json \n\nImport CSV, TSV or JSON data into MongoDB.\n\nWhen importing " +
"JSON documents, each document must be a separate line of the input file."
opts := commonopts.New("mongoimport", "0.0.1", usageStr)
inputOpts := &options.InputOptions{}
opts.AddOptions(inputOpts)
ingestOpts := &options.IngestOptions{}
opts.AddOptions(ingestOpts)
_, err := opts.Parse()
if err != nil {
util.Panicf("error parsing command line options: %v", err)
}
// print help, if specified
if opts.PrintHelp() {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// create a session provider to connect to the db
sessionProvider, err := db.InitSessionProvider(*opts)
if err != nil {
util.Panicf("error initializing database session: %v", err)
}
importer := mongoimport.MongoImport{
ToolOptions: opts,
InputOptions: inputOpts,
IngestOptions: ingestOpts,
SessionProvider: sessionProvider,
}
if err = importer.ValidateSettings(); err != nil {
util.PrintfTimeStamped("Error validating settings: %v\n", err)
os.Exit(1)
}
numDocs, err := importer.ImportDocuments()
if !opts.Quiet {
message := fmt.Sprintf("imported 1 object\n")
if numDocs != 1 {
message = fmt.Sprintf("imported %v objects\n", numDocs)
}
util.PrintfTimeStamped(message)
}
if err != nil {
fmt.Fprintf(os.Stderr, "Error importing documents: %v\n", err)
os.Exit(1)
}
}
func main() {
// initialize command-line opts
opts := commonopts.New("mongotop", "0.0.1", "<options> <sleeptime>")
// add mongotop-specific options
outputOpts := &options.Output{}
opts.AddOptions(outputOpts)
extra, err := opts.Parse()
if err != nil {
util.Panicf("error parsing command line options: %v", err)
}
// print help, if specified
if opts.PrintHelp() {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// pull out the sleeptime
// TODO: validate args length
sleeptime := DEFAULT_SLEEP_TIME
if len(extra) > 0 {
sleeptime, err = strconv.Atoi(extra[0])
if err != nil {
util.Panicf("bad sleep time: %v", extra[0])
}
}
// create a session provider to connect to the db
sessionProvider, err := db.InitSessionProvider(*opts)
if err != nil {
util.Panicf("error initializing database session: %v", err)
}
// instantiate a mongotop instance
top := &mongotop.MongoTop{
Options: opts,
OutputOptions: outputOpts,
Outputter: &output.TerminalOutputter{},
SessionProvider: sessionProvider,
Sleeptime: time.Duration(sleeptime) * time.Second,
Once: outputOpts.Once,
}
// kick it off
if err := top.Run(); err != nil {
util.Panicf("error running mongotop: %v", err)
}
}
func main() {
// initialize command-line opts
opts := commonopts.New("mongodump", "0.0.1", "<options>")
inputOpts := &options.InputOptions{}
opts.AddOptions(inputOpts)
outputOpts := &options.OutputOptions{}
opts.AddOptions(outputOpts)
_, err := opts.Parse()
if err != nil {
fmt.Printf("error parsing command line options: %v\n\n", err)
fmt.Printf("try 'mongodump --help' for more information\n")
os.Exit(2)
}
// print help, if specified
if opts.PrintHelp() {
return
}
// print version, if specified
if opts.PrintVersion() {
return
}
// init logger
log.InitToolLogger(opts.Verbosity)
// create a session provider to connect to the db
sessionProvider, err := db.InitSessionProvider(opts)
if err != nil {
fmt.Printf("error initializing database session: %v\n", err)
os.Exit(1) //TODO copy legacy exit code
}
dump := mongodump.MongoDump{
ToolOptions: opts,
OutputOptions: outputOpts,
InputOptions: inputOpts,
SessionProvider: sessionProvider,
}
err = dump.Dump()
if err != nil {
util.Exitf(1, "%v", err)
}
}
func getDocSource(exp MongoExport) (db.DocSource, error) {
if exp.ToolOptions.Namespace.DBPath != "" {
shimPath, err := db.LocateShim()
if err != nil {
return nil, err
}
bsonTool := db.StorageShim{
DBPath: exp.ToolOptions.Namespace.DBPath,
Database: exp.ToolOptions.Namespace.DB,
Collection: exp.ToolOptions.Namespace.Collection,
Query: exp.InputOpts.Query,
Skip: exp.InputOpts.Skip,
Limit: exp.InputOpts.Limit,
ShimPath: shimPath,
}
iter, _, err := bsonTool.Open()
if err != nil {
return nil, err
}
return db.NewDecodedBSONSource(iter), nil
}
sessionProvider, err := db.InitSessionProvider(exp.ToolOptions)
if err != nil {
return nil, err
}
session, err := sessionProvider.GetSession()
if err != nil {
return nil, err
}
collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)
query := map[string]interface{}{}
if exp.InputOpts != nil && exp.InputOpts.Query != "" {
var err error
query, err = getObjectFromArg(exp.InputOpts.Query)
if err != nil {
return nil, err
}
}
q := collection.Find(query)
if exp.InputOpts != nil && exp.InputOpts.Skip > 0 {
q = q.Skip(exp.InputOpts.Skip)
}
if exp.InputOpts != nil && exp.InputOpts.Limit > 0 {
q = q.Limit(exp.InputOpts.Limit)
}
if exp.InputOpts != nil && exp.InputOpts.Sort != "" {
sortD, err := getSortFromArg(exp.InputOpts.Sort)
if err != nil {
return nil, err
}
sortFields, err := bsonutil.MakeSortString(sortD)
if err != nil {
return nil, err
}
q = q.Sort(sortFields...)
}
if len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.ForceTableScan != true && exp.InputOpts.Sort == "" {
q = q.Snapshot()
}
cursor := q.Iter()
return &db.CursorDocSource{cursor, session}, nil
}
func TestImportDocuments(t *testing.T) {
Convey("Given a mongoimport instance with which to import documents, on "+
"calling importDocuments", t, func() {
Convey("no error should be thrown for CSV import on test data and all "+
"CSV data lines should be imported correctly", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
Fields: "a,b,c",
}
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--ignoreBlanks only fields without blanks should be imported",
func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_blanks.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
IgnoreBlanks: true,
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2},
bson.M{"_id": 5, "c": "6e"},
bson.M{"_id": 7, "b": 8, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data without "+
"--ignoreBlanks supplied - fields with blanks should be imported",
func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_blanks.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": ""},
bson.M{"_id": 5, "b": "", "c": "6e"},
bson.M{"_id": 7, "b": 8, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--upsert", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
Upsert: true,
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--stopOnError. Only documents before error should be imported",
func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
StopOnError: true,
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"duplicate _id if --stopOnError is not set", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_duplicate.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 4)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
bson.M{"_id": 8, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--drop", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
Drop: true,
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--headerLine", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
HeaderLine: true,
}
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 2)
So(err, ShouldBeNil)
})
Convey("EOF should be thrown for CSV import on test data with "+
"--headerLine if the input file is empty", func() {
toolOptions := getBasicToolOptions()
csvFile, err := ioutil.TempFile("", "mongoimport_")
So(err, ShouldBeNil)
csvFile.Close()
inputOptions := &options.InputOptions{
Type: CSV,
File: csvFile.Name(),
HeaderLine: true,
}
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 0)
So(err, ShouldEqual, io.EOF)
})
Convey("no error should be thrown for CSV import on test data with "+
"--upsert and --upsertFields", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
Upsert: true,
UpsertFields: "_id",
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
"--upsert and --upsertFields and duplicate _id if --stopOnError "+
"is not set", func() {
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_duplicate.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
Upsert: true,
UpsertFields: "_id",
}
toolOptions := getBasicToolOptions()
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 5)
So(err, ShouldBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 9},
bson.M{"_id": 8, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("an error should be thrown for CSV import on test data with "+
"duplicate _id if --stopOnError is set", func() {
toolOptions := getBasicToolOptions()
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_duplicate.csv",
Fields: "_id,b,c",
}
ingestOptions := &options.IngestOptions{
StopOnError: true,
}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 3)
So(err, ShouldNotBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Convey("an error should be thrown for invalid CSV import on test data",
func() {
inputOptions := &options.InputOptions{
Type: CSV,
File: "testdata/test_bad.csv",
Fields: "_id,b,c",
}
toolOptions := getBasicToolOptions()
ingestOptions := &options.IngestOptions{}
sessionProvider, err := db.InitSessionProvider(*toolOptions)
So(err, ShouldBeNil)
mongoImport := MongoImport{
ToolOptions: toolOptions,
InputOptions: inputOptions,
IngestOptions: ingestOptions,
SessionProvider: sessionProvider,
}
numImported, err := mongoImport.ImportDocuments()
So(numImported, ShouldEqual, 1)
So(err, ShouldNotBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
}
So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
})
Reset(func() {
session, err := sessionProvider.GetSession()
if err != nil {
t.Fatalf("Error doing cleanup: %v", err)
}
defer session.Close()
session.DB(testDB).C(testCollection).DropCollection()
})
})
}
}
namespace = &commonOpts.Namespace{
DB: testDB,
Collection: testCollection,
}
connection = &commonOpts.Connection{
Host: testServer,
Port: testPort,
}
toolOptions = &commonOpts.ToolOptions{
SSL: ssl,
Namespace: namespace,
Connection: connection,
Auth: &commonOpts.Auth{},
}
sessionProvider, _ = db.InitSessionProvider(*toolOptions)
)
func TestMongoImportValidateSettings(t *testing.T) {
Convey("Given a mongoimport instance for validation, ", t, func() {
Convey("an error should be thrown if no database is given", func() {
namespace := &commonOpts.Namespace{}
toolOptions := &commonOpts.ToolOptions{
Namespace: namespace,
}
inputOptions := &options.InputOptions{
Type: CSV,
}
ingestOptions := &options.IngestOptions{}
mongoImport := MongoImport{
ToolOptions: toolOptions,