Пример #1
0
func main() {
	// initialize command-line opts
	usageStr := " --host myhost --db my_cms --collection docs < mydocfile." +
		"json \n\nImport CSV, TSV or JSON data into MongoDB.\n\nWhen importing " +
		"JSON documents, each document must be a separate line of the input file."
	opts := commonopts.New("mongoimport", "0.0.1", usageStr)

	inputOpts := &options.InputOptions{}
	opts.AddOptions(inputOpts)
	ingestOpts := &options.IngestOptions{}
	opts.AddOptions(ingestOpts)

	_, err := opts.Parse()
	if err != nil {
		util.Panicf("error parsing command line options: %v", err)
	}

	// print help, if specified
	if opts.PrintHelp() {
		return
	}

	// print version, if specified
	if opts.PrintVersion() {
		return
	}

	// create a session provider to connect to the db
	sessionProvider, err := db.InitSessionProvider(*opts)
	if err != nil {
		util.Panicf("error initializing database session: %v", err)
	}

	importer := mongoimport.MongoImport{
		ToolOptions:     opts,
		InputOptions:    inputOpts,
		IngestOptions:   ingestOpts,
		SessionProvider: sessionProvider,
	}

	if err = importer.ValidateSettings(); err != nil {
		util.PrintfTimeStamped("Error validating settings: %v\n", err)
		os.Exit(1)
	}

	numDocs, err := importer.ImportDocuments()
	if !opts.Quiet {
		message := fmt.Sprintf("imported 1 object\n")
		if numDocs != 1 {
			message = fmt.Sprintf("imported %v objects\n", numDocs)
		}
		util.PrintfTimeStamped(message)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error importing documents: %v\n", err)
		os.Exit(1)
	}
}
Пример #2
0
func main() {

	// initialize command-line opts
	opts := commonopts.New("mongotop", "0.0.1", "<options> <sleeptime>")

	// add mongotop-specific options
	outputOpts := &options.Output{}
	opts.AddOptions(outputOpts)

	extra, err := opts.Parse()
	if err != nil {
		util.Panicf("error parsing command line options: %v", err)
	}

	// print help, if specified
	if opts.PrintHelp() {
		return
	}

	// print version, if specified
	if opts.PrintVersion() {
		return
	}

	// pull out the sleeptime
	// TODO: validate args length
	sleeptime := DEFAULT_SLEEP_TIME
	if len(extra) > 0 {
		sleeptime, err = strconv.Atoi(extra[0])
		if err != nil {
			util.Panicf("bad sleep time: %v", extra[0])
		}
	}

	// create a session provider to connect to the db
	sessionProvider, err := db.InitSessionProvider(*opts)
	if err != nil {
		util.Panicf("error initializing database session: %v", err)
	}

	// instantiate a mongotop instance
	top := &mongotop.MongoTop{
		Options:         opts,
		OutputOptions:   outputOpts,
		Outputter:       &output.TerminalOutputter{},
		SessionProvider: sessionProvider,
		Sleeptime:       time.Duration(sleeptime) * time.Second,
		Once:            outputOpts.Once,
	}

	// kick it off
	if err := top.Run(); err != nil {
		util.Panicf("error running mongotop: %v", err)
	}
}
Пример #3
0
func main() {
	// initialize command-line opts
	opts := commonopts.New("mongodump", "0.0.1", "<options>")

	inputOpts := &options.InputOptions{}
	opts.AddOptions(inputOpts)
	outputOpts := &options.OutputOptions{}
	opts.AddOptions(outputOpts)

	_, err := opts.Parse()
	if err != nil {
		fmt.Printf("error parsing command line options: %v\n\n", err)
		fmt.Printf("try 'mongodump --help' for more information\n")
		os.Exit(2)
	}

	// print help, if specified
	if opts.PrintHelp() {
		return
	}

	// print version, if specified
	if opts.PrintVersion() {
		return
	}

	// init logger
	log.InitToolLogger(opts.Verbosity)

	// create a session provider to connect to the db
	sessionProvider, err := db.InitSessionProvider(opts)
	if err != nil {
		fmt.Printf("error initializing database session: %v\n", err)
		os.Exit(1) //TODO copy legacy exit code
	}

	dump := mongodump.MongoDump{
		ToolOptions:     opts,
		OutputOptions:   outputOpts,
		InputOptions:    inputOpts,
		SessionProvider: sessionProvider,
	}

	err = dump.Dump()
	if err != nil {
		util.Exitf(1, "%v", err)
	}

}
Пример #4
0
func getDocSource(exp MongoExport) (db.DocSource, error) {
	if exp.ToolOptions.Namespace.DBPath != "" {
		shimPath, err := db.LocateShim()
		if err != nil {
			return nil, err
		}
		bsonTool := db.StorageShim{
			DBPath:     exp.ToolOptions.Namespace.DBPath,
			Database:   exp.ToolOptions.Namespace.DB,
			Collection: exp.ToolOptions.Namespace.Collection,
			Query:      exp.InputOpts.Query,
			Skip:       exp.InputOpts.Skip,
			Limit:      exp.InputOpts.Limit,
			ShimPath:   shimPath,
		}

		iter, _, err := bsonTool.Open()
		if err != nil {
			return nil, err
		}
		return db.NewDecodedBSONSource(iter), nil
	}

	sessionProvider, err := db.InitSessionProvider(exp.ToolOptions)
	if err != nil {
		return nil, err
	}
	session, err := sessionProvider.GetSession()
	if err != nil {
		return nil, err
	}
	collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)

	query := map[string]interface{}{}
	if exp.InputOpts != nil && exp.InputOpts.Query != "" {
		var err error
		query, err = getObjectFromArg(exp.InputOpts.Query)
		if err != nil {
			return nil, err
		}
	}

	q := collection.Find(query)

	if exp.InputOpts != nil && exp.InputOpts.Skip > 0 {
		q = q.Skip(exp.InputOpts.Skip)
	}
	if exp.InputOpts != nil && exp.InputOpts.Limit > 0 {
		q = q.Limit(exp.InputOpts.Limit)
	}

	if exp.InputOpts != nil && exp.InputOpts.Sort != "" {
		sortD, err := getSortFromArg(exp.InputOpts.Sort)
		if err != nil {
			return nil, err
		}
		sortFields, err := bsonutil.MakeSortString(sortD)
		if err != nil {
			return nil, err
		}
		q = q.Sort(sortFields...)
	}

	if len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.ForceTableScan != true && exp.InputOpts.Sort == "" {
		q = q.Snapshot()
	}

	cursor := q.Iter()
	return &db.CursorDocSource{cursor, session}, nil
}
Пример #5
0
func TestImportDocuments(t *testing.T) {
	Convey("Given a mongoimport instance with which to import documents, on "+
		"calling importDocuments", t, func() {
		Convey("no error should be thrown for CSV import on test data and all "+
			"CSV data lines should be imported correctly", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test.csv",
				Fields: "a,b,c",
			}
			ingestOptions := &options.IngestOptions{}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 3)
			So(err, ShouldBeNil)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--ignoreBlanks only fields without blanks should be imported",
			func() {
				toolOptions := getBasicToolOptions()
				inputOptions := &options.InputOptions{
					Type:   CSV,
					File:   "testdata/test_blanks.csv",
					Fields: "_id,b,c",
				}
				ingestOptions := &options.IngestOptions{
					IgnoreBlanks: true,
				}
				sessionProvider, err := db.InitSessionProvider(*toolOptions)
				So(err, ShouldBeNil)
				mongoImport := MongoImport{
					ToolOptions:     toolOptions,
					InputOptions:    inputOptions,
					IngestOptions:   ingestOptions,
					SessionProvider: sessionProvider,
				}
				numImported, err := mongoImport.ImportDocuments()
				So(numImported, ShouldEqual, 3)
				So(err, ShouldBeNil)
				expectedDocuments := []bson.M{
					bson.M{"_id": 1, "b": 2},
					bson.M{"_id": 5, "c": "6e"},
					bson.M{"_id": 7, "b": 8, "c": 6},
				}
				So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
			})
		Convey("no error should be thrown for CSV import on test data without "+
			"--ignoreBlanks supplied - fields with blanks should be imported",
			func() {
				toolOptions := getBasicToolOptions()
				inputOptions := &options.InputOptions{
					Type:   CSV,
					File:   "testdata/test_blanks.csv",
					Fields: "_id,b,c",
				}
				ingestOptions := &options.IngestOptions{}
				sessionProvider, err := db.InitSessionProvider(*toolOptions)
				So(err, ShouldBeNil)
				mongoImport := MongoImport{
					ToolOptions:     toolOptions,
					InputOptions:    inputOptions,
					IngestOptions:   ingestOptions,
					SessionProvider: sessionProvider,
				}
				numImported, err := mongoImport.ImportDocuments()
				So(numImported, ShouldEqual, 3)
				So(err, ShouldBeNil)
				expectedDocuments := []bson.M{
					bson.M{"_id": 1, "b": 2, "c": ""},
					bson.M{"_id": 5, "b": "", "c": "6e"},
					bson.M{"_id": 7, "b": 8, "c": 6},
				}
				So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
			})
		Convey("no error should be thrown for CSV import on test data with "+
			"--upsert", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{
				Upsert: true,
			}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 3)
			So(err, ShouldBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--stopOnError. Only documents before error should be imported",
			func() {
				toolOptions := getBasicToolOptions()
				inputOptions := &options.InputOptions{
					Type:   CSV,
					File:   "testdata/test.csv",
					Fields: "_id,b,c",
				}
				ingestOptions := &options.IngestOptions{
					StopOnError: true,
				}
				sessionProvider, err := db.InitSessionProvider(*toolOptions)
				So(err, ShouldBeNil)
				mongoImport := MongoImport{
					ToolOptions:     toolOptions,
					InputOptions:    inputOptions,
					IngestOptions:   ingestOptions,
					SessionProvider: sessionProvider,
				}
				numImported, err := mongoImport.ImportDocuments()
				So(numImported, ShouldEqual, 3)
				So(err, ShouldBeNil)
				expectedDocuments := []bson.M{
					bson.M{"_id": 1, "b": 2, "c": 3},
					bson.M{"_id": 3, "b": 5.4, "c": "string"},
					bson.M{"_id": 5, "b": 6, "c": 6},
				}
				So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
			})
		Convey("no error should be thrown for CSV import on test data with "+
			"duplicate _id if --stopOnError is not set", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test_duplicate.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 4)
			So(err, ShouldBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 6},
				bson.M{"_id": 8, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--drop", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{
				Drop: true,
			}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 3)
			So(err, ShouldBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--headerLine", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:       CSV,
				File:       "testdata/test.csv",
				HeaderLine: true,
			}
			ingestOptions := &options.IngestOptions{}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 2)
			So(err, ShouldBeNil)
		})
		Convey("EOF should be thrown for CSV import on test data with "+
			"--headerLine if the input file is empty", func() {
			toolOptions := getBasicToolOptions()
			csvFile, err := ioutil.TempFile("", "mongoimport_")
			So(err, ShouldBeNil)
			csvFile.Close()
			inputOptions := &options.InputOptions{
				Type:       CSV,
				File:       csvFile.Name(),
				HeaderLine: true,
			}
			ingestOptions := &options.IngestOptions{}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 0)
			So(err, ShouldEqual, io.EOF)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--upsert and --upsertFields", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{
				Upsert:       true,
				UpsertFields: "_id",
			}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 3)
			So(err, ShouldBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("no error should be thrown for CSV import on test data with "+
			"--upsert and --upsertFields and duplicate _id if --stopOnError "+
			"is not set", func() {
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test_duplicate.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{
				Upsert:       true,
				UpsertFields: "_id",
			}
			toolOptions := getBasicToolOptions()
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 5)
			So(err, ShouldBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 9},
				bson.M{"_id": 8, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("an error should be thrown for CSV import on test data with "+
			"duplicate _id if --stopOnError is set", func() {
			toolOptions := getBasicToolOptions()
			inputOptions := &options.InputOptions{
				Type:   CSV,
				File:   "testdata/test_duplicate.csv",
				Fields: "_id,b,c",
			}
			ingestOptions := &options.IngestOptions{
				StopOnError: true,
			}
			sessionProvider, err := db.InitSessionProvider(*toolOptions)
			So(err, ShouldBeNil)
			mongoImport := MongoImport{
				ToolOptions:     toolOptions,
				InputOptions:    inputOptions,
				IngestOptions:   ingestOptions,
				SessionProvider: sessionProvider,
			}
			numImported, err := mongoImport.ImportDocuments()
			So(numImported, ShouldEqual, 3)
			So(err, ShouldNotBeNil)
			expectedDocuments := []bson.M{
				bson.M{"_id": 1, "b": 2, "c": 3},
				bson.M{"_id": 3, "b": 5.4, "c": "string"},
				bson.M{"_id": 5, "b": 6, "c": 6},
			}
			So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
		})
		Convey("an error should be thrown for invalid CSV import on test data",
			func() {
				inputOptions := &options.InputOptions{
					Type:   CSV,
					File:   "testdata/test_bad.csv",
					Fields: "_id,b,c",
				}
				toolOptions := getBasicToolOptions()
				ingestOptions := &options.IngestOptions{}
				sessionProvider, err := db.InitSessionProvider(*toolOptions)
				So(err, ShouldBeNil)
				mongoImport := MongoImport{
					ToolOptions:     toolOptions,
					InputOptions:    inputOptions,
					IngestOptions:   ingestOptions,
					SessionProvider: sessionProvider,
				}
				numImported, err := mongoImport.ImportDocuments()
				So(numImported, ShouldEqual, 1)
				So(err, ShouldNotBeNil)
				expectedDocuments := []bson.M{
					bson.M{"_id": 1, "b": 2, "c": 3},
				}
				So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
			})
		Reset(func() {
			session, err := sessionProvider.GetSession()
			if err != nil {
				t.Fatalf("Error doing cleanup: %v", err)
			}
			defer session.Close()
			session.DB(testDB).C(testCollection).DropCollection()
		})

	})
}
Пример #6
0
	}
	namespace = &commonOpts.Namespace{
		DB:         testDB,
		Collection: testCollection,
	}
	connection = &commonOpts.Connection{
		Host: testServer,
		Port: testPort,
	}
	toolOptions = &commonOpts.ToolOptions{
		SSL:        ssl,
		Namespace:  namespace,
		Connection: connection,
		Auth:       &commonOpts.Auth{},
	}
	sessionProvider, _ = db.InitSessionProvider(*toolOptions)
)

func TestMongoImportValidateSettings(t *testing.T) {
	Convey("Given a mongoimport instance for validation, ", t, func() {
		Convey("an error should be thrown if no database is given", func() {
			namespace := &commonOpts.Namespace{}
			toolOptions := &commonOpts.ToolOptions{
				Namespace: namespace,
			}
			inputOptions := &options.InputOptions{
				Type: CSV,
			}
			ingestOptions := &options.IngestOptions{}
			mongoImport := MongoImport{
				ToolOptions:   toolOptions,