예제 #1
0
파일: router.go 프로젝트: philc/gumshoedb
func main() {
	configFile := flag.String("config", "config.toml", "path to a DB config (to get the schema)")
	shardsFlag := flag.String("shards", "", "comma-separated list of shard addresses (with ports)")
	port := flag.Int("port", 9090, "port on which to listen")
	flag.Parse()
	shardAddrs := strings.Split(*shardsFlag, ",")
	if *shardsFlag == "" || len(shardAddrs) == 0 {
		Log.Fatal("At least one shard required")
	}
	f, err := os.Open(*configFile)
	if err != nil {
		Log.Fatal(err)
	}
	defer f.Close()
	_, schema, err := config.LoadTOMLConfig(f)
	if err != nil {
		Log.Fatal(err)
	}
	schema.Initialize()

	r := NewRouter(shardAddrs, schema)
	addr := fmt.Sprintf(":%d", *port)
	server := &http.Server{
		Addr:    addr,
		Handler: r,
	}
	Log.Println("Now serving on", addr)
	Log.Fatal(server.ListenAndServe())
}
예제 #2
0
func TestSanity(t *testing.T) {
	const configText = `
listen_addr = ""
database_dir = "MEMORY"
flush_interval = "1h"
statsd_addr = "localhost:8125"
open_file_limit = 1000
query_parallelism = 10
retention_days = 7

[schema]
segment_size = "1MB"
interval_duration = "1h"
timestamp_column = ["at", "uint32"]
dimension_columns = [["dim1", "uint32"]]
metric_columns = [["metric1", "uint32"]]
	`
	conf, schema, err := config.LoadTOMLConfig(strings.NewReader(configText))
	if err != nil {
		t.Fatal(err)
	}
	server := httptest.NewServer(NewServer(conf, schema))
	defer server.Close()

	resp, err := http.Get(server.URL + "/")
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Error("Expected 200 at /")
	}
	resp.Body.Close()
}
예제 #3
0
파일: merge.go 프로젝트: philc/gumshoedb
func merge(args []string) {
	flags := flag.NewFlagSet("gumtool merge", flag.ExitOnError)
	var (
		newConfigFilename string
		oldDBPaths        stringsFlag
		parallelism       int
		numOpenFiles      int
		flushSegments     int
	)
	flags.StringVar(&newConfigFilename, "new-db-config", "", "Filename of the new DB config")
	flags.Var(&oldDBPaths, "db-paths", "Paths to dirs of DBs to merge")
	flags.IntVar(&parallelism, "parallelism", 4, "Parallelism for merge workers")
	flags.IntVar(&numOpenFiles, "rlimit-nofile", 10000, "Value for RLIMIT_NOFILE")
	flags.IntVar(&flushSegments, "flush-segments", 500, "Flush after merging each N segments")
	flags.Parse(args)

	if len(oldDBPaths) == 0 {
		log.Fatalln("Need at least one entry in -db-paths; got 0")
	}

	setRlimit(numOpenFiles)

	f, err := os.Open(newConfigFilename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	_, schema, err := config.LoadTOMLConfig(f)
	if err != nil {
		log.Fatal(err)
	}
	newDB, err := gumshoe.NewDB(schema)
	if err != nil {
		log.Fatal(err)
	}
	defer newDB.Close()

	dbs := make([]*gumshoe.DB, len(oldDBPaths))
	for i, path := range oldDBPaths {
		db, err := gumshoe.OpenDBDir(path)
		if err != nil {
			log.Fatalf("Error opening DB at %s: %s", path, err)
		}
		if err := db.Schema.Equivalent(schema); err != nil {
			log.Fatalf("Schema of DB at %s didn't match config at %s: %s", path, newConfigFilename, err)
		}
		dbs[i] = db
	}

	for _, db := range dbs {
		log.Printf("Merging db %s", db.Schema.Dir)
		if err := mergeDB(newDB, db, parallelism, flushSegments); err != nil {
			log.Fatalln("Error merging:", err)
		}
		db.Close()
	}
}
예제 #4
0
파일: server.go 프로젝트: philc/gumshoedb
func main() {
	flag.Parse()

	f, err := os.Open(*configFile)
	if err != nil {
		Log.Fatal(err)
	}
	defer f.Close()
	conf, schema, err := config.LoadTOMLConfig(f)
	if err != nil {
		Log.Fatal(err)
	}

	// Try to set the RLIMIT_NOFILE to the config value. This might fail if the binary lacks sufficient
	// permissions/capabilities, or on non-Linux OSes.
	rlimit := &syscall.Rlimit{uint64(conf.OpenFileLimit), uint64(conf.OpenFileLimit)}
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, rlimit); err != nil {
		Log.Println("Error raising RLIMIT_NOFILE:", err)
	}

	// Set up the pprof server, if enabled.
	if *profileAddr != "" {
		go func() {
			Log.Println("Pprof listening on", *profileAddr)
			Log.Printf("Go to http://%s/debug/pprof to see more", *profileAddr)
			Log.Fatal(http.ListenAndServe(*profileAddr, nil))
		}()
	}

	// Configure the gumshoe logger
	gumshoe.Log = log.New(os.Stdout, "[gumshoe] ", logFlags)

	// Configure the statsd client
	statsd, err = gostc.NewClient(conf.StatsdAddr)
	if err != nil {
		Log.Fatal(err)
	}

	// Listen for signals so we can try to flush before shutdown
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		<-c
		close(shutdown)
	}()

	server := NewServer(conf, schema)
	Log.Fatal(server.ListenAndServe())
}
예제 #5
0
파일: migrate.go 프로젝트: philc/gumshoedb
func migrate(args []string) {
	flags := flag.NewFlagSet("gumtool migrate", flag.ExitOnError)
	oldDBPath := flags.String("old-db-path", "", "Path of old DB directory")
	newConfigFilename := flags.String("new-db-config", "", "Filename of new DB config file")
	parallelism := flags.Int("parallelism", 4, "Parallelism for reading old DB")
	numOpenFiles := flags.Int("rlimit-nofile", 10000, "The value to set RLIMIT_NOFILE")
	flushSegments := flags.Int("flush-segments", 500, "Flush after every N (old) segments")
	flags.Parse(args)

	// Attempt to raise the open file limit; necessary for big migrations
	setRlimit(*numOpenFiles)

	oldDB, err := gumshoe.OpenDBDir(*oldDBPath)
	if err != nil {
		log.Fatal(err)
	}
	defer oldDB.Close()

	f, err := os.Open(*newConfigFilename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	_, schema, err := config.LoadTOMLConfig(f)
	if err != nil {
		log.Fatal(err)
	}
	newDB, err := gumshoe.NewDB(schema)
	if err != nil {
		log.Fatal(err)
	}
	defer newDB.Close()

	if err := migrateDBs(newDB, oldDB, *parallelism, *flushSegments); err != nil {
		log.Fatal(err)
	}
	fmt.Println("done")
}
예제 #6
0
파일: balance.go 프로젝트: philc/gumshoedb
func balance(args []string) {
	flags := flag.NewFlagSet("gumtool balance", flag.ExitOnError)
	var (
		username      string
		sourceDBDirs  stringsFlag
		destDBConfigs stringsFlag
		binDir        string
		workDir       string
	)
	flags.StringVar(&username, "username", "", "SSH username")
	flags.Var(&sourceDBDirs, "source-db-dirs",
		"Dirs of source DBs in form host.com:/path/to/db; comma-separated")
	flags.Var(&destDBConfigs, "dest-db-configs",
		"Filenames of dest DB configs in form host.com:/path/to/conf.toml; comma-separated")
	flags.StringVar(&binDir, "bindir", "", "Path to dir containing gumshoedb binaries (same on all servers)")
	flags.StringVar(&workDir, "workdir", "", "Path to dir for temporary partial DBs (same on all servers)")
	flags.Parse(args)

	if len(sourceDBDirs) == 0 {
		log.Fatal("Need at least one dir in -source-db-dirs")
	}
	if len(destDBConfigs) == 0 {
		log.Fatal("Need at least one config in -dest-db-configs")
	}
	if binDir == "" {
		log.Fatal("-bindir must be given")
	}
	if workDir == "" {
		log.Fatal("-workdir must be given")
	}

	sources := make([]balanceSource, len(sourceDBDirs))
	for i, hostDir := range sourceDBDirs {
		parts := strings.SplitN(hostDir, ":", 2)
		if len(parts) != 2 {
			log.Fatalln("Bad host:dir:", hostDir)
		}
		sources[i] = balanceSource{parts[0], parts[1]}
	}

	dests := make([]balanceDest, len(destDBConfigs))
	for i, hostConfig := range destDBConfigs {
		parts := strings.SplitN(hostConfig, ":", 2)
		if len(parts) != 2 {
			log.Fatalln("Bad host:config:", hostConfig)
		}
		dests[i] = balanceDest{parts[0], parts[1]}
	}

	s := NewSSH(username)

	//
	// 1. Load all the schemas from the source DB dirs and check mutual equivalence.
	//

	// These *gumshoeDBs only have schemas and static table metadata filled out.
	sourceDBs := make([]*gumshoe.DB, len(sources))
	for i, source := range sources {
		sftpClient, err := s.getSFTPClient(source.host)
		if err != nil {
			log.Fatal(err)
		}
		dbJSONPath := path.Join(source.dbDir, "db.json")
		f, err := sftpClient.Open(dbJSONPath)
		if err != nil {
			log.Fatalf("Cannot open %s:%s: %s", source.host, dbJSONPath, err)
		}
		db := new(gumshoe.DB)
		if err := json.NewDecoder(f).Decode(db); err != nil {
			log.Fatalf("Error loading db.json schema from %s: %s", source.host, err)
		}
		f.Close()
		sourceDBs[i] = db
		if i > 0 {
			if err := sourceDBs[0].Schema.Equivalent(db.Schema); err != nil {
				log.Fatalln("Found non-equivalent schemas amongst source DBs:", err)
			}
		}
	}
	log.Printf("Loaded %d source DBs and confirmed equivalence", len(sourceDBs))

	//
	// 2. Load all the dest DB configs and check equivalence.
	//

	for _, dest := range dests {
		sftpClient, err := s.getSFTPClient(dest.host)
		if err != nil {
			log.Fatal(err)
		}
		f, err := sftpClient.Open(dest.config)
		if err != nil {
			log.Fatalf("Error opening toml config at %s:%s: %s", dest.host, dest.config, err)
		}
		_, schema, err := config.LoadTOMLConfig(f)
		if err != nil {
			log.Fatalf("Error loading config TOML from %s: %s", dest.host, err)
		}
		f.Close()
		if err := sourceDBs[0].Schema.Equivalent(schema); err != nil {
			log.Fatalf("Found schema in DB config on %s that is not equivalent to the source DBs: %s", dest.host, err)
		}
	}
	log.Printf("Checked %d dest DB configs and confirmed equivalence to source DBs", len(dests))

	//
	// 3. Do the per-source work (in parallel). For each source DB:
	//    a. Partition the DB intervals by dest DB
	//    b. Create new DB dirs called /tmp/gumshoedb/db.partial.shardN
	//    c. Copy in the segment files for the interval to the appropriate partial DB
	//    d. Copy in the source dimension files to each partial DB
	//    e. Write out a db.json for each partial DB
	//    f. Gzip each partial DB
	//    g. SCP each partial DB to the dest DB host at /tmp/gumshoedb/db.partial.fromhostXXX.gz
	//

	if err := MakeWorkDirs(sources, dests, workDir, s); err != nil {
		log.Fatal(err)
	}
	var (
		wg sync.WaitGroup
		mu sync.Mutex // protects partialsByDest
		// Map each dest to all the sources that have a partial for it.
		partialsByDest = make(map[balanceDest][]balanceSource)
	)
	for i := range sourceDBs {
		source := sources[i]
		db := sourceDBs[i]
		wg.Add(1)
		go func() {
			defer wg.Done()
			partialDests, err := PreparePartials(source, dests, db, workDir, s)
			if err != nil {
				log.Fatal(err)
			}
			mu.Lock()
			for _, dest := range partialDests {
				partialsByDest[dest] = append(partialsByDest[dest], source)
			}
			mu.Unlock()
		}()
	}
	wg.Wait()

	//
	// 4. Do the per-dest work (in parallel). For each dest DB:
	//    a. Un-gzip each partial DB
	//    b. gumtool merge the partial DBs
	//

	for i := range dests {
		dest := dests[i]
		sources := partialsByDest[dest]
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := MergeDBs(dest, sources, workDir, binDir, s); err != nil {
				log.Fatal(err)
			}
		}()
	}
	wg.Wait()
	if err := RemoveWorkDirs(sources, dests, workDir, s); err != nil {
		log.Fatal(err)
	}
}