func collectShards(dbs []os.FileInfo) tsdb.ShardInfos { // Get the list of shards for conversion. var shards tsdb.ShardInfos for _, db := range dbs { d := tsdb.NewDatabase(filepath.Join(opts.DataPath, db.Name())) shs, err := d.Shards() if err != nil { log.Fatalf("Failed to access shards for database %v: %v\n", d.Name(), err) } shards = append(shards, shs...) } sort.Sort(shards) shards = shards.FilterFormat(tsdb.TSM1) if len(dbs) > 0 { shards = shards.ExclusiveDatabases(opts.DBs) } return shards }
func main() { pg := NewParallelGroup(1) flag.Parse() if len(flag.Args()) < 1 { fmt.Fprintf(os.Stderr, "No data directory specified\n") os.Exit(1) } dataPath = flag.Args()[0] if tsmSz > maxTSMSz { fmt.Fprintf(os.Stderr, "Maximum TSM file size is %d\n", maxTSMSz) os.Exit(1) } // Check if specific directories were requested. reqDs := strings.Split(ds, ",") if len(reqDs) == 1 && reqDs[0] == "" { reqDs = nil } // Determine the list of databases dbs, err := ioutil.ReadDir(dataPath) if err != nil { fmt.Fprintf(os.Stderr, "failed to access data directory at %s: %s\n", dataPath, err.Error()) os.Exit(1) } fmt.Println() // Cleanly separate output from start of program. // Dump summary of what is about to happen. fmt.Println("b1 and bz1 shard conversion.") fmt.Println("-----------------------------------") fmt.Println("Data directory is: ", dataPath) fmt.Println("Databases specified: ", allDBs(reqDs)) fmt.Println("Database backups enabled:", yesno(!disBack)) fmt.Println("Parallel mode enabled: ", yesno(parallel)) fmt.Println() // Get the list of shards for conversion. var shards []*tsdb.ShardInfo for _, db := range dbs { if strings.HasSuffix(db.Name(), backupExt) { fmt.Printf("Skipping %s as it looks like a backup.\n", db.Name()) continue } d := tsdb.NewDatabase(filepath.Join(dataPath, db.Name())) shs, err := d.Shards() if err != nil { fmt.Fprintf(os.Stderr, "failed to access shards for database %s: %s\n", d.Name(), err.Error()) os.Exit(1) } shards = append(shards, shs...) } sort.Sort(tsdb.ShardInfos(shards)) usl := len(shards) shards = tsdb.ShardInfos(shards).FilterFormat(tsdb.TSM1).ExclusiveDatabases(reqDs) sl := len(shards) // Anything to convert? fmt.Printf("\n%d shard(s) detected, %d non-TSM shards detected.\n", usl, sl) if len(shards) == 0 { fmt.Printf("Nothing to do.\n") os.Exit(0) } // Display list of convertible shards. fmt.Println() w := new(tabwriter.Writer) w.Init(os.Stdout, 0, 8, 1, '\t', 0) fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize") for _, si := range shards { fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(dataPath), si.FormatAsString(), si.Size) } w.Flush() // Get confirmation from user. fmt.Printf("\nThese shards will be converted. Proceed? y/N: ") liner := bufio.NewReader(os.Stdin) yn, err := liner.ReadString('\n') if err != nil { fmt.Fprintf(os.Stderr, "failed to read response: %s", err.Error()) os.Exit(1) } yn = strings.TrimRight(strings.ToLower(yn), "\n") if yn != "y" { fmt.Println("Conversion aborted.") os.Exit(1) } fmt.Println("Conversion starting....") // Backup each directory. if !disBack { databases := tsdb.ShardInfos(shards).Databases() if parallel { pg = NewParallelGroup(len(databases)) } for _, db := range databases { pg.Request() go func(db string) { defer pg.Release() start := time.Now() err := backupDatabase(filepath.Join(dataPath, db)) if err != nil { fmt.Fprintf(os.Stderr, "Backup of database %s failed: %s\n", db, err.Error()) os.Exit(1) } fmt.Printf("Database %s backed up (%v)\n", db, time.Now().Sub(start)) }(db) } pg.Wait() } else { fmt.Println("Database backup disabled.") } // Convert each shard. if parallel { pg = NewParallelGroup(len(shards)) } for _, si := range shards { pg.Request() go func(si *tsdb.ShardInfo) { defer pg.Release() start := time.Now() if err := convertShard(si); err != nil { fmt.Fprintf(os.Stderr, "Failed to convert %s: %s\n", si.FullPath(dataPath), err.Error()) os.Exit(1) } fmt.Printf("Conversion of %s successful (%s)\n", si.FullPath(dataPath), time.Now().Sub(start)) }(si) } pg.Wait() }