func runServer(_ *docopt.Args) error { addr := ":" + os.Getenv("PORT") db := postgres.Wait(nil, nil) if err := dbMigrations.Migrate(db); err != nil { return fmt.Errorf("error running DB migrations: %s", err) } mux := http.NewServeMux() repo, err := data.NewFileRepoFromEnv(db) if err != nil { return err } hb, err := discoverd.AddServiceAndRegister("blobstore", addr) if err != nil { return err } shutdown.BeforeExit(func() { hb.Close() }) log.Println("Blobstore serving files on " + addr) mux.Handle("/", handler(repo)) mux.Handle(status.Path, status.Handler(func() status.Status { if err := db.Exec("SELECT 1"); err != nil { return status.Unhealthy } return status.Healthy })) h := httphelper.ContextInjector("blobstore", httphelper.NewRequestLogger(mux)) return http.ListenAndServe(addr, h) }
func runMigrate(args *docopt.Args) error { deleteFiles := args.Bool["--delete"] concurrency, err := strconv.Atoi(args.String["--concurrency"]) if err != nil { return err } if concurrency < 1 { concurrency = 4 } prefix := args.String["--prefix"] db := postgres.Wait(nil, nil) repo, err := data.NewFileRepoFromEnv(db) if err != nil { return nil } files, err := repo.ListFilesExcludingDefaultBackend(prefix) if err != nil { return nil } var wg sync.WaitGroup tokens := make(chan struct{}, concurrency) var errorCount int64 dest := repo.DefaultBackend().Name() for i, f := range files { log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest) tokens <- struct{}{} wg.Add(1) go func(f data.BackendFile) { if err := moveFile(db, repo, f, deleteFiles); err != nil { log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err) atomic.AddInt64(&errorCount, 1) } <-tokens wg.Done() }(f) } wg.Wait() db.Close() if errorCount > 0 { return fmt.Errorf("Finished with %d errors", errorCount) } log.Printf("Finished with no errors.") return nil }
func main() { deleteFiles := flag.Bool("delete", false, "enable deletion of files from source backend") concurrency := flag.Int("concurrency", 4, "number of parallel file moves to run at a time") prefix := flag.String("prefix", "", "only migrate files with a name that starts with this prefix") flag.Parse() db := postgres.Wait(nil, nil) repo, err := data.NewFileRepoFromEnv(db) if err != nil { log.Fatal(err) } files, err := repo.ListFilesExcludingDefaultBackend(*prefix) if err != nil { log.Fatal(err) } var wg sync.WaitGroup tokens := make(chan struct{}, *concurrency) var errorCount int64 dest := repo.DefaultBackend().Name() for i, f := range files { log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest) tokens <- struct{}{} wg.Add(1) go func(f data.BackendFile) { if err := moveFile(db, repo, f, *deleteFiles); err != nil { log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err) atomic.AddInt64(&errorCount, 1) } <-tokens wg.Done() }(f) } wg.Wait() db.Close() if errorCount > 0 { log.Printf("Finished with %d errors", errorCount) os.Exit(1) } else { log.Printf("Finished with no errors.") } }
func runCleanup(args *docopt.Args) error { concurrency, err := strconv.Atoi(args.String["--concurrency"]) if err != nil { return err } if concurrency < 1 { concurrency = 4 } db := postgres.Wait(nil, nil) repo, err := data.NewFileRepoFromEnv(db) if err != nil { return err } files, err := repo.ListDeletedFilesForCleanup() if err != nil { return err } var wg sync.WaitGroup tokens := make(chan struct{}, concurrency) for i, f := range files { if f.Backend == nil { log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID) continue } tokens <- struct{}{} wg.Add(1) go func(f data.BackendFile) { if err := f.Backend.Delete(nil, f.FileInfo); err == nil { log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name()) } <-tokens wg.Done() }(f) } wg.Wait() db.Close() log.Printf("Done.") return nil }
func main() { concurrency := flag.Int("concurrency", 4, "number of parallel file deletions to run at a time") flag.Parse() db := postgres.Wait(nil, nil) repo, err := data.NewFileRepoFromEnv(db) if err != nil { log.Fatal(err) } files, err := repo.ListDeletedFilesForCleanup() if err != nil { log.Fatal(err) } var wg sync.WaitGroup tokens := make(chan struct{}, *concurrency) for i, f := range files { if f.Backend == nil { log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID) continue } tokens <- struct{}{} wg.Add(1) go func(f data.BackendFile) { if err := f.Backend.Delete(nil, f.FileInfo); err == nil { log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name()) } <-tokens wg.Done() }(f) } wg.Wait() db.Close() log.Printf("Done.") }