func runServer(_ *docopt.Args) error {
addr := ":" + os.Getenv("PORT")
db := postgres.Wait(nil, nil)
if err := dbMigrations.Migrate(db); err != nil {
return fmt.Errorf("error running DB migrations: %s", err)
}
mux := http.NewServeMux()
repo, err := data.NewFileRepoFromEnv(db)
if err != nil {
return err
}
hb, err := discoverd.AddServiceAndRegister("blobstore", addr)
if err != nil {
return err
}
shutdown.BeforeExit(func() { hb.Close() })
log.Println("Blobstore serving files on " + addr)
mux.Handle("/", handler(repo))
mux.Handle(status.Path, status.Handler(func() status.Status {
if err := db.Exec("SELECT 1"); err != nil {
return status.Unhealthy
}
return status.Healthy
}))
h := httphelper.ContextInjector("blobstore", httphelper.NewRequestLogger(mux))
return http.ListenAndServe(addr, h)
}
func runMigrate(args *docopt.Args) error {
deleteFiles := args.Bool["--delete"]
concurrency, err := strconv.Atoi(args.String["--concurrency"])
if err != nil {
return err
}
if concurrency < 1 {
concurrency = 4
}
prefix := args.String["--prefix"]
db := postgres.Wait(nil, nil)
repo, err := data.NewFileRepoFromEnv(db)
if err != nil {
return nil
}
files, err := repo.ListFilesExcludingDefaultBackend(prefix)
if err != nil {
return nil
}
var wg sync.WaitGroup
tokens := make(chan struct{}, concurrency)
var errorCount int64
dest := repo.DefaultBackend().Name()
for i, f := range files {
log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest)
tokens <- struct{}{}
wg.Add(1)
go func(f data.BackendFile) {
if err := moveFile(db, repo, f, deleteFiles); err != nil {
log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err)
atomic.AddInt64(&errorCount, 1)
}
<-tokens
wg.Done()
}(f)
}
wg.Wait()
db.Close()
if errorCount > 0 {
return fmt.Errorf("Finished with %d errors", errorCount)
}
log.Printf("Finished with no errors.")
return nil
}
func main() {
deleteFiles := flag.Bool("delete", false, "enable deletion of files from source backend")
concurrency := flag.Int("concurrency", 4, "number of parallel file moves to run at a time")
prefix := flag.String("prefix", "", "only migrate files with a name that starts with this prefix")
flag.Parse()
db := postgres.Wait(nil, nil)
repo, err := data.NewFileRepoFromEnv(db)
if err != nil {
log.Fatal(err)
}
files, err := repo.ListFilesExcludingDefaultBackend(*prefix)
if err != nil {
log.Fatal(err)
}
var wg sync.WaitGroup
tokens := make(chan struct{}, *concurrency)
var errorCount int64
dest := repo.DefaultBackend().Name()
for i, f := range files {
log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest)
tokens <- struct{}{}
wg.Add(1)
go func(f data.BackendFile) {
if err := moveFile(db, repo, f, *deleteFiles); err != nil {
log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err)
atomic.AddInt64(&errorCount, 1)
}
<-tokens
wg.Done()
}(f)
}
wg.Wait()
db.Close()
if errorCount > 0 {
log.Printf("Finished with %d errors", errorCount)
os.Exit(1)
} else {
log.Printf("Finished with no errors.")
}
}
func runCleanup(args *docopt.Args) error {
concurrency, err := strconv.Atoi(args.String["--concurrency"])
if err != nil {
return err
}
if concurrency < 1 {
concurrency = 4
}
db := postgres.Wait(nil, nil)
repo, err := data.NewFileRepoFromEnv(db)
if err != nil {
return err
}
files, err := repo.ListDeletedFilesForCleanup()
if err != nil {
return err
}
var wg sync.WaitGroup
tokens := make(chan struct{}, concurrency)
for i, f := range files {
if f.Backend == nil {
log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID)
continue
}
tokens <- struct{}{}
wg.Add(1)
go func(f data.BackendFile) {
if err := f.Backend.Delete(nil, f.FileInfo); err == nil {
log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name())
}
<-tokens
wg.Done()
}(f)
}
wg.Wait()
db.Close()
log.Printf("Done.")
return nil
}
func main() {
concurrency := flag.Int("concurrency", 4, "number of parallel file deletions to run at a time")
flag.Parse()
db := postgres.Wait(nil, nil)
repo, err := data.NewFileRepoFromEnv(db)
if err != nil {
log.Fatal(err)
}
files, err := repo.ListDeletedFilesForCleanup()
if err != nil {
log.Fatal(err)
}
var wg sync.WaitGroup
tokens := make(chan struct{}, *concurrency)
for i, f := range files {
if f.Backend == nil {
log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID)
continue
}
tokens <- struct{}{}
wg.Add(1)
go func(f data.BackendFile) {
if err := f.Backend.Delete(nil, f.FileInfo); err == nil {
log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name())
}
<-tokens
wg.Done()
}(f)
}
wg.Wait()
db.Close()
log.Printf("Done.")
}