Esempio n. 1
0
func runServer(_ *docopt.Args) error {
	addr := ":" + os.Getenv("PORT")

	db := postgres.Wait(nil, nil)
	if err := dbMigrations.Migrate(db); err != nil {
		return fmt.Errorf("error running DB migrations: %s", err)
	}

	mux := http.NewServeMux()

	repo, err := data.NewFileRepoFromEnv(db)
	if err != nil {
		return err
	}

	hb, err := discoverd.AddServiceAndRegister("blobstore", addr)
	if err != nil {
		return err
	}
	shutdown.BeforeExit(func() { hb.Close() })

	log.Println("Blobstore serving files on " + addr)

	mux.Handle("/", handler(repo))
	mux.Handle(status.Path, status.Handler(func() status.Status {
		if err := db.Exec("SELECT 1"); err != nil {
			return status.Unhealthy
		}
		return status.Healthy
	}))

	h := httphelper.ContextInjector("blobstore", httphelper.NewRequestLogger(mux))
	return http.ListenAndServe(addr, h)
}
Esempio n. 2
0
func runMigrate(args *docopt.Args) error {
	deleteFiles := args.Bool["--delete"]
	concurrency, err := strconv.Atoi(args.String["--concurrency"])
	if err != nil {
		return err
	}
	if concurrency < 1 {
		concurrency = 4
	}
	prefix := args.String["--prefix"]

	db := postgres.Wait(nil, nil)
	repo, err := data.NewFileRepoFromEnv(db)
	if err != nil {
		return nil
	}

	files, err := repo.ListFilesExcludingDefaultBackend(prefix)
	if err != nil {
		return nil
	}

	var wg sync.WaitGroup
	tokens := make(chan struct{}, concurrency)
	var errorCount int64

	dest := repo.DefaultBackend().Name()
	for i, f := range files {
		log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest)
		tokens <- struct{}{}
		wg.Add(1)
		go func(f data.BackendFile) {
			if err := moveFile(db, repo, f, deleteFiles); err != nil {
				log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err)
				atomic.AddInt64(&errorCount, 1)
			}
			<-tokens
			wg.Done()
		}(f)
	}

	wg.Wait()
	db.Close()
	if errorCount > 0 {
		return fmt.Errorf("Finished with %d errors", errorCount)
	}

	log.Printf("Finished with no errors.")
	return nil
}
Esempio n. 3
0
func main() {
	deleteFiles := flag.Bool("delete", false, "enable deletion of files from source backend")
	concurrency := flag.Int("concurrency", 4, "number of parallel file moves to run at a time")
	prefix := flag.String("prefix", "", "only migrate files with a name that starts with this prefix")
	flag.Parse()

	db := postgres.Wait(nil, nil)
	repo, err := data.NewFileRepoFromEnv(db)
	if err != nil {
		log.Fatal(err)
	}

	files, err := repo.ListFilesExcludingDefaultBackend(*prefix)
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	tokens := make(chan struct{}, *concurrency)
	var errorCount int64

	dest := repo.DefaultBackend().Name()
	for i, f := range files {
		log.Printf("[%d/%d] Moving %s (%s, %d bytes) from %s to %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Size, f.Backend.Name(), dest)
		tokens <- struct{}{}
		wg.Add(1)
		go func(f data.BackendFile) {
			if err := moveFile(db, repo, f, *deleteFiles); err != nil {
				log.Printf("Error moving %s (%s): %s", f.FileInfo.Name, f.ID, err)
				atomic.AddInt64(&errorCount, 1)
			}
			<-tokens
			wg.Done()
		}(f)
	}

	wg.Wait()
	db.Close()
	if errorCount > 0 {
		log.Printf("Finished with %d errors", errorCount)
		os.Exit(1)
	} else {
		log.Printf("Finished with no errors.")
	}
}
Esempio n. 4
0
func runCleanup(args *docopt.Args) error {
	concurrency, err := strconv.Atoi(args.String["--concurrency"])
	if err != nil {
		return err
	}
	if concurrency < 1 {
		concurrency = 4
	}

	db := postgres.Wait(nil, nil)
	repo, err := data.NewFileRepoFromEnv(db)
	if err != nil {
		return err
	}

	files, err := repo.ListDeletedFilesForCleanup()
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	tokens := make(chan struct{}, concurrency)
	for i, f := range files {
		if f.Backend == nil {
			log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID)
			continue
		}
		tokens <- struct{}{}
		wg.Add(1)
		go func(f data.BackendFile) {
			if err := f.Backend.Delete(nil, f.FileInfo); err == nil {
				log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name())
			}
			<-tokens
			wg.Done()
		}(f)
	}

	wg.Wait()
	db.Close()

	log.Printf("Done.")
	return nil
}
Esempio n. 5
0
func main() {
	concurrency := flag.Int("concurrency", 4, "number of parallel file deletions to run at a time")
	flag.Parse()

	db := postgres.Wait(nil, nil)
	repo, err := data.NewFileRepoFromEnv(db)
	if err != nil {
		log.Fatal(err)
	}

	files, err := repo.ListDeletedFilesForCleanup()
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	tokens := make(chan struct{}, *concurrency)
	for i, f := range files {
		if f.Backend == nil {
			log.Printf("[%d/%d] Skipping %s (%s) because backend is not configured", i+1, len(files), f.FileInfo.Name, f.ID)
			continue
		}
		tokens <- struct{}{}
		wg.Add(1)
		go func(f data.BackendFile) {
			if err := f.Backend.Delete(nil, f.FileInfo); err == nil {
				log.Printf("[%d/%d] Successfully deleted %s (%s) from backend %s", i+1, len(files), f.FileInfo.Name, f.ID, f.Backend.Name())
			}
			<-tokens
			wg.Done()
		}(f)
	}

	wg.Wait()
	db.Close()

	log.Printf("Done.")
}