func (r *FileRepo) ListDeletedFilesForCleanup() ([]BackendFile, error) {
rows, err := r.db.Query(
"SELECT file_id, external_id, backend, name FROM files WHERE backend != 'postgres' AND deleted_at IS NOT NULL",
)
if err != nil {
return nil, err
}
defer rows.Close()
var res []BackendFile
for rows.Next() {
var info backend.FileInfo
var backendName string
var externalID *string
if rows.Scan(&info.ID, &externalID, &backendName, &info.Name); err != nil {
return nil, err
}
if externalID != nil {
info.ExternalID = *externalID
}
f := BackendFile{FileInfo: info}
f.Backend, _ = r.getBackend(backendName)
res = append(res, f)
}
return res, rows.Err()
}
func (r *FileRepo) ListFilesExcludingDefaultBackend(prefix string) ([]BackendFile, error) {
rows, err := r.db.Query(
"SELECT file_id, file_oid, external_id, backend, name, type, size, sha512, updated_at FROM files WHERE backend != $1 AND NAME LIKE $2 || '%' AND deleted_at IS NULL",
r.defaultBackend.Name(), prefix,
)
if err != nil {
return nil, err
}
defer rows.Close()
var res []BackendFile
for rows.Next() {
var info backend.FileInfo
var backendName string
var externalID *string
var sha512 []byte
if rows.Scan(&info.ID, &info.Oid, &externalID, &backendName, &info.Name, &info.Type, &info.Size, &sha512, &info.ModTime); err != nil {
return nil, err
}
if externalID != nil {
info.ExternalID = *externalID
}
info.ETag = base64.StdEncoding.EncodeToString(sha512)
f := BackendFile{FileInfo: info}
f.Backend, _ = r.getBackend(backendName)
res = append(res, f)
}
return res, rows.Err()
}
func (r *FileRepo) Copy(to, from string) error {
tx, err := r.db.Begin()
if err != nil {
return err
}
info := backend.FileInfo{
Name: from,
}
var backendName string
var externalID *string
var sha512, sha512State []byte
if err := tx.QueryRow(
"SELECT file_id, file_oid, external_id, backend, type, size, sha512, sha512_state, updated_at FROM files WHERE name = $1 AND deleted_at IS NULL", from,
).Scan(&info.ID, &info.Oid, &externalID, &backendName, &info.Type, &info.Size, &sha512, &sha512State, &info.ModTime); err != nil {
if err == pgx.ErrNoRows {
return backend.ErrNotFound
}
tx.Rollback()
return err
}
if externalID != nil {
info.ExternalID = *externalID
}
b, err := r.getBackend(backendName)
if err != nil {
tx.Rollback()
return err
}
toInfo := backend.FileInfo{
Name: to,
Type: info.Type,
}
if err := tx.QueryRow(
"INSERT INTO files (backend, name, type, size, sha512, sha512_state) VALUES ($1, $2, $3, $4, $5, $6) RETURNING file_id",
backendName, to, info.Type, info.Size, sha512, sha512State,
).Scan(&toInfo.ID); err != nil {
tx.Rollback()
return err
}
if err := b.Copy(tx, toInfo, info); err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
// Get is like Open, except the FileStream is not populated (useful for HEAD requests)
func (r *FileRepo) Get(name string, body bool) (*backend.File, error) {
tx, err := r.db.Begin()
if err != nil {
return nil, err
}
var info backend.FileInfo
var backendName string
var externalID *string
var sha512 []byte
if err := tx.QueryRow(
"SELECT file_id, file_oid, external_id, backend, name, type, size, sha512, updated_at FROM files WHERE name = $1 AND deleted_at IS NULL",
name,
).Scan(&info.ID, &info.Oid, &externalID, &backendName, &info.Name, &info.Type, &info.Size, &sha512, &info.ModTime); err != nil {
if err == pgx.ErrNoRows {
err = backend.ErrNotFound
}
tx.Rollback()
return nil, err
}
if externalID != nil {
info.ExternalID = *externalID
}
info.ETag = base64.StdEncoding.EncodeToString(sha512)
if !body {
tx.Rollback()
return &backend.File{FileInfo: info, FileStream: fakeSizeSeekerFileStream{info.Size}}, nil
}
b, err := r.getBackend(backendName)
if err != nil {
tx.Rollback()
return nil, err
}
stream, err := b.Open(tx, info, true)
if err != nil {
tx.Rollback()
return nil, err
}
return &backend.File{FileInfo: info, FileStream: stream}, nil
}
func (r *FileRepo) Delete(name string) error {
tx, err := r.db.Begin()
if err != nil {
return err
}
// use a regular expression so that either a file with the name is
// deleted, or any file prefixed with "{name}/" is deleted (so in other
// words, mimic either deleting a file or recursively deleting a
// directory)
rows, err := tx.Query(
"UPDATE files SET deleted_at = now() WHERE name ~ ('^' || $1 || '(/.*)?$') AND deleted_at IS NULL RETURNING file_id, external_id, backend, name", name,
)
if err != nil {
tx.Rollback()
return err
}
backendFiles := make(map[string][]backend.FileInfo)
for rows.Next() {
var info backend.FileInfo
var backendName string
var externalID *string
if err := rows.Scan(&info.ID, &externalID, &backendName, &info.Name); err != nil {
rows.Close()
tx.Rollback()
return err
}
if externalID != nil {
info.ExternalID = *externalID
}
backendFiles[backendName] = append(backendFiles[backendName], info)
}
rows.Close()
if err := rows.Err(); err != nil {
tx.Rollback()
return err
}
if err := tx.Commit(); err != nil {
return err
}
var errors []error
for name, files := range backendFiles {
if name == "postgres" {
// no need to call delete, it is done automatically by a trigger
continue
}
b, err := r.getBackend(name)
if err != nil {
errors = append(errors, err)
}
for _, f := range files {
if err := b.Delete(nil, f); err != nil {
errors = append(errors, err)
}
}
}
if len(errors) > 0 {
return errors[0]
}
return nil
}
func (r *FileRepo) Put(name string, data io.Reader, offset int64, typ string) error {
tx, err := r.db.Begin()
if err != nil {
return err
}
info := backend.FileInfo{
Name: name,
Type: typ,
}
h := sha512.New().(resumable.Hash)
b := r.defaultBackend
create:
err = tx.QueryRow("INSERT INTO files (name, backend, type) VALUES ($1, $2, $3) RETURNING file_id", name, b.Name(), typ).Scan(&info.ID)
if postgres.IsUniquenessError(err, "") {
tx.Rollback()
tx, err = r.db.Begin()
if err != nil {
return err
}
if offset > 0 {
var backendName string
var sha512State []byte
var externalID *string
// file exists, get details
if err := tx.QueryRow(
"SELECT file_id, file_oid, external_id, backend, size, sha512_state FROM files WHERE name = $1 AND deleted_at IS NULL", name,
).Scan(&info.ID, &info.Oid, &externalID, &backendName, &info.Size, &sha512State); err != nil {
tx.Rollback()
return err
}
if externalID != nil {
info.ExternalID = *externalID
}
b, err = r.getBackend(backendName)
if err != nil {
tx.Rollback()
return err
}
if offset != info.Size {
tx.Rollback()
// TODO: pass error via HTTP response
return fmt.Errorf("blobstore: offset (%d) does not match blob size (%d), unable to append", offset, info.Size)
}
if len(sha512State) > 0 {
err = h.Restore(sha512State)
}
if (len(sha512State) == 0 || err != nil || h.Len() != info.Size) && info.Size > 0 {
// hash state is not resumable, read current data into hash
f, err := b.Open(tx, info, false)
if err != nil {
tx.Rollback()
return err
}
h.Reset()
if _, err := io.Copy(h, io.LimitReader(f, info.Size)); err != nil {
f.Close()
tx.Rollback()
return err
}
f.Close()
}
} else {
// file exists, not appending, overwrite by deleting
var di backend.FileInfo
var externalID *string
var backendName string
if err := tx.QueryRow(
"UPDATE files SET deleted_at = now() WHERE name = $1 AND deleted_at IS NULL RETURNING file_id, external_id, backend", name,
).Scan(&di.ID, &externalID, &backendName); err != nil {
tx.Rollback()
return err
}
if externalID != nil {
di.ExternalID = *externalID
}
// delete old file from backend
if err := func() error {
if backendName == "postgres" {
// no need to call delete, it is done automatically by a trigger
return nil
}
b, err := r.getBackend(backendName)
if err != nil {
return err
}
return b.Delete(nil, di)
}(); err != nil {
log.Printf("Error deleting %s (%s) from backend %s: %s", di.ExternalID, name, backendName, err)
}
goto create
}
} else if err != nil {
tx.Rollback()
return err
}
sr := newSizeReader(data)
sr.size = info.Size
if err := b.Put(tx, info, io.TeeReader(sr, h), offset > 0); err != nil {
tx.Rollback()
return err
}
sha512State, _ := h.State()
if err := tx.Exec(
"UPDATE files SET size = $2, sha512 = $3, sha512_state = $4, updated_at = now() WHERE file_id = $1",
info.ID, sr.Size(), h.Sum(nil), sha512State,
); err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}