func (s *server) copy(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
src := getPathFromReq(r)
u, err := url.Parse(r.Header.Get("Destination"))
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusBadRequest)
return
}
dst := path.Clean(strings.TrimPrefix(u.Path, remoteURL))
if dst == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
logger.Infof("src is %s", src)
logger.Infof("dst is %s", dst)
con, err := getConnection(s.p.metaServer)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer con.Close()
client := metapb.NewMetaClient(con)
in := &metapb.CpReq{}
in.Src = src
in.Dst = dst
in.AccessToken = authlib.MustFromTokenContext(ctx)
_, err = client.Cp(ctx, in)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
w.WriteHeader(http.StatusNoContent)
}
func (s *server) mkcol(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
p := getPathFromReq(r)
logger.Infof("path is %s", p)
con, err := getConnection(s.p.metaServer)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer con.Close()
client := metapb.NewMetaClient(con)
in := &metapb.MkdirReq{}
in.AccessToken = authlib.MustFromTokenContext(ctx)
in.Path = p
_, err = client.Mkdir(ctx, in)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
w.WriteHeader(http.StatusCreated)
}
func getMeta(ctx context.Context, addr, p string, children bool) (*metapb.Metadata, error) {
in := &metapb.StatReq{}
in.AccessToken = authlib.MustFromTokenContext(ctx)
in.Children = children
in.Path = p
con, err := getConnection(addr)
if err != nil {
return nil, err
}
defer con.Close()
client := metapb.NewMetaClient(con)
meta, err := client.Stat(ctx, in)
if err != nil {
return nil, err
}
return meta, nil
}
func (s *server) upload(ctx context.Context, w http.ResponseWriter, r *http.Request) {
log := MustFromLogContext(ctx)
p := lib.MustFromContext(ctx)
pp := s.getPhysicalPath(p)
log.Infof("physical path is %s", pp)
tmpFn, tmpFile, err := s.tmpFile()
if err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
log.Infof("created tmp file %s", tmpFn)
var mw io.Writer
var hasher hash.Hash
var isChecksumed bool
var computedChecksum string
switch s.p.checksum {
case "md5":
hasher = md5.New()
isChecksumed = true
mw = io.MultiWriter(tmpFile, hasher)
case "sha1":
hasher = sha1.New()
isChecksumed = true
mw = io.MultiWriter(tmpFile, hasher)
case "adler32":
hasher = adler32.New()
isChecksumed = true
mw = io.MultiWriter(tmpFile, hasher)
default:
mw = io.MultiWriter(tmpFile)
}
// TODO(labkode) Sometimes ContentLength = -1 because it is a binary
// upload with TransferEncoding: chunked.
// Instead using Copy we shoudl use a LimitedReader with a max file upload
// configuration value.
_, err = io.Copy(mw, r.Body)
if err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
chk := s.getChecksumInfo(r)
if isChecksumed {
log.Infof("file sent with checksum %s", chk.String())
// checksums are given in hexadecimal format.
computedChecksum = fmt.Sprintf("%x", string(hasher.Sum(nil)))
if chk.Type == s.p.checksum && chk.Type != "" {
isCorrupted := computedChecksum != chk.Sum
if isCorrupted {
log.Errorf("corrupted file. expected %s and got %s",
s.p.checksum+":"+computedChecksum, chk.Sum)
http.Error(w, "", http.StatusPreconditionFailed)
return
}
}
}
log.Infof("copied r.Body into tmp file %s", tmpFn)
err = tmpFile.Close()
if err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
log.Infof("closed tmp file %s", tmpFn)
if err = os.Rename(tmpFn, pp); err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
log.Infof("renamed tmp file %s to %s", tmpFn, pp)
con, err := grpc.Dial(s.p.prop, grpc.WithInsecure())
if err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer con.Close()
log.Infof("created connection to %s", s.p.prop)
client := pb.NewPropClient(con)
in := &pb.PutReq{}
in.Path = p
in.AccessToken = authlib.MustFromTokenContext(ctx)
in.Checksum = chk.String()
_, err = client.Put(ctx, in)
if err != nil {
log.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
log.Infof("saved path %s into %s", p, s.p.prop)
w.WriteHeader(http.StatusCreated)
}
func (s *server) putChunked(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
p := getPathFromReq(r)
logger.Infof("path is %s", p)
chunkInfo, err := getChunkPathInfo(p)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
// if client sends etag check against the current value
// if file is not found or etags do not match return 412
ifMatchHeader := r.Header.Get("If-Match")
if ifMatchHeader != "" {
meta, err := getMeta(ctx, s.p.metaServer, chunkInfo.ResourcePath, false)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusPreconditionFailed)
return
}
// TODO(labkode) refactor this
if ifMatchHeader != `"`+meta.Etag+`"` {
logger.Warnf("etags do not match. client send %s and server has %s", ifMatchHeader, meta.Etag)
http.Error(w, "", http.StatusPreconditionFailed)
return
}
}
logger.Infof("%s", chunkInfo.String())
tmpFn, tmpFile, err := s.tmpFile()
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer tmpFile.Close()
logger.Infof("created tmp file %s", tmpFn)
_, err = io.Copy(tmpFile, r.Body)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("copied r.Body to %s", tmpFn)
err = tmpFile.Close()
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("closed %s", tmpFn)
chunkFolder, err := s.getChunkFolder(chunkInfo)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("chunk folder is %s", chunkFolder)
chunkDst := path.Join(chunkFolder, path.Clean(strconv.FormatUint(chunkInfo.CurrentChunk, 10)))
logger.Infof("chunk path is %s", chunkDst)
err = os.Rename(tmpFn, chunkDst)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("moved chunk from %s to %s", tmpFn, chunkDst)
// Check that all chunks are uploaded.
// This is very inefficient, the server has to check that it has all the
// chunks after each uploaded chunk.
// A two-phase upload like DropBox is better, because the server will
// assembly the chunks when the client asks for it.
fdChunkFolder, err := os.Open(chunkFolder)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer fdChunkFolder.Close()
logger.Infof("opened chunk folder %s", chunkFolder)
fns, err := fdChunkFolder.Readdirnames(-1)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("there are %d out of %d chunks", len(fns), chunkInfo.TotalChunks)
if len(fns) < int(chunkInfo.TotalChunks) {
logger.Infof("chunk upload is not complete")
w.WriteHeader(http.StatusCreated)
return
}
assemblyFn, assemblyFile, err := s.tmpFile()
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer assemblyFile.Close()
logger.Infof("created assembly file at %s", assemblyFn)
for chunk := 0; chunk < int(chunkInfo.TotalChunks); chunk++ {
cp := path.Join(chunkFolder,
strconv.FormatInt(int64(chunk), 10))
logger.Infof("going to process chunk %d with path %s", chunk, cp)
fdChunk, err := os.Open(cp)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer fdChunk.Close()
logger.Infof("opened chunk at %s", cp)
_, err = io.Copy(assemblyFile, fdChunk)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
// close fd now. If we defer it we will have thousands of open fd
// until assembly process
err = fdChunk.Close()
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("copied chunk %s into assembly file %s", cp, assemblyFn)
}
// Now that the chunks are assembled into one larger file
// we can remove the chunk folder to free space
defer func() {
err := os.RemoveAll(chunkFolder)
if err != nil {
log.Error(err)
}
}()
// Point fd to beginning of file to start copying
_, err = assemblyFile.Seek(0, 0)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("assembly file sought to first offset for copy")
// TODO(labkode) instead uploading the file when it is assembled the best is to provide
// a core API like DropBox chunk sessions that is compatible with everyone
c := &http.Client{}
req, err := http.NewRequest("PUT", s.p.dataServer+path.Join("/", chunkInfo.ResourcePath), assemblyFile)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
req.Close = true
req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
req.Header.Add("CIO-Checksum", r.Header.Get("OC-Checksum"))
req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))
res, err := c.Do(req)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
logger.Infof("assembly file %s uploaded to %s", assemblyFn, s.p.dataServer)
defer res.Body.Close()
if res.StatusCode != 201 {
w.WriteHeader(res.StatusCode)
return
}
// The assembled file has been uploaded correctly so
// we can free space
defer func() {
err := os.RemoveAll(assemblyFn)
if err != nil {
log.Error(err)
}
}()
meta, err := getMeta(ctx, s.p.metaServer, chunkInfo.ResourcePath, false)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.NotFound:
http.Error(w, "", http.StatusNotFound)
return
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
// TODO(labkode) Analyze side effects on not respecting OC-Mtime
w.Header().Set("ETag", meta.Etag)
w.Header().Set("OC-FileId", meta.Id)
w.Header().Set("OC-ETag", meta.Etag)
t := time.Unix(int64(meta.Modified), 0)
lastModifiedString := t.Format(time.RFC1123)
w.Header().Set("Last-Modified", lastModifiedString)
w.Header().Set("X-OC-MTime", "accepted")
w.WriteHeader(http.StatusCreated)
}
func (s *server) put(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
p := getPathFromReq(r)
logger.Infof("path is %s", p)
// if client sends etag check against the current value
// if file is not found or etags do not match return 412
ifMatchHeader := r.Header.Get("If-Match")
if ifMatchHeader != "" {
meta, err := getMeta(ctx, s.p.metaServer, p, false)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusPreconditionFailed)
return
}
// TODO(labkode) refactor this
if ifMatchHeader != `"`+meta.Etag+`"` {
logger.Warnf("etags do not match. client send %s and server has %s", ifMatchHeader, meta.Etag)
http.Error(w, "", http.StatusPreconditionFailed)
return
}
}
c := &http.Client{}
req, err := http.NewRequest("PUT", s.p.dataServer+path.Join("/", p), r.Body)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
req.Close = true
req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
req.Header.Add("CIO-Checksum", r.Header.Get("OC-Checksum"))
req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))
res, err := c.Do(req)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer res.Body.Close()
if res.StatusCode != 201 {
w.WriteHeader(res.StatusCode)
return
}
meta, err := getMeta(ctx, s.p.metaServer, p, false)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.NotFound:
http.Error(w, "", http.StatusNotFound)
return
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
w.Header().Set("ETag", meta.Etag)
w.Header().Set("OC-FileId", meta.Id)
w.Header().Set("OC-ETag", meta.Etag)
t := time.Unix(int64(meta.Modified), 0)
lastModifiedString := t.Format(time.RFC1123)
w.Header().Set("Last-Modified", lastModifiedString)
w.Header().Set("X-OC-MTime", "accepted")
w.WriteHeader(http.StatusCreated)
}
func (s *server) get(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
p := getPathFromReq(r)
logger.Infof("path is %s", p)
meta, err := getMeta(ctx, s.p.metaServer, p, false)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.NotFound:
http.Error(w, "", http.StatusNotFound)
return
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
logger.Debugf("meta is %s", meta)
c := &http.Client{}
req, err := http.NewRequest("GET", s.p.dataServer+p, nil)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))
res, err := c.Do(req)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer res.Body.Close()
if res.StatusCode != 200 {
http.Error(w, "", res.StatusCode)
return
}
w.Header().Set("Content-Type", meta.MimeType)
w.Header().Set("ETag", meta.Etag)
w.Header().Set("OC-FileId", meta.Id)
w.Header().Set("OC-ETag", meta.Etag)
t := time.Unix(int64(meta.Modified), 0)
lastModifiedString := t.Format(time.RFC1123)
w.Header().Set("Last-Modified", lastModifiedString)
logger.Infof("file checksum is %s", meta.Checksum)
if meta.Checksum != "" {
w.Header().Set("OC-Checksum", meta.Checksum)
}
io.Copy(w, res.Body)
}
func (s *server) move(ctx context.Context, w http.ResponseWriter, r *http.Request) {
logger := MustFromLogContext(ctx)
src := getPathFromReq(r)
u, err := url.Parse(r.Header.Get("Destination"))
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusBadRequest)
return
}
dst := path.Clean(strings.TrimPrefix(u.Path, remoteURL))
if dst == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
logger.Infof("src is %s", src)
logger.Infof("dst is %s", dst)
con, err := getConnection(s.p.metaServer)
if err != nil {
logger.Error(err)
http.Error(w, "", http.StatusInternalServerError)
return
}
defer con.Close()
client := metapb.NewMetaClient(con)
in := &metapb.MvReq{}
in.Src = src
in.Dst = dst
in.AccessToken = authlib.MustFromTokenContext(ctx)
_, err = client.Mv(ctx, in)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
meta, err := getMeta(ctx, s.p.metaServer, dst, false)
if err != nil {
logger.Error(err)
gErr := grpc.Code(err)
switch {
case gErr == codes.NotFound:
http.Error(w, "", http.StatusNotFound)
return
case gErr == codes.PermissionDenied:
http.Error(w, "", http.StatusForbidden)
return
default:
http.Error(w, "", http.StatusInternalServerError)
return
}
}
w.Header().Set("ETag", meta.Etag)
w.Header().Set("OC-FileId", meta.Id)
w.Header().Set("OC-ETag", meta.Etag)
t := time.Unix(int64(meta.Modified), 0)
lastModifiedString := t.Format(time.RFC1123)
w.Header().Set("Last-Modified", lastModifiedString)
// TODO(labkode) is resource existed (overwrite the code should be 204)
w.WriteHeader(http.StatusCreated)
}