Пример #1
0
func (s *server) copy(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	src := getPathFromReq(r)
	u, err := url.Parse(r.Header.Get("Destination"))
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	dst := path.Clean(strings.TrimPrefix(u.Path, remoteURL))
	if dst == "" {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	logger.Infof("src is %s", src)
	logger.Infof("dst is %s", dst)

	con, err := getConnection(s.p.metaServer)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer con.Close()

	client := metapb.NewMetaClient(con)

	in := &metapb.CpReq{}
	in.Src = src
	in.Dst = dst
	in.AccessToken = authlib.MustFromTokenContext(ctx)

	_, err = client.Cp(ctx, in)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	w.WriteHeader(http.StatusNoContent)
}
Пример #2
0
func (s *server) mkcol(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	p := getPathFromReq(r)

	logger.Infof("path is %s", p)

	con, err := getConnection(s.p.metaServer)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer con.Close()

	client := metapb.NewMetaClient(con)

	in := &metapb.MkdirReq{}
	in.AccessToken = authlib.MustFromTokenContext(ctx)
	in.Path = p

	_, err = client.Mkdir(ctx, in)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	w.WriteHeader(http.StatusCreated)
}
Пример #3
0
func getMeta(ctx context.Context, addr, p string, children bool) (*metapb.Metadata, error) {

	in := &metapb.StatReq{}
	in.AccessToken = authlib.MustFromTokenContext(ctx)
	in.Children = children
	in.Path = p

	con, err := getConnection(addr)
	if err != nil {
		return nil, err
	}

	defer con.Close()

	client := metapb.NewMetaClient(con)

	meta, err := client.Stat(ctx, in)
	if err != nil {
		return nil, err
	}

	return meta, nil
}
Пример #4
0
func (s *server) upload(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	log := MustFromLogContext(ctx)
	p := lib.MustFromContext(ctx)

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	tmpFn, tmpFile, err := s.tmpFile()
	if err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	log.Infof("created tmp file %s", tmpFn)

	var mw io.Writer
	var hasher hash.Hash
	var isChecksumed bool
	var computedChecksum string

	switch s.p.checksum {
	case "md5":
		hasher = md5.New()
		isChecksumed = true
		mw = io.MultiWriter(tmpFile, hasher)
	case "sha1":
		hasher = sha1.New()
		isChecksumed = true
		mw = io.MultiWriter(tmpFile, hasher)
	case "adler32":
		hasher = adler32.New()
		isChecksumed = true
		mw = io.MultiWriter(tmpFile, hasher)
	default:
		mw = io.MultiWriter(tmpFile)
	}

	// TODO(labkode) Sometimes ContentLength = -1 because it is a binary
	// upload with TransferEncoding: chunked.
	// Instead using Copy we shoudl use a LimitedReader with a max file upload
	// configuration value.
	_, err = io.Copy(mw, r.Body)
	if err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	chk := s.getChecksumInfo(r)

	if isChecksumed {
		log.Infof("file sent with checksum %s", chk.String())

		// checksums are given in hexadecimal format.
		computedChecksum = fmt.Sprintf("%x", string(hasher.Sum(nil)))

		if chk.Type == s.p.checksum && chk.Type != "" {

			isCorrupted := computedChecksum != chk.Sum

			if isCorrupted {
				log.Errorf("corrupted file. expected %s and got %s",
					s.p.checksum+":"+computedChecksum, chk.Sum)
				http.Error(w, "", http.StatusPreconditionFailed)
				return
			}
		}
	}

	log.Infof("copied r.Body into tmp file %s", tmpFn)

	err = tmpFile.Close()
	if err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	log.Infof("closed tmp file %s", tmpFn)

	if err = os.Rename(tmpFn, pp); err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	log.Infof("renamed tmp file %s to %s", tmpFn, pp)

	con, err := grpc.Dial(s.p.prop, grpc.WithInsecure())
	if err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	defer con.Close()

	log.Infof("created connection to %s", s.p.prop)

	client := pb.NewPropClient(con)

	in := &pb.PutReq{}
	in.Path = p
	in.AccessToken = authlib.MustFromTokenContext(ctx)
	in.Checksum = chk.String()

	_, err = client.Put(ctx, in)
	if err != nil {
		log.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	log.Infof("saved path %s into %s", p, s.p.prop)

	w.WriteHeader(http.StatusCreated)
}
Пример #5
0
func (s *server) putChunked(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	p := getPathFromReq(r)

	logger.Infof("path is %s", p)

	chunkInfo, err := getChunkPathInfo(p)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	// if client sends etag check against the current value
	// if file is not found or etags do not match return 412
	ifMatchHeader := r.Header.Get("If-Match")
	if ifMatchHeader != "" {

		meta, err := getMeta(ctx, s.p.metaServer, chunkInfo.ResourcePath, false)
		if err != nil {
			logger.Error(err)
			http.Error(w, "", http.StatusPreconditionFailed)
			return
		}

		// TODO(labkode) refactor this
		if ifMatchHeader != `"`+meta.Etag+`"` {
			logger.Warnf("etags do not match. client send %s and server has %s", ifMatchHeader, meta.Etag)
			http.Error(w, "", http.StatusPreconditionFailed)
			return
		}

	}

	logger.Infof("%s", chunkInfo.String())

	tmpFn, tmpFile, err := s.tmpFile()
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}
	defer tmpFile.Close()

	logger.Infof("created tmp file %s", tmpFn)

	_, err = io.Copy(tmpFile, r.Body)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("copied r.Body to %s", tmpFn)

	err = tmpFile.Close()
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("closed %s", tmpFn)

	chunkFolder, err := s.getChunkFolder(chunkInfo)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("chunk folder is %s", chunkFolder)

	chunkDst := path.Join(chunkFolder, path.Clean(strconv.FormatUint(chunkInfo.CurrentChunk, 10)))

	logger.Infof("chunk path is %s", chunkDst)

	err = os.Rename(tmpFn, chunkDst)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("moved chunk from %s to %s", tmpFn, chunkDst)

	// Check that all chunks are uploaded.
	// This is very inefficient, the server has to check that it has all the
	// chunks after each uploaded chunk.
	// A two-phase upload like DropBox is better, because the server will
	// assembly the chunks when the client asks for it.

	fdChunkFolder, err := os.Open(chunkFolder)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer fdChunkFolder.Close()

	logger.Infof("opened chunk folder %s", chunkFolder)

	fns, err := fdChunkFolder.Readdirnames(-1)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("there are %d out of %d chunks", len(fns), chunkInfo.TotalChunks)

	if len(fns) < int(chunkInfo.TotalChunks) {
		logger.Infof("chunk upload is not complete")
		w.WriteHeader(http.StatusCreated)
		return
	}

	assemblyFn, assemblyFile, err := s.tmpFile()
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer assemblyFile.Close()

	logger.Infof("created assembly file at %s", assemblyFn)

	for chunk := 0; chunk < int(chunkInfo.TotalChunks); chunk++ {

		cp := path.Join(chunkFolder,
			strconv.FormatInt(int64(chunk), 10))

		logger.Infof("going to process chunk %d with path %s", chunk, cp)

		fdChunk, err := os.Open(cp)
		if err != nil {
			logger.Error(err)
			http.Error(w, "", http.StatusInternalServerError)
			return
		}

		defer fdChunk.Close()
		logger.Infof("opened chunk at %s", cp)

		_, err = io.Copy(assemblyFile, fdChunk)
		if err != nil {
			logger.Error(err)
			http.Error(w, "", http.StatusInternalServerError)
			return
		}

		// close fd now. If we defer it we will have thousands of open fd
		// until assembly process
		err = fdChunk.Close()
		if err != nil {
			logger.Error(err)
			http.Error(w, "", http.StatusInternalServerError)
			return
		}

		logger.Infof("copied chunk %s into assembly file %s", cp, assemblyFn)
	}

	// Now that the chunks are assembled into one larger file
	// we can remove the chunk folder to free space
	defer func() {
		err := os.RemoveAll(chunkFolder)
		if err != nil {
			log.Error(err)
		}
	}()

	// Point fd to beginning of file to start copying
	_, err = assemblyFile.Seek(0, 0)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("assembly file sought to first offset for copy")

	// TODO(labkode) instead uploading the file when it is assembled the best is to provide
	// a core API like DropBox chunk sessions that is compatible with everyone
	c := &http.Client{}
	req, err := http.NewRequest("PUT", s.p.dataServer+path.Join("/", chunkInfo.ResourcePath), assemblyFile)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	req.Close = true
	req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
	req.Header.Add("CIO-Checksum", r.Header.Get("OC-Checksum"))
	req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))

	res, err := c.Do(req)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	logger.Infof("assembly file %s uploaded to %s", assemblyFn, s.p.dataServer)

	defer res.Body.Close()

	if res.StatusCode != 201 {
		w.WriteHeader(res.StatusCode)
		return
	}

	// The assembled file has been uploaded correctly so
	// we can free space
	defer func() {
		err := os.RemoveAll(assemblyFn)
		if err != nil {
			log.Error(err)
		}
	}()

	meta, err := getMeta(ctx, s.p.metaServer, chunkInfo.ResourcePath, false)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.NotFound:
			http.Error(w, "", http.StatusNotFound)
			return
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	// TODO(labkode) Analyze side effects on not respecting OC-Mtime
	w.Header().Set("ETag", meta.Etag)
	w.Header().Set("OC-FileId", meta.Id)
	w.Header().Set("OC-ETag", meta.Etag)
	t := time.Unix(int64(meta.Modified), 0)
	lastModifiedString := t.Format(time.RFC1123)
	w.Header().Set("Last-Modified", lastModifiedString)
	w.Header().Set("X-OC-MTime", "accepted")

	w.WriteHeader(http.StatusCreated)
}
Пример #6
0
func (s *server) put(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	p := getPathFromReq(r)

	logger.Infof("path is %s", p)

	// if client sends etag check against the current value
	// if file is not found or etags do not match return 412
	ifMatchHeader := r.Header.Get("If-Match")
	if ifMatchHeader != "" {

		meta, err := getMeta(ctx, s.p.metaServer, p, false)
		if err != nil {
			logger.Error(err)
			http.Error(w, "", http.StatusPreconditionFailed)
			return
		}

		// TODO(labkode) refactor this
		if ifMatchHeader != `"`+meta.Etag+`"` {
			logger.Warnf("etags do not match. client send %s and server has %s", ifMatchHeader, meta.Etag)
			http.Error(w, "", http.StatusPreconditionFailed)
			return
		}

	}

	c := &http.Client{}
	req, err := http.NewRequest("PUT", s.p.dataServer+path.Join("/", p), r.Body)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	req.Close = true
	req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
	req.Header.Add("CIO-Checksum", r.Header.Get("OC-Checksum"))
	req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))

	res, err := c.Do(req)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer res.Body.Close()

	if res.StatusCode != 201 {
		w.WriteHeader(res.StatusCode)
		return
	}

	meta, err := getMeta(ctx, s.p.metaServer, p, false)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.NotFound:
			http.Error(w, "", http.StatusNotFound)
			return
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	w.Header().Set("ETag", meta.Etag)
	w.Header().Set("OC-FileId", meta.Id)
	w.Header().Set("OC-ETag", meta.Etag)
	t := time.Unix(int64(meta.Modified), 0)
	lastModifiedString := t.Format(time.RFC1123)
	w.Header().Set("Last-Modified", lastModifiedString)
	w.Header().Set("X-OC-MTime", "accepted")

	w.WriteHeader(http.StatusCreated)

}
Пример #7
0
func (s *server) get(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	p := getPathFromReq(r)

	logger.Infof("path is %s", p)

	meta, err := getMeta(ctx, s.p.metaServer, p, false)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.NotFound:
			http.Error(w, "", http.StatusNotFound)
			return
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	logger.Debugf("meta is %s", meta)

	c := &http.Client{}
	req, err := http.NewRequest("GET", s.p.dataServer+p, nil)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	req.Header.Add("Authorization", "Bearer "+authlib.MustFromTokenContext(ctx))
	req.Header.Add("CIO-TraceID", logger.Data["trace"].(string))

	res, err := c.Do(req)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer res.Body.Close()

	if res.StatusCode != 200 {
		http.Error(w, "", res.StatusCode)
		return
	}

	w.Header().Set("Content-Type", meta.MimeType)
	w.Header().Set("ETag", meta.Etag)
	w.Header().Set("OC-FileId", meta.Id)
	w.Header().Set("OC-ETag", meta.Etag)
	t := time.Unix(int64(meta.Modified), 0)
	lastModifiedString := t.Format(time.RFC1123)
	w.Header().Set("Last-Modified", lastModifiedString)

	logger.Infof("file checksum is %s", meta.Checksum)
	if meta.Checksum != "" {
		w.Header().Set("OC-Checksum", meta.Checksum)
	}

	io.Copy(w, res.Body)
}
Пример #8
0
func (s *server) move(ctx context.Context, w http.ResponseWriter, r *http.Request) {

	logger := MustFromLogContext(ctx)

	src := getPathFromReq(r)
	u, err := url.Parse(r.Header.Get("Destination"))
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	dst := path.Clean(strings.TrimPrefix(u.Path, remoteURL))
	if dst == "" {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	logger.Infof("src is %s", src)
	logger.Infof("dst is %s", dst)

	con, err := getConnection(s.p.metaServer)
	if err != nil {
		logger.Error(err)
		http.Error(w, "", http.StatusInternalServerError)
		return
	}

	defer con.Close()

	client := metapb.NewMetaClient(con)

	in := &metapb.MvReq{}
	in.Src = src
	in.Dst = dst
	in.AccessToken = authlib.MustFromTokenContext(ctx)

	_, err = client.Mv(ctx, in)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	meta, err := getMeta(ctx, s.p.metaServer, dst, false)
	if err != nil {
		logger.Error(err)

		gErr := grpc.Code(err)
		switch {
		case gErr == codes.NotFound:
			http.Error(w, "", http.StatusNotFound)
			return
		case gErr == codes.PermissionDenied:
			http.Error(w, "", http.StatusForbidden)
			return
		default:
			http.Error(w, "", http.StatusInternalServerError)
			return
		}
	}

	w.Header().Set("ETag", meta.Etag)
	w.Header().Set("OC-FileId", meta.Id)
	w.Header().Set("OC-ETag", meta.Etag)
	t := time.Unix(int64(meta.Modified), 0)
	lastModifiedString := t.Format(time.RFC1123)
	w.Header().Set("Last-Modified", lastModifiedString)

	// TODO(labkode) is resource existed (overwrite the code should be 204)
	w.WriteHeader(http.StatusCreated)
}