func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
if !n.IsChunkedManifest() {
return false
}
chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
if e != nil {
glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e)
return false
}
if fileName == "" && chunkManifest.Name != "" {
fileName = chunkManifest.Name
}
mType := ""
if chunkManifest.Mime != "" {
mt := chunkManifest.Mime
if !strings.HasPrefix(mt, "application/octet-stream") {
mType = mt
}
}
w.Header().Set("X-File-Store", "chunked")
chunkedFileReader := &operation.ChunkedFileReader{
Manifest: chunkManifest,
Master: vs.GetMasterNode(),
}
defer chunkedFileReader.Close()
if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil {
glog.V(2).Infoln("response write error:", e)
}
return true
}
func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
n := new(storage.Needle)
vid, fid, _, _, _ := parseURLPath(r.URL.Path)
volumeId, _ := storage.NewVolumeId(vid)
n.ParsePath(fid)
glog.V(2).Infoln("deleting", n)
cookie := n.Cookie
_, ok := vs.store.ReadVolumeNeedle(volumeId, n)
if ok != nil {
m := make(map[string]uint32)
m["size"] = 0
writeJsonQuiet(w, r, http.StatusNotFound, m)
return
}
defer n.ReleaseMemory()
if n.Cookie != cookie {
glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
return
}
count := int64(n.Size)
if n.IsChunkedManifest() {
chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
if e != nil {
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
return
}
// make sure all chunks had deleted before delete manifest
if e := chunkManifest.DeleteChunks(vs.GetMasterNode()); e != nil {
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
return
}
count = chunkManifest.Size
}
_, err := topology.ReplicatedDelete(vs.GetMasterNode(), vs.store, volumeId, n, r)
if err == nil {
m := make(map[string]int64)
m["size"] = count
writeJsonQuiet(w, r, http.StatusAccepted, m)
} else {
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
}
}
func ReplicatedWrite(masterNode string, s *storage.Store,
volumeId storage.VolumeId, needle *storage.Needle,
r *http.Request) (size uint32, errorStatus string) {
//check JWT
jwt := security.GetJwt(r)
ret, err := s.Write(volumeId, needle)
needToReplicate := !s.HasVolume(volumeId)
if err != nil {
errorStatus = "Failed to write to local disk (" + err.Error() + ")"
} else if ret > 0 {
needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
} else {
errorStatus = "Failed to write to local disk"
}
if !needToReplicate && ret > 0 {
needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
}
if needToReplicate { //send to other replica locations
if r.FormValue("type") != "replicate" {
if err = distributedOperation(masterNode, s, volumeId, func(location operation.Location) error {
u := url.URL{
Scheme: "http",
Host: location.Url,
Path: r.URL.Path,
}
q := url.Values{
"type": {"replicate"},
}
if needle.LastModified > 0 {
q.Set("ts", strconv.FormatUint(needle.LastModified, 10))
}
if needle.IsChunkedManifest() {
q.Set("cm", "true")
}
u.RawQuery = q.Encode()
_, err := operation.Upload(u.String(),
string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
jwt)
return err
}); err != nil {
ret = 0
errorStatus = fmt.Sprintf("Failed to write to replicas for volume %d: %v", volumeId, err)
}
}
}
size = ret
return
}
func ReplicatedWrite(masterNode string, s *storage.Store,
volumeId storage.VolumeId, needle *storage.Needle,
r *http.Request) (size uint32, errorStatus string) {
//check JWT
jwt := security.GetJwt(r)
defer func() {
if errorStatus == "" {
return
}
ReplicatedDelete(masterNode, s, volumeId, needle, r)
}()
ret, err := s.Write(volumeId, needle)
if err != nil {
errorStatus = "Failed to write to local disk (" + err.Error() + ")"
} else if ret <= 0 {
errorStatus = "Failed to write to local disk"
}
//send to other replica locations
if r.FormValue("type") != "replicate" {
repWrite := func(location operation.Location) bool {
args := url.Values{
"type": {"replicate"},
}
if needle.LastModified > 0 {
args.Set("ts", strconv.FormatUint(needle.LastModified, 10))
}
if needle.IsChunkedManifest() {
args.Set("cm", "true")
}
u := util.MkUrl(location.Url, r.URL.Path, args)
glog.V(4).Infoln("write replication to", u)
_, err := operation.Upload(u,
string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
jwt)
if err != nil {
glog.V(0).Infof("write replication to %s err, %v", u, err)
}
return err == nil
}
if !distributedOperation(masterNode, s, volumeId, repWrite) {
ret = 0
errorStatus = "Failed to write to replicas for volume " + volumeId.String()
}
}
size = ret
return
}
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
var ret []operation.DeleteResult
for _, fid := range r.Form["fid"] {
vid, id_cookie, err := operation.ParseFileId(fid)
if err != nil {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusBadRequest,
Error: err.Error()})
continue
}
n := new(storage.Needle)
volumeId, _ := storage.NewVolumeId(vid)
n.ParsePath(id_cookie)
glog.V(4).Infoln("batch deleting", n)
cookie := n.Cookie
if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusNotFound,
Error: err.Error(),
})
continue
}
if n.IsChunkedManifest() {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusNotAcceptable,
Error: "ChunkManifest: not allowed in batch delete mode.",
})
n.ReleaseMemory()
continue
}
if n.Cookie != cookie {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusBadRequest,
Error: "File Random Cookie does not match.",
})
glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
n.ReleaseMemory()
return
}
if size, err := vs.store.Delete(volumeId, n); err != nil {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusInternalServerError,
Error: err.Error()},
)
} else {
ret = append(ret, operation.DeleteResult{
Fid: fid,
Status: http.StatusAccepted,
Size: int(size)},
)
}
n.ReleaseMemory()
}
writeJsonQuiet(w, r, http.StatusAccepted, ret)
}