Exemplo n.º 1
0
func upload(filename string, server string, fid string) (int, error) {
	debug("Start uploading file:", filename)
	fh, err := os.Open(filename)
	if err != nil {
		debug("Failed to open file:", filename)
		return 0, err
	}
	ret, e := operation.Upload("http://"+server+"/"+fid, path.Base(filename), fh)
	if e != nil {
		return 0, e
	}
	return ret.Size, e
}
Exemplo n.º 2
0
func PostHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	vid, _, _ := parseURLPath(r.URL.Path)
	volumeId, e := storage.NewVolumeId(vid)
	if e != nil {
		writeJson(w, r, e)
	} else {
		needle, filename, ne := storage.NewNeedle(r)
		if ne != nil {
			writeJson(w, r, ne)
		} else {
			ret, err := store.Write(volumeId, needle)
			errorStatus := ""
			needToReplicate := !store.HasVolume(volumeId)
			if err != nil {
				errorStatus = "Failed to write to local disk (" + err.Error() + ")"
			} else if ret > 0 {
				needToReplicate = needToReplicate || store.GetVolume(volumeId).NeedToReplicate()
			} else {
				errorStatus = "Failed to write to local disk"
			}
			if !needToReplicate && ret > 0 {
				needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
			}
			if needToReplicate { //send to other replica locations
				if r.FormValue("type") != "standard" {
					if !distributedOperation(volumeId, func(location operation.Location) bool {
						_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=standard", filename, bytes.NewReader(needle.Data))
						return err == nil
					}) {
						ret = 0
						errorStatus = "Failed to write to replicas for volume " + volumeId.String()
					}
				}
			}
			m := make(map[string]interface{})
			if errorStatus == "" {
				w.WriteHeader(http.StatusCreated)
			} else {
				store.Delete(volumeId, needle)
				distributedOperation(volumeId, func(location operation.Location) bool {
					return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard")
				})
				w.WriteHeader(http.StatusInternalServerError)
				m["error"] = errorStatus
			}
			m["size"] = ret
			writeJson(w, r, m)
		}
	}
}
Exemplo n.º 3
0
func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		m["error"] = "Only submit via POST!"
		writeJsonQuiet(w, r, m)
		return
	}

	debug("parsing upload file...")
	fname, data, mimeType, isGzipped, lastModified, pe := storage.ParseUpload(r)
	if pe != nil {
		writeJsonError(w, r, pe)
		return
	}

	debug("assigning file id for", fname)
	assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"))
	if ae != nil {
		writeJsonError(w, r, ae)
		return
	}

	url := "http://" + assignResult.PublicUrl + "/" + assignResult.Fid
	if lastModified != 0 {
		url = url + "?ts=" + strconv.FormatUint(lastModified, 10)
	}

	debug("upload file to store", url)
	uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}

	m["fileName"] = fname
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = uploadResult.Size
	writeJsonQuiet(w, r, m)
	return
}
Exemplo n.º 4
0
func upload(filename string, server string, fid string) (int, error) {
	debug("Start uploading file:", filename)
	fh, err := os.Open(filename)
	if err != nil {
		debug("Failed to open file:", filename)
		return 0, err
	}
	fi, fiErr := fh.Stat()
	if fiErr != nil {
		debug("Failed to stat file:", filename)
		return 0, fiErr
	}
	filename = path.Base(filename)
	isGzipped := path.Ext(filename) == ".gz"
	if isGzipped {
		filename = filename[0 : len(filename)-3]
	}
	ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), filename, fh, isGzipped)
	if e != nil {
		return 0, e
	}
	return ret.Size, e
}
Exemplo n.º 5
0
func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) {
	ret, err := s.Write(volumeId, needle)
	needToReplicate := !s.HasVolume(volumeId)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret > 0 {
		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
	} else {
		errorStatus = "Failed to write to local disk"
	}
	if !needToReplicate && ret > 0 {
		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "standard" {
			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=standard", string(needle.Name), bytes.NewReader(needle.Data))
				return err == nil
			}) {
				ret = 0
				errorStatus = "Failed to write to replicas for volume " + volumeId.String()
			}
		}
	}
	if errorStatus != "" {
		if _, err = s.Delete(volumeId, needle); err != nil {
			errorStatus += "\nCannot delete " + strconv.FormatUint(needle.Id, 10) + " from " +
				volumeId.String() + ": " + err.Error()
		} else {
			distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
				return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard")
			})
		}
	}
	size = ret
	return
}