func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) bool) bool { if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil { length := 0 selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port)) results := make(chan bool) for _, location := range lookupResult.Locations { if location.Url != selfUrl { length++ go func(location operation.Location, results chan bool) { results <- op(location) }(location, results) } } ret := true for i := 0; i < length; i++ { ret = ret && <-results } if volume := store.GetVolume(volumeId); volume != nil { if length+1 < volume.ReplicaPlacement.GetCopyCount() { glog.V(0).Infof("replicating opetations [%d] is less than volume's replication copy count [%d]", length+1, volume.ReplicaPlacement.GetCopyCount()) ret = false } } return ret } else { glog.V(0).Infoln("Failed to lookup for", volumeId, lookupErr.Error()) } return false }
func readFiles(fileIdLineChan chan string, s *stat) { defer wait.Done() for fid := range fileIdLineChan { if len(fid) == 0 { continue } if fid[0] == '#' { continue } if *cmdBenchmark.IsDebug { fmt.Printf("reading file %s\n", fid) } parts := strings.SplitN(fid, ",", 2) vid := parts[0] start := time.Now() ret, err := operation.Lookup(*b.server, vid) if err != nil || len(ret.Locations) == 0 { s.failed++ println("!!!! volume id ", vid, " location not found!!!!!") continue } server := ret.Locations[rand.Intn(len(ret.Locations))].Url url := "http://" + server + "/" + fid if bytesRead, err := util.Get(url); err == nil { s.completed++ s.transferred += int64(len(bytesRead)) readStats.addSample(time.Now().Sub(start)) } else { s.failed++ fmt.Printf("Failed to read %s error:%v\n", url, err) } } }
func runBackup(cmd *Command, args []string) bool { if *s.volumeId == -1 { return false } vid := storage.VolumeId(*s.volumeId) // find volume location, replication, ttl info lookup, err := operation.Lookup(*s.master, vid.String()) if err != nil { fmt.Printf("Error looking up volume %d: %v\n", vid, err) return true } volumeServer := lookup.Locations[0].Url stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String()) if err != nil { fmt.Printf("Error get volume %d status: %v\n", vid, err) return true } ttl, err := storage.ReadTTL(stats.Ttl) if err != nil { fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err) return true } replication, err := storage.NewReplicaPlacementFromString(stats.Replication) if err != nil { fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err) return true } v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl) if err != nil { fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) return true } if err := v.Synchronize(volumeServer); err != nil { fmt.Printf("Error synchronizing volume %d: %v\n", vid, err) return true } return true }
func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) bool) bool { if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil { length := 0 selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port)) results := make(chan bool) for _, location := range lookupResult.Locations { if location.Url != selfUrl { length++ go func(location operation.Location, results chan bool) { results <- op(location) }(location, results) } } ret := true for i := 0; i < length; i++ { ret = ret && <-results } return ret } else { glog.V(0).Infoln("Failed to lookup for", volumeId, lookupErr.Error()) } return false }
func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { if strings.HasSuffix(r.URL.Path, "/") { if fs.disableDirListing { w.WriteHeader(http.StatusMethodNotAllowed) return } fs.listDirectoryHandler(w, r) return } fileId, err := fs.filer.FindFile(r.URL.Path) if err == leveldb.ErrNotFound { glog.V(3).Infoln("Not found in db", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } parts := strings.Split(fileId, ",") if len(parts) != 2 { glog.V(1).Infoln("Invalid fileId", fileId) w.WriteHeader(http.StatusNotFound) return } lookup, lookupError := operation.Lookup(fs.master, parts[0]) if lookupError != nil { glog.V(1).Infoln("Invalid lookup", lookupError.Error()) w.WriteHeader(http.StatusNotFound) return } if len(lookup.Locations) == 0 { glog.V(1).Infoln("Can not find location for volume", parts[0]) w.WriteHeader(http.StatusNotFound) return } urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].Url urlString := "http://" + urlLocation + "/" + fileId if fs.redirectOnRead { http.Redirect(w, r, urlString, http.StatusFound) return } u, _ := url.Parse(urlString) request := &http.Request{ Method: r.Method, URL: u, Proto: r.Proto, ProtoMajor: r.ProtoMajor, ProtoMinor: r.ProtoMinor, Header: r.Header, Body: r.Body, Host: r.Host, ContentLength: r.ContentLength, } glog.V(3).Infoln("retrieving from", u) resp, do_err := util.Do(request) if do_err != nil { glog.V(0).Infoln("failing to connect to volume server", do_err.Error()) writeJsonError(w, r, http.StatusInternalServerError, do_err) return } defer resp.Body.Close() for k, v := range resp.Header { w.Header()[k] = v } w.WriteHeader(resp.StatusCode) io.Copy(w, resp.Body) }
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { n := new(storage.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { glog.V(2).Infoln("parsing error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } err = n.ParsePath(fid) if err != nil { glog.V(2).Infoln("parsing fid error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } glog.V(4).Infoln("volume", volumeId, "reading", n) if !vs.store.HasVolume(volumeId) { if !vs.ReadRedirect { glog.V(2).Infoln("volume is not local:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String()) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { http.Redirect(w, r, util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)+r.URL.Path, http.StatusMovedPermanently) } else { glog.V(2).Infoln("lookup error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) } return } cookie := n.Cookie count, e := vs.store.ReadVolumeNeedle(volumeId, n) glog.V(4).Infoln("read bytes", count, "error", e) if e != nil || count <= 0 { glog.V(0).Infoln("read error:", e, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } if n.Cookie != cookie { glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent()) w.WriteHeader(http.StatusNotFound) return } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { if t.Unix() >= int64(n.LastModified) { w.WriteHeader(http.StatusNotModified) return } } } } etag := n.Etag() if inm := r.Header.Get("If-None-Match"); inm == etag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Etag", etag) if vs.tryHandleChunkedFile(n, filename, w, r) { return } if n.NameSize > 0 && filename == "" { filename = string(n.Name) if ext == "" { ext = path.Ext(filename) } } mtype := "" if n.MimeSize > 0 { mt := string(n.Mime) if !strings.HasPrefix(mt, "application/octet-stream") { mtype = mt } } if ext != ".gz" { if n.IsGzipped() { if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { if n.Data, err = operation.UnGzipData(n.Data); err != nil { glog.V(0).Infoln("ungzip error:", err, r.URL.Path) } } } } if ext == ".png" || ext == ".jpg" || ext == ".gif" { width, height := 0, 0 if r.FormValue("width") != "" { width, _ = strconv.Atoi(r.FormValue("width")) } if r.FormValue("height") != "" { height, _ = strconv.Atoi(r.FormValue("height")) } n.Data, _, _ = images.Resized(ext, n.Data, width, height) } if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil { glog.V(2).Infoln("response write error:", e) } }
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { n := new(storage.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { glog.V(2).Infoln("parsing error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } err = n.ParsePath(fid) if err != nil { glog.V(2).Infoln("parsing fid error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } glog.V(4).Infoln("volume", volumeId, "reading", n) if !vs.store.HasVolume(volumeId) { lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String()) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { http.Redirect(w, r, util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)+r.URL.Path, http.StatusMovedPermanently) } else { glog.V(2).Infoln("lookup error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) } return } cookie := n.Cookie count, e := vs.store.ReadVolumeNeedle(volumeId, n) glog.V(4).Infoln("read bytes", count, "error", e) if e != nil || count <= 0 { glog.V(0).Infoln("read error:", e, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } if n.Cookie != cookie { glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent()) w.WriteHeader(http.StatusNotFound) return } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { if t.Unix() >= int64(n.LastModified) { w.WriteHeader(http.StatusNotModified) return } } } } etag := n.Etag() if inm := r.Header.Get("If-None-Match"); inm == etag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Etag", etag) if n.NameSize > 0 && filename == "" { filename = string(n.Name) dotIndex := strings.LastIndex(filename, ".") if dotIndex > 0 { ext = filename[dotIndex:] } } mtype := "" if ext != "" { mtype = mime.TypeByExtension(ext) } if n.MimeSize > 0 { mt := string(n.Mime) if !strings.HasPrefix(mt, "application/octet-stream") { mtype = mt } } if mtype != "" { w.Header().Set("Content-Type", mtype) } if filename != "" { w.Header().Set("Content-Disposition", "filename=\""+fileNameEscaper.Replace(filename)+"\"") } if ext != ".gz" { if n.IsGzipped() { if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { if n.Data, err = storage.UnGzipData(n.Data); err != nil { glog.V(0).Infoln("lookup error:", err, r.URL.Path) } } } } if ext == ".png" || ext == ".jpg" || ext == ".gif" { width, height := 0, 0 if r.FormValue("width") != "" { width, _ = strconv.Atoi(r.FormValue("width")) } if r.FormValue("height") != "" { height, _ = strconv.Atoi(r.FormValue("height")) } n.Data, _, _ = images.Resized(ext, n.Data, width, height) } w.Header().Set("Accept-Ranges", "bytes") if r.Method == "HEAD" { w.Header().Set("Content-Length", strconv.Itoa(len(n.Data))) return } rangeReq := r.Header.Get("Range") if rangeReq == "" { w.Header().Set("Content-Length", strconv.Itoa(len(n.Data))) if _, e = w.Write(n.Data); e != nil { glog.V(0).Infoln("response write error:", e) } return } //the rest is dealing with partial content request //mostly copy from src/pkg/net/http/fs.go size := int64(len(n.Data)) ranges, err := parseRange(rangeReq, size) if err != nil { http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) return } if sumRangesSize(ranges) > size { // The total number of bytes in all the ranges // is larger than the size of the file by // itself, so this is probably an attack, or a // dumb client. Ignore the range request. ranges = nil return } if len(ranges) == 0 { return } if len(ranges) == 1 { // RFC 2616, Section 14.16: // "When an HTTP message includes the content of a single // range (for example, a response to a request for a // single range, or to a request for a set of ranges // that overlap without any holes), this content is // transmitted with a Content-Range header, and a // Content-Length header showing the number of bytes // actually transferred. // ... // A response to a request for a single range MUST NOT // be sent using the multipart/byteranges media type." ra := ranges[0] w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10)) w.Header().Set("Content-Range", ra.contentRange(size)) w.WriteHeader(http.StatusPartialContent) if _, e = w.Write(n.Data[ra.start : ra.start+ra.length]); e != nil { glog.V(0).Infoln("response write error:", e) } return } // process mulitple ranges for _, ra := range ranges { if ra.start > size { http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable) return } } sendSize := rangesMIMESize(ranges, mtype, size) pr, pw := io.Pipe() mw := multipart.NewWriter(pw) w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) sendContent := pr defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. go func() { for _, ra := range ranges { part, err := mw.CreatePart(ra.mimeHeader(mtype, size)) if err != nil { pw.CloseWithError(err) return } if _, err = part.Write(n.Data[ra.start : ra.start+ra.length]); err != nil { pw.CloseWithError(err) return } } mw.Close() pw.Close() }() if w.Header().Get("Content-Encoding") == "" { w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) } w.WriteHeader(http.StatusPartialContent) io.CopyN(w, sendContent, sendSize) }