func readFiles(fileIdLineChan chan string, s *stat) { defer wait.Done() for fid := range fileIdLineChan { if len(fid) == 0 { continue } if fid[0] == '#' { continue } if *cmdBenchmark.IsDebug { fmt.Printf("reading file %s\n", fid) } parts := strings.SplitN(fid, ",", 2) vid := parts[0] start := time.Now() ret, err := operation.Lookup(*b.server, vid, "") if err != nil || len(ret.Locations) == 0 { s.failed++ println("!!!! volume id ", vid, " location not found!!!!!") continue } server := ret.Locations.PickForRead().Url if bytesRead, err := util.Get(server, "/"+fid, nil); err == nil { s.completed++ s.transferred += int64(len(bytesRead)) readStats.addSample(time.Now().Sub(start)) } else { s.failed++ fmt.Printf("Failed to read %s/%s error:%v\n", server, fid, err) } } }
func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) error) error { if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil { length := 0 selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port)) results := make(chan RemoteResult) for _, location := range lookupResult.Locations { if location.Url != selfUrl { length++ go func(location operation.Location, results chan RemoteResult) { results <- RemoteResult{location.Url, op(location)} }(location, results) } } ret := DistributedOperationResult(make(map[string]error)) for i := 0; i < length; i++ { result := <-results ret[result.Host] = result.Error } if volume := store.GetVolume(volumeId); volume != nil { if length+1 < volume.ReplicaPlacement.GetCopyCount() { return fmt.Errorf("replicating opetations [%d] is less than volume's replication copy count [%d]", length+1, volume.ReplicaPlacement.GetCopyCount()) } } return ret.Error() } else { glog.V(0).Infoln() return fmt.Errorf("Failed to lookup for %d: %v", volumeId, lookupErr) } return nil }
func runBackup(cmd *Command, args []string) bool { if *s.volumeId == -1 { return false } vid := storage.VolumeId(*s.volumeId) // find volume location, replication, ttl info lookup, err := operation.Lookup(*s.master, vid.String()) if err != nil { fmt.Printf("Error looking up volume %d: %v\n", vid, err) return true } volumeServer := lookup.Locations[0].Url stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String()) if err != nil { fmt.Printf("Error get volume %d status: %v\n", vid, err) return true } ttl, err := storage.ReadTTL(stats.Ttl) if err != nil { fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err) return true } replication, err := storage.NewReplicaPlacementFromString(stats.Replication) if err != nil { fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err) return true } v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl) if err != nil { fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) return true } if err := v.Synchronize(volumeServer); err != nil { fmt.Printf("Error synchronizing volume %d: %v\n", vid, err) return true } return true }
func (s *Store) ReadRemoteNeedle(fid *FileId, collection string) (*Needle, error) { cacheKey := fid.String() if cn, cacheHit := s.needleCache.Get(cacheKey); cacheHit { glog.V(2).Infoln("Remote needle cache hit:", fid) return cn.(*Needle), nil } glog.V(2).Infoln("Remote needle cache miss:", fid) vid := fid.VolumeId.String() lookupResult, err := operation.Lookup(s.GetMaster(), vid, collection) glog.V(2).Infoln("volume", vid, "found on", lookupResult, "error", err) if err != nil || len(lookupResult.Locations) == 0 { return nil, errors.New("lookup error:" + err.Error()) } u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations.PickForRead().Url)) u.Path = "/admin/sync/needle" args := url.Values{ "volume": {vid}, "nid": {fid.Nid()}, } u.RawQuery = args.Encode() req, _ := http.NewRequest("GET", u.String(), nil) resp, err := util.HttpDo(req) if err != nil { return nil, err } defer resp.Body.Close() var buf []byte if buf, err = ioutil.ReadAll(resp.Body); err != nil { return nil, err } if resp.StatusCode != http.StatusOK { errMsg := strconv.Itoa(resp.StatusCode) m := map[string]string{} if e := json.Unmarshal(buf, &m); e == nil { if s, ok := m["error"]; ok { errMsg += ", " + s } } return nil, errors.New(errMsg) } n := &Needle{ Cookie: fid.Cookie, Id: fid.Key, } n.Data = buf n.DataSize = uint32(len(n.Data)) if h := resp.Header.Get("Seaweed-Flags"); h != "" { if i, err := strconv.ParseInt(h, 16, 64); err == nil { n.Flags = byte(i) } } if h := resp.Header.Get("Seaweed-Checksum"); h != "" { if i, err := strconv.ParseInt(h, 16, 64); err == nil { n.Checksum = CRC(i) newChecksum := NewCRC(n.Data) if n.Checksum != newChecksum { return nil, fmt.Errorf("CRC error! Read remote data corrupted (%x!=%x), fid=%v", n.Checksum, newChecksum, fid.String()) } } } if h := resp.Header.Get("Seaweed-LastModified"); h != "" { if i, err := strconv.ParseUint(h, 16, 64); err == nil { n.LastModified = i n.SetHasLastModifiedDate() } } if h := resp.Header.Get("Seaweed-Name"); h != "" { n.Name = []byte(h) n.SetHasName() } if h := resp.Header.Get("Seaweed-Mime"); h != "" { n.Mime = []byte(h) n.SetHasMime() } s.needleCache.Add(cacheKey, n) return n, nil }
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { vid, nid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { glog.V(2).Infoln("parsing error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } var ( n *storage.Needle ) fid, err := storage.NewFileIdFromNid(vid, nid) if err != nil { glog.V(2).Infoln("parsing fid error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } glog.V(4).Infoln("volume", volumeId, "reading", n) if vs.store.HasVolume(volumeId) { n, err = vs.store.ReadLocalNeedle(fid) glog.V(4).Infoln("read local needle", fid, "error", err) if err != nil { glog.V(0).Infoln("read local error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } } else if vs.ReadRemoteNeedle { n, err = vs.store.ReadRemoteNeedle(fid, r.FormValue("collection")) glog.V(4).Infoln("read remote needle", fid, "error", err) if err != nil { glog.V(0).Infoln("read remote error:", err, ",url path:", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } } else if vs.ReadRedirect { lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String(), r.FormValue("collection")) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations.PickForRead().PublicUrl)) u.Path = r.URL.Path http.Redirect(w, r, u.String(), http.StatusMovedPermanently) } else { glog.V(2).Infoln("lookup error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) } return } else { glog.V(2).Infoln("volume is not local:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { if t.Unix() >= int64(n.LastModified) { w.WriteHeader(http.StatusNotModified) return } } } } etag := n.Etag() if inm := r.Header.Get("If-None-Match"); inm == etag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Etag", etag) if vs.tryHandleChunkedFile(volumeId, n, filename, w, r) { return } if n.NameSize > 0 && filename == "" { filename = string(n.Name) if ext == "" { ext = path.Ext(filename) } } mtype := "" if n.MimeSize > 0 { mt := string(n.Mime) if !strings.HasPrefix(mt, "application/octet-stream") { mtype = mt } } needleData := n.Data if ext != ".gz" { if n.IsGzipped() { if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { if needleData, err = operation.UnGzipData(needleData); err != nil { glog.V(0).Infoln("ungzip error:", err, r.URL.Path) } } } } if ext == ".png" || ext == ".jpg" || ext == ".gif" { width, height := 0, 0 if r.FormValue("width") != "" { width, _ = strconv.Atoi(r.FormValue("width")) } if r.FormValue("height") != "" { height, _ = strconv.Atoi(r.FormValue("height")) } if needleData, _, _, err = images.Resized(ext, needleData, width, height); err != nil { glog.V(0).Infoln("resize image error,", err, r.URL.Path) } } if err = writeResponseContent(filename, mtype, bytes.NewReader(needleData), w, r); err != nil { glog.V(2).Infoln("response write error:", err, r.URL.Path) } }
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { n := new(storage.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) if err != nil { glog.V(2).Infoln("parsing error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } err = n.ParsePath(fid) if err != nil { glog.V(2).Infoln("parsing fid error:", err, r.URL.Path) w.WriteHeader(http.StatusBadRequest) return } glog.V(4).Infoln("volume", volumeId, "reading", n) if !vs.store.HasVolume(volumeId) { if !vs.ReadRedirect { glog.V(2).Infoln("volume is not local:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String()) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)) u.Path = r.URL.Path arg := url.Values{} if c := r.FormValue("collection"); c != "" { arg.Set("collection", c) } u.RawQuery = arg.Encode() http.Redirect(w, r, u.String(), http.StatusMovedPermanently) } else { glog.V(2).Infoln("lookup error:", err, r.URL.Path) w.WriteHeader(http.StatusNotFound) } return } cookie := n.Cookie count, e := vs.store.ReadVolumeNeedle(volumeId, n) glog.V(4).Infoln("read bytes", count, "error", e) if e != nil || count <= 0 { glog.V(0).Infoln("read error:", e, r.URL.Path) w.WriteHeader(http.StatusNotFound) return } defer n.ReleaseMemory() if n.Cookie != cookie { glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent()) w.WriteHeader(http.StatusNotFound) return } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { if t.Unix() >= int64(n.LastModified) { w.WriteHeader(http.StatusNotModified) return } } } } etag := n.Etag() if inm := r.Header.Get("If-None-Match"); inm == etag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Etag", etag) if vs.tryHandleChunkedFile(n, filename, w, r) { return } if n.NameSize > 0 && filename == "" { filename = string(n.Name) if ext == "" { ext = path.Ext(filename) } } mtype := "" if n.MimeSize > 0 { mt := string(n.Mime) if !strings.HasPrefix(mt, "application/octet-stream") { mtype = mt } } if ext != ".gz" { if n.IsGzipped() { if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { w.Header().Set("Content-Encoding", "gzip") } else { if n.Data, err = operation.UnGzipData(n.Data); err != nil { glog.V(0).Infoln("ungzip error:", err, r.URL.Path) } } } } if ext == ".png" || ext == ".jpg" || ext == ".gif" { width, height := 0, 0 if r.FormValue("width") != "" { width, _ = strconv.Atoi(r.FormValue("width")) } if r.FormValue("height") != "" { height, _ = strconv.Atoi(r.FormValue("height")) } n.Data, _, _ = images.Resized(ext, n.Data, width, height) } if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil { glog.V(2).Infoln("response write error:", e) } }