func (bctl *BucketCtl) Lookup(bname, key string, req *http.Request) (reply *reply.LookupResult, err error) { bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthEmpty) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("lookup: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.ReaderIOFlags)) log.Printf("lookup-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) reply, err = bucket.lookup_serialize(false, s.ParallelLookup(key)) return }
func (bctl *BucketCtl) Delete(bname, key string, req *http.Request) (err error) { bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("delete: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("delete-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) for r := range s.Remove(key) { err = r.Error() } return }
func lookup_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply { bucket := strings[0] key := strings[1] reply, err := proxy.bctl.Lookup(bucket, key, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } reply_json, err := json.Marshal(reply) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("lookup: json marshal failed: %q", err)) return Reply{ err: err, status: http.StatusServiceUnavailable, } } w.WriteHeader(http.StatusOK) w.Write(reply_json) return GoodReply() }
func stat_handler(w http.ResponseWriter, req *http.Request, str ...string) Reply { bnames := make([]string, 0) bnames_combined := strings.SplitN(str[0], "/", 2) if len(bnames_combined[0]) != 0 { bnames = strings.Split(bnames_combined[0], ",") if len(bnames[0]) == 0 { bnames = []string{} } } reply, err := proxy.bctl.Stat(req, bnames) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } reply_json, err := json.Marshal(reply) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("stat: json marshal failed: %q", err)) return Reply{ err: err, status: http.StatusBadRequest, } } w.WriteHeader(http.StatusOK) w.Write(reply_json) return GoodReply() }
func (bctl *BucketCtl) Stream(bname, key string, w http.ResponseWriter, req *http.Request) (err error) { bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthEmpty) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("stream: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("stream: could not create data session: %v", err)) return } defer s.Delete() s.SetFilter(elliptics.SessionFilterAll) s.SetNamespace(bucket.Name) bctl.SetGroupsTimeout(s, bucket, key) log.Printf("stream-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) offset, size, err := URIOffsetSize(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("stream: %v", err)) return } if offset != 0 || size != 0 { if size == 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } else { req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)) } } rs, err := elliptics.NewReadSeekerOffsetSize(s, key, offset, size) if err != nil { err = errors.NewKeyErrorFromEllipticsError(err, req.URL.String(), "stream: could not create read-seeker") return } defer rs.Free() bctl.SetContentType(key, w) http.ServeContent(w, req, key, rs.Mtime, rs) return }
func nobucket_upload_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply { key := strings[0] resp, bucket, err := proxy.bctl.Upload(key, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } return proxy.send_upload_reply(w, req, bucket, key, resp) }
func get_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply { bucket := strings[0] key := strings[1] err := proxy.bctl.Stream(bucket, key, w, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } return GoodReply() }
func delete_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply { bucket := strings[0] key := strings[1] err := proxy.bctl.Delete(bucket, key, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } w.WriteHeader(http.StatusOK) return GoodReply() }
func (bctl *BucketCtl) BulkDelete(bname string, keys []string, req *http.Request) (reply map[string]interface{}, err error) { reply = make(map[string]interface{}) bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("bulk_delete: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("bulk-delete-trace-id: %x: url: %s, bucket: %s, keys: %v\n", s.GetTraceID(), req.URL.String(), bucket.Name, keys) for r := range s.BulkRemove(keys) { err = r.Error() if err != nil { reply[r.Key()] = err.Error() } } err = nil return }
func (bctl *BucketCtl) bucket_upload(bucket *Bucket, key string, req *http.Request) (reply *reply.LookupResult, err error) { err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } lheader, ok := req.Header["Content-Length"] if !ok { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, "upload: there is no Content-Length header") return } total_size, err := strconv.ParseUint(lheader[0], 0, 64) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: invalid content length conversion: %v", err)) return } if total_size == 0 { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, "upload: attempting to perform invalid zero-length upload") return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("upload: could not create data session: %v", err)) return } defer s.Delete() s.SetFilter(elliptics.SessionFilterAll) s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetTimeout(100) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("upload-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) ranges, err := ranges.ParseRange(req.Header.Get("Range"), int64(total_size)) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: %v", err)) return } var offset uint64 = 0 if len(ranges) != 0 { offset = uint64(ranges[0].Start) } start := time.Now() reply, err = bucket.lookup_serialize(true, s.WriteData(key, req.Body, offset, total_size)) // PID controller should aim at some destination performance point // it can be velocity pf the vehicle or deisred write rate // // Let's consider our desired control point as number of useconds needed to write 1 byte into the storage // In the ideal world it would be zero time_us := time.Since(start).Nanoseconds() / 1000 e := float64(time_us) / float64(total_size) bctl.RLock() str := make([]string, 0) for _, res := range reply.Servers { sg, ok := bucket.Group[res.Group] if ok { st, back_err := sg.FindStatBackend(res.Addr, res.Backend) if back_err == nil { old_pain := st.PIDPain() update_pain := e estring := "ok" if res.Error != nil { update_pain = BucketWriteErrorPain estring = res.Error.Error() } st.PIDUpdate(update_pain) str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: %v, pain: %f -> %f}", res.Group, time_us, e, estring, old_pain, st.PIDPain())) } else { str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no backend stat}", res.Group, time_us, e)) } } else { str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no group stat}", res.Group, time_us, e)) } } if len(reply.SuccessGroups) == 0 { for _, group_id := range bucket.Meta.Groups { str = append(str, fmt.Sprintf("{error-group: %d, time: %d us}", group_id, time_us)) } } bctl.RUnlock() log.Printf("bucket-upload: bucket: %s, key: %s, size: %d: %v\n", bucket.Name, key, total_size, str) return }
func bulk_delete_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply { bucket := strings[0] var err error var v map[string]interface{} = make(map[string]interface{}) if err = json.NewDecoder(req.Body).Decode(&v); err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("bulk_delete: could not parse input json: %v", err)) return Reply{ err: err, status: errors.ErrorStatus(err), } } kv, ok := v["keys"] if !ok { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("bulk_delete: there is no 'keys' array")) return Reply{ err: err, status: errors.ErrorStatus(err), } } var keys []string = make([]string, 0) for _, v := range kv.([]interface{}) { keys = append(keys, v.(string)) } if len(keys) == 0 { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("bulk_delete: 'keys' array is empty")) return Reply{ err: err, status: errors.ErrorStatus(err), } } reply, err := proxy.bctl.BulkDelete(bucket, keys, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } reply_json, err := json.Marshal(reply) if err != nil { log.Printf("url: %s: bulk_delete: json marshal failed: %q\n", req.URL, err) return Reply{ err: err, status: http.StatusBadRequest, } } w.WriteHeader(http.StatusOK) w.Write(reply_json) return GoodReply() }
func redirect_handler(w http.ResponseWriter, req *http.Request, string_keys ...string) Reply { if proxy.bctl.Conf.Proxy.RedirectPort == 0 || proxy.bctl.Conf.Proxy.RedirectPort >= 65536 { err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("redirect is not allowed because of invalid redirect port %d", proxy.bctl.Conf.Proxy.RedirectPort)) return Reply{ err: err, status: errors.ErrorStatus(err), } } bname := string_keys[0] key := string_keys[1] reply, err := proxy.bctl.Lookup(bname, key, req) if err != nil { return Reply{ err: err, status: errors.ErrorStatus(err), } } srv := reply.Servers[rand.Intn(len(reply.Servers))] scheme := "http" if req.URL.Scheme != "" { scheme = req.URL.Scheme } if len(srv.Filename) == 0 { err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("lookup returned invalid filename: %s", srv.Filename)) return Reply{ err: err, status: errors.ErrorStatus(err), } } filename := srv.Filename if len(proxy.bctl.Conf.Proxy.RedirectRoot) != 0 { if strings.HasPrefix(filename, proxy.bctl.Conf.Proxy.RedirectRoot) { filename = filename[len(proxy.bctl.Conf.Proxy.RedirectRoot):] } } slash := "/" if filename[0] == '/' { slash = "" } ranges, err := ranges.ParseRange(req.Header.Get("Range"), int64(srv.Size)) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: %v", err)) return Reply{ err: err, status: errors.ErrorStatus(err), } } offset, size, err := bucket.URIOffsetSize(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("redirect: %v", err)) return Reply{ err: err, status: errors.ErrorStatus(err), } } if len(ranges) != 0 { offset = uint64(ranges[0].Start) size = uint64(ranges[0].Length) } if offset >= srv.Size { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("redirect: offset is beyond size of the object: offset: %d, size: %d", offset, srv.Size)) return Reply{ err: err, status: errors.ErrorStatus(err), } } if size == 0 || offset+size >= srv.Size { size = srv.Size - offset } timestamp := time.Now().Unix() url_str := fmt.Sprintf("%s://%s:%d%s%s:%d:%d", scheme, srv.Server.HostString(), proxy.bctl.Conf.Proxy.RedirectPort, slash, filename, srv.Offset+offset, size) u, err := url.Parse(url_str) if err != nil { err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("could not parse generated redirect url '%s'", url_str)) return Reply{ err: err, status: errors.ErrorStatus(err), } } req.URL = u w.Header().Set("X-Ell-Mtime", fmt.Sprintf("%d", srv.Info.Mtime.Unix())) w.Header().Set("X-Ell-Signtime", fmt.Sprintf("%d", timestamp)) w.Header().Set("X-Ell-Signature-Timeout", fmt.Sprintf("%d", proxy.bctl.Conf.Proxy.RedirectSignatureTimeout)) w.Header().Set("X-Ell-File-Offset", fmt.Sprintf("%d", srv.Offset)) w.Header().Set("X-Ell-Total-Size", fmt.Sprintf("%d", srv.Size)) w.Header().Set("X-Ell-File", filename) signature, err := auth.GenerateSignature(proxy.bctl.Conf.Proxy.RedirectToken, "GET", req.URL, w.Header()) if err != nil { err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("could not generate signature for redirect url '%s': %v", url_str, err)) return Reply{ err: err, status: errors.ErrorStatus(err), } } w.Header().Set(auth.AuthHeaderStr, signature) http.Redirect(w, req, url_str, http.StatusFound) return GoodReply() }