func (bctl *BucketCtl) Delete(bname, key string, req *http.Request) (err error) { bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("delete: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("delete-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) for r := range s.Remove(key) { err = r.Error() } return }
func (bctl *BucketCtl) Lookup(bname, key string, req *http.Request) (reply *reply.LookupResult, err error) { bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthEmpty) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("lookup: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.ReaderIOFlags)) log.Printf("lookup-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) reply, err = bucket.lookup_serialize(false, s.ParallelLookup(key)) return }
func (e *Elliptics) DataSession(req *http.Request) (*elliptics.Session, error) { s, err := elliptics.NewSession(e.Node) if err != nil { return nil, err } values := req.URL.Query() var val uint64 var trace_id uint64 trace, ok := req.Header["X-Request"] if !ok { trace_id = uint64(rand.Int63()) } else { trace_id, err = strconv.ParseUint(trace[0], 0, 64) if err != nil { trace_id = uint64(rand.Int63()) } } ioflags, ok := values["ioflags"] if ok { val, err = strconv.ParseUint(ioflags[0], 0, 32) if err == nil { s.SetIOflags(elliptics.IOflag(val)) } } cflags, ok := values["cflags"] if ok { val, err = strconv.ParseUint(cflags[0], 0, 64) if err == nil { s.SetCflags(elliptics.Cflag(val)) } } trace, ok = values["trace_id"] if ok { trace_id, err = strconv.ParseUint(trace[0], 0, 64) if err != nil { trace_id = uint64(rand.Int63()) } } s.SetTraceID(elliptics.TraceID(trace_id)) return s, nil }
func (bctl *BucketCtl) BulkDelete(bname string, keys []string, req *http.Request) (reply map[string]interface{}, err error) { reply = make(map[string]interface{}) bucket, err := bctl.FindBucket(bname) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error()) return } err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("bulk_delete: could not create data session: %v", err)) return } defer s.Delete() s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("bulk-delete-trace-id: %x: url: %s, bucket: %s, keys: %v\n", s.GetTraceID(), req.URL.String(), bucket.Name, keys) for r := range s.BulkRemove(keys) { err = r.Error() if err != nil { reply[r.Key()] = err.Error() } } err = nil return }
func (bctl *BucketCtl) SetGroupsTimeout(s *elliptics.Session, bucket *Bucket, key string) { // sort groups by defrag state, increase timeout if needed groups := make([]uint32, 0) defrag_groups := make([]uint32, 0) timeout := 30 for group_id, sg := range bucket.Group { sb, err := sg.FindStatBackendKey(s, key, group_id) if err != nil { continue } if sb.DefragState != 0 { defrag_groups = append(defrag_groups, group_id) } else { groups = append(groups, group_id) } } // Not being defragmented backends first, then those which are currently being defragmented groups = append(groups, defrag_groups...) if len(groups) == len(defrag_groups) { timeout = 90 } // if there are no backends being defragmented, use weights to mix read states // if there are such backends, use strict order and read from non-defragmented backends first ioflags := elliptics.IOflag(bctl.Conf.Proxy.ReaderIOFlags) | s.GetIOflags() if len(defrag_groups) == 0 { ioflags |= elliptics.DNET_IO_FLAGS_MIX_STATES } s.SetIOflags(ioflags) // there are no stats for bucket groups, use what we have in metadata if len(groups) == 0 { groups = bucket.Meta.Groups } s.SetGroups(groups) s.SetTimeout(timeout) }
func (bctl *BucketCtl) bucket_upload(bucket *Bucket, key string, req *http.Request) (reply *reply.LookupResult, err error) { err = bucket.check_auth(req, BucketAuthWrite) if err != nil { err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err), fmt.Sprintf("upload: %s", errors.ErrorData(err))) return } lheader, ok := req.Header["Content-Length"] if !ok { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, "upload: there is no Content-Length header") return } total_size, err := strconv.ParseUint(lheader[0], 0, 64) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: invalid content length conversion: %v", err)) return } if total_size == 0 { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, "upload: attempting to perform invalid zero-length upload") return } s, err := bctl.e.DataSession(req) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable, fmt.Sprintf("upload: could not create data session: %v", err)) return } defer s.Delete() s.SetFilter(elliptics.SessionFilterAll) s.SetNamespace(bucket.Name) s.SetGroups(bucket.Meta.Groups) s.SetTimeout(100) s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags)) log.Printf("upload-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n", s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key)) ranges, err := ranges.ParseRange(req.Header.Get("Range"), int64(total_size)) if err != nil { err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: %v", err)) return } var offset uint64 = 0 if len(ranges) != 0 { offset = uint64(ranges[0].Start) } start := time.Now() reply, err = bucket.lookup_serialize(true, s.WriteData(key, req.Body, offset, total_size)) // PID controller should aim at some destination performance point // it can be velocity pf the vehicle or deisred write rate // // Let's consider our desired control point as number of useconds needed to write 1 byte into the storage // In the ideal world it would be zero time_us := time.Since(start).Nanoseconds() / 1000 e := float64(time_us) / float64(total_size) bctl.RLock() str := make([]string, 0) for _, res := range reply.Servers { sg, ok := bucket.Group[res.Group] if ok { st, back_err := sg.FindStatBackend(res.Addr, res.Backend) if back_err == nil { old_pain := st.PIDPain() update_pain := e estring := "ok" if res.Error != nil { update_pain = BucketWriteErrorPain estring = res.Error.Error() } st.PIDUpdate(update_pain) str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: %v, pain: %f -> %f}", res.Group, time_us, e, estring, old_pain, st.PIDPain())) } else { str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no backend stat}", res.Group, time_us, e)) } } else { str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no group stat}", res.Group, time_us, e)) } } if len(reply.SuccessGroups) == 0 { for _, group_id := range bucket.Meta.Groups { str = append(str, fmt.Sprintf("{error-group: %d, time: %d us}", group_id, time_us)) } } bctl.RUnlock() log.Printf("bucket-upload: bucket: %s, key: %s, size: %d: %v\n", bucket.Name, key, total_size, str) return }