func (bctl *BucketCtl) Lookup(bname, key string, req *http.Request) (reply *reply.LookupResult, err error) {
bucket, err := bctl.FindBucket(bname)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error())
return
}
err = bucket.check_auth(req, BucketAuthEmpty)
if err != nil {
err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err),
fmt.Sprintf("upload: %s", errors.ErrorData(err)))
return
}
s, err := bctl.e.DataSession(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("lookup: could not create data session: %v", err))
return
}
defer s.Delete()
s.SetNamespace(bucket.Name)
s.SetGroups(bucket.Meta.Groups)
s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.ReaderIOFlags))
log.Printf("lookup-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n",
s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key))
reply, err = bucket.lookup_serialize(false, s.ParallelLookup(key))
return
}
func (bctl *BucketCtl) Delete(bname, key string, req *http.Request) (err error) {
bucket, err := bctl.FindBucket(bname)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error())
return
}
err = bucket.check_auth(req, BucketAuthWrite)
if err != nil {
err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err),
fmt.Sprintf("upload: %s", errors.ErrorData(err)))
return
}
s, err := bctl.e.DataSession(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("delete: could not create data session: %v", err))
return
}
defer s.Delete()
s.SetNamespace(bucket.Name)
s.SetGroups(bucket.Meta.Groups)
s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags))
log.Printf("delete-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n",
s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key))
for r := range s.Remove(key) {
err = r.Error()
}
return
}
func lookup_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply {
bucket := strings[0]
key := strings[1]
reply, err := proxy.bctl.Lookup(bucket, key, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
reply_json, err := json.Marshal(reply)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("lookup: json marshal failed: %q", err))
return Reply{
err: err,
status: http.StatusServiceUnavailable,
}
}
w.WriteHeader(http.StatusOK)
w.Write(reply_json)
return GoodReply()
}
func stat_handler(w http.ResponseWriter, req *http.Request, str ...string) Reply {
bnames := make([]string, 0)
bnames_combined := strings.SplitN(str[0], "/", 2)
if len(bnames_combined[0]) != 0 {
bnames = strings.Split(bnames_combined[0], ",")
if len(bnames[0]) == 0 {
bnames = []string{}
}
}
reply, err := proxy.bctl.Stat(req, bnames)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
reply_json, err := json.Marshal(reply)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("stat: json marshal failed: %q", err))
return Reply{
err: err,
status: http.StatusBadRequest,
}
}
w.WriteHeader(http.StatusOK)
w.Write(reply_json)
return GoodReply()
}
func (bctl *BucketCtl) Stream(bname, key string, w http.ResponseWriter, req *http.Request) (err error) {
bucket, err := bctl.FindBucket(bname)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error())
return
}
err = bucket.check_auth(req, BucketAuthEmpty)
if err != nil {
err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err),
fmt.Sprintf("stream: %s", errors.ErrorData(err)))
return
}
s, err := bctl.e.DataSession(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("stream: could not create data session: %v", err))
return
}
defer s.Delete()
s.SetFilter(elliptics.SessionFilterAll)
s.SetNamespace(bucket.Name)
bctl.SetGroupsTimeout(s, bucket, key)
log.Printf("stream-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n",
s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key))
offset, size, err := URIOffsetSize(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("stream: %v", err))
return
}
if offset != 0 || size != 0 {
if size == 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
} else {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size-1))
}
}
rs, err := elliptics.NewReadSeekerOffsetSize(s, key, offset, size)
if err != nil {
err = errors.NewKeyErrorFromEllipticsError(err, req.URL.String(), "stream: could not create read-seeker")
return
}
defer rs.Free()
bctl.SetContentType(key, w)
http.ServeContent(w, req, key, rs.Mtime, rs)
return
}
func nobucket_upload_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply {
key := strings[0]
resp, bucket, err := proxy.bctl.Upload(key, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
return proxy.send_upload_reply(w, req, bucket, key, resp)
}
func get_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply {
bucket := strings[0]
key := strings[1]
err := proxy.bctl.Stream(bucket, key, w, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
return GoodReply()
}
func delete_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply {
bucket := strings[0]
key := strings[1]
err := proxy.bctl.Delete(bucket, key, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
w.WriteHeader(http.StatusOK)
return GoodReply()
}
func (bctl *BucketCtl) BulkDelete(bname string, keys []string, req *http.Request) (reply map[string]interface{}, err error) {
reply = make(map[string]interface{})
bucket, err := bctl.FindBucket(bname)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, err.Error())
return
}
err = bucket.check_auth(req, BucketAuthWrite)
if err != nil {
err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err),
fmt.Sprintf("upload: %s", errors.ErrorData(err)))
return
}
s, err := bctl.e.DataSession(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("bulk_delete: could not create data session: %v", err))
return
}
defer s.Delete()
s.SetNamespace(bucket.Name)
s.SetGroups(bucket.Meta.Groups)
s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags))
log.Printf("bulk-delete-trace-id: %x: url: %s, bucket: %s, keys: %v\n",
s.GetTraceID(), req.URL.String(), bucket.Name, keys)
for r := range s.BulkRemove(keys) {
err = r.Error()
if err != nil {
reply[r.Key()] = err.Error()
}
}
err = nil
return
}
func (bctl *BucketCtl) bucket_upload(bucket *Bucket, key string, req *http.Request) (reply *reply.LookupResult, err error) {
err = bucket.check_auth(req, BucketAuthWrite)
if err != nil {
err = errors.NewKeyError(req.URL.String(), errors.ErrorStatus(err),
fmt.Sprintf("upload: %s", errors.ErrorData(err)))
return
}
lheader, ok := req.Header["Content-Length"]
if !ok {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
"upload: there is no Content-Length header")
return
}
total_size, err := strconv.ParseUint(lheader[0], 0, 64)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
fmt.Sprintf("upload: invalid content length conversion: %v", err))
return
}
if total_size == 0 {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
"upload: attempting to perform invalid zero-length upload")
return
}
s, err := bctl.e.DataSession(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("upload: could not create data session: %v", err))
return
}
defer s.Delete()
s.SetFilter(elliptics.SessionFilterAll)
s.SetNamespace(bucket.Name)
s.SetGroups(bucket.Meta.Groups)
s.SetTimeout(100)
s.SetIOflags(elliptics.IOflag(bctl.Conf.Proxy.WriterIOFlags))
log.Printf("upload-trace-id: %x: url: %s, bucket: %s, key: %s, id: %s\n",
s.GetTraceID(), req.URL.String(), bucket.Name, key, s.Transform(key))
ranges, err := ranges.ParseRange(req.Header.Get("Range"), int64(total_size))
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: %v", err))
return
}
var offset uint64 = 0
if len(ranges) != 0 {
offset = uint64(ranges[0].Start)
}
start := time.Now()
reply, err = bucket.lookup_serialize(true, s.WriteData(key, req.Body, offset, total_size))
// PID controller should aim at some destination performance point
// it can be velocity pf the vehicle or deisred write rate
//
// Let's consider our desired control point as number of useconds needed to write 1 byte into the storage
// In the ideal world it would be zero
time_us := time.Since(start).Nanoseconds() / 1000
e := float64(time_us) / float64(total_size)
bctl.RLock()
str := make([]string, 0)
for _, res := range reply.Servers {
sg, ok := bucket.Group[res.Group]
if ok {
st, back_err := sg.FindStatBackend(res.Addr, res.Backend)
if back_err == nil {
old_pain := st.PIDPain()
update_pain := e
estring := "ok"
if res.Error != nil {
update_pain = BucketWriteErrorPain
estring = res.Error.Error()
}
st.PIDUpdate(update_pain)
str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: %v, pain: %f -> %f}",
res.Group, time_us, e, estring, old_pain, st.PIDPain()))
} else {
str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no backend stat}",
res.Group, time_us, e))
}
} else {
str = append(str, fmt.Sprintf("{group: %d, time: %d us, e: %f, error: no group stat}",
res.Group, time_us, e))
}
}
if len(reply.SuccessGroups) == 0 {
for _, group_id := range bucket.Meta.Groups {
str = append(str, fmt.Sprintf("{error-group: %d, time: %d us}", group_id, time_us))
}
}
bctl.RUnlock()
log.Printf("bucket-upload: bucket: %s, key: %s, size: %d: %v\n", bucket.Name, key, total_size, str)
return
}
func bulk_delete_handler(w http.ResponseWriter, req *http.Request, strings ...string) Reply {
bucket := strings[0]
var err error
var v map[string]interface{} = make(map[string]interface{})
if err = json.NewDecoder(req.Body).Decode(&v); err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
fmt.Sprintf("bulk_delete: could not parse input json: %v", err))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
kv, ok := v["keys"]
if !ok {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
fmt.Sprintf("bulk_delete: there is no 'keys' array"))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
var keys []string = make([]string, 0)
for _, v := range kv.([]interface{}) {
keys = append(keys, v.(string))
}
if len(keys) == 0 {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
fmt.Sprintf("bulk_delete: 'keys' array is empty"))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
reply, err := proxy.bctl.BulkDelete(bucket, keys, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
reply_json, err := json.Marshal(reply)
if err != nil {
log.Printf("url: %s: bulk_delete: json marshal failed: %q\n", req.URL, err)
return Reply{
err: err,
status: http.StatusBadRequest,
}
}
w.WriteHeader(http.StatusOK)
w.Write(reply_json)
return GoodReply()
}
func redirect_handler(w http.ResponseWriter, req *http.Request, string_keys ...string) Reply {
if proxy.bctl.Conf.Proxy.RedirectPort == 0 || proxy.bctl.Conf.Proxy.RedirectPort >= 65536 {
err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("redirect is not allowed because of invalid redirect port %d",
proxy.bctl.Conf.Proxy.RedirectPort))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
bname := string_keys[0]
key := string_keys[1]
reply, err := proxy.bctl.Lookup(bname, key, req)
if err != nil {
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
srv := reply.Servers[rand.Intn(len(reply.Servers))]
scheme := "http"
if req.URL.Scheme != "" {
scheme = req.URL.Scheme
}
if len(srv.Filename) == 0 {
err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("lookup returned invalid filename: %s", srv.Filename))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
filename := srv.Filename
if len(proxy.bctl.Conf.Proxy.RedirectRoot) != 0 {
if strings.HasPrefix(filename, proxy.bctl.Conf.Proxy.RedirectRoot) {
filename = filename[len(proxy.bctl.Conf.Proxy.RedirectRoot):]
}
}
slash := "/"
if filename[0] == '/' {
slash = ""
}
ranges, err := ranges.ParseRange(req.Header.Get("Range"), int64(srv.Size))
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("upload: %v", err))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
offset, size, err := bucket.URIOffsetSize(req)
if err != nil {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest, fmt.Sprintf("redirect: %v", err))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
if len(ranges) != 0 {
offset = uint64(ranges[0].Start)
size = uint64(ranges[0].Length)
}
if offset >= srv.Size {
err = errors.NewKeyError(req.URL.String(), http.StatusBadRequest,
fmt.Sprintf("redirect: offset is beyond size of the object: offset: %d, size: %d",
offset, srv.Size))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
if size == 0 || offset+size >= srv.Size {
size = srv.Size - offset
}
timestamp := time.Now().Unix()
url_str := fmt.Sprintf("%s://%s:%d%s%s:%d:%d",
scheme, srv.Server.HostString(), proxy.bctl.Conf.Proxy.RedirectPort,
slash, filename, srv.Offset+offset, size)
u, err := url.Parse(url_str)
if err != nil {
err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("could not parse generated redirect url '%s'", url_str))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
req.URL = u
w.Header().Set("X-Ell-Mtime", fmt.Sprintf("%d", srv.Info.Mtime.Unix()))
w.Header().Set("X-Ell-Signtime", fmt.Sprintf("%d", timestamp))
w.Header().Set("X-Ell-Signature-Timeout", fmt.Sprintf("%d", proxy.bctl.Conf.Proxy.RedirectSignatureTimeout))
w.Header().Set("X-Ell-File-Offset", fmt.Sprintf("%d", srv.Offset))
w.Header().Set("X-Ell-Total-Size", fmt.Sprintf("%d", srv.Size))
w.Header().Set("X-Ell-File", filename)
signature, err := auth.GenerateSignature(proxy.bctl.Conf.Proxy.RedirectToken, "GET", req.URL, w.Header())
if err != nil {
err := errors.NewKeyError(req.URL.String(), http.StatusServiceUnavailable,
fmt.Sprintf("could not generate signature for redirect url '%s': %v", url_str, err))
return Reply{
err: err,
status: errors.ErrorStatus(err),
}
}
w.Header().Set(auth.AuthHeaderStr, signature)
http.Redirect(w, req, url_str, http.StatusFound)
return GoodReply()
}