func uploader(ch chan int) {
for {
<-ch
v := make([]byte, 33000)
filename := md5Hash(v)
req, err := multipartRequest("/blob/upload/", filename, v)
if err != nil {
log.Errorf("err creating request %v", err)
return
}
res, err := doReq(req)
if err != nil {
log.Errorf("err sending request %v", err)
}
ur := new(protocol.UploadResponse)
parseResponse(res, ur)
if res.StatusCode != 201 {
log.Errorf("Unexpected status code %d", res.StatusCode)
}
count()
}
}
func handle(conn net.Conn) {
defer conn.Close()
err := conn.SetDeadline(time.Now().Add(Timeout))
if err != nil {
log.Errorf("Error setting deadline on conn")
return
}
buf := getBuf(bufSize)
defer putBuf(buf)
n, err := conn.Read(buf)
if err != nil {
log.Error("Error reading from conn")
return
}
log.Printf("Got %+q", buf[:n])
resp, err := parseRequest(buf[:n])
if err != nil {
log.Error(err)
return
}
n, err = conn.Write(resp)
if err != nil || n != len(resp) {
log.Errorf("Error writing to conn")
return
}
}
func (kvstore *KVStore) dial() (redis.Conn, error) {
conn, err := redis.Dial("tcp", kvstore.cfg.addr)
if err != nil {
return nil, err
}
if kvstore.cfg.password != "" {
if _, err := conn.Do("AUTH", kvstore.cfg.password); err != nil {
log.Errorf("Redis AUTH err: %v", err)
conn.Close()
return nil, err
}
}
if kvstore.cfg.db != 0 {
if _, err := conn.Do("SELECT", kvstore.cfg.db); err != nil {
log.Errorf("Redis SELECT err: %v", err)
conn.Close()
return nil, err
}
}
return conn, nil
}
func handleBatchRemove(req *http.Request, storage blobserver.Storage) (interface{}, error) {
res := new(protocol.RemoveResponse)
n := 0
toRemove := make([]blob.Ref, 0)
for {
n++
if n > maxRemovesPerRequest {
return nil, newRateLimitError(maxRemovesPerRequest)
}
key := fmt.Sprintf("blob%v", n)
value := req.FormValue(key)
ref, ok := blob.Parse(value)
if !ok {
break
}
toRemove = append(toRemove, ref)
}
err := storage.RemoveBlobs(toRemove)
if err != nil {
log.Errorf("Server error during remove: %v", err)
return nil, newHTTPError("Server error", http.StatusInternalServerError)
}
res.Removed = toRemove
return res, nil
}
func (sto *swiftStorage) createContainer(name string) (err error) {
for i := 0; i < 3; i++ {
if err = sto.createCheckContainer(name); err != nil {
log.Errorf("create container failed %d, %v", i, err)
}
}
return
}
func handleStat(req *http.Request, storage blobserver.BlobStatter, blobs []string) (interface{}, error) {
res := new(protocol.StatResponse)
needStat := map[blob.Ref]bool{}
n := 0
for _, value := range blobs {
n++
if value == "" {
n--
break
}
if n > maxStatBlobs {
return nil, newRateLimitError(maxStatBlobs)
}
ref, ok := blob.Parse(value)
if !ok {
return nil, newHTTPError("Bogus blobref for value", http.StatusBadRequest)
}
needStat[ref] = true
}
toStat := make([]blob.Ref, 0, len(needStat))
for br := range needStat {
toStat = append(toStat, br)
}
log.Printf("Need to stat blob cnt: %d, got %d", len(needStat), len(blobs))
blobch := make(chan blob.SizedInfoRef)
errch := make(chan error, 1)
go func() {
err := storage.StatBlobs(blobch, toStat)
close(blobch)
errch <- err
}()
for sb := range blobch {
res.Stat = append(res.Stat, protocol.RefInfo{
Ref: sb.Ref,
Size: uint32(sb.Size),
MD5: sb.MD5,
})
delete(needStat, sb.Ref)
}
err := <-errch
if err != nil {
log.Errorf("Stat error: %v", err)
return nil, newHTTPError("Server Error", http.StatusInternalServerError)
}
return res, nil
}
// createUploadHandler returns the handler that receives multi-part form uploads.
func createUploadHandler(storage blobserver.Storage) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
res, err := handleMultiPartUpload(r, storage)
if err != nil {
log.Errorf("upload: %v", err)
httputil.ServeJSONError(rw, err)
} else {
httputil.ReturnJSONCode(rw, http.StatusCreated, res)
}
})
}
func RecoveryHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if rec := recover(); rec != nil {
st := stack(3)
log.Errorf("Recovered: %s\n%s", rec, st)
w.WriteHeader(http.StatusInternalServerError)
}
}()
h.ServeHTTP(w, r)
})
}
func handleRemove(req *http.Request, storage blobserver.Storage) (interface{}, error) {
res := new(protocol.RemoveResponse)
vars := mux.Vars(req)
ref, ok := blob.Parse(vars["blobRef"])
if !ok {
return nil, newHTTPError("Invalid blob ref", http.StatusBadRequest)
}
toRemove := []blob.Ref{ref}
err := storage.RemoveBlobs(toRemove)
if err != nil {
log.Errorf("Server error during remove: %v", err)
return nil, newHTTPError("Server error", http.StatusInternalServerError)
}
res.Removed = toRemove
return res, nil
}
func (c *Client) multiStat(paths []string) (Resources, error) {
results := make(Resources, 0, len(paths))
values := url.Values{}
for _, path := range paths {
fp, err := os.Open(path)
if err != nil {
return nil, err
}
hasher := md5.New()
io.Copy(hasher, fp)
resMD5 := hex.EncodeToString(hasher.Sum(nil))
err = fp.Close()
if err != nil {
return nil, err
}
res := &Resource{
MD5: resMD5,
Filename: filepath.Base(path),
Path: path,
Created: true,
}
results = append(results, res)
values.Add("blob", res.Filename)
}
uri := c.absURL("/blob/stat/", values)
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return results, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return results, err
}
sr := new(protocol.StatResponse)
if err := parseResponse(res, sr); err != nil {
return nil, err
}
for _, si := range sr.Stat {
cur := results.findByPath(si.Path)
if cur == nil {
log.Errorf("unexpected result %v", si)
continue
}
cur.Created = si.MD5 != cur.MD5
cur.URL = c.CDNBaseURL + si.Path
}
return results, nil
}