Exemple #1
0
func uploader(ch chan int) {
	for {
		<-ch
		v := make([]byte, 33000)

		filename := md5Hash(v)
		req, err := multipartRequest("/blob/upload/", filename, v)

		if err != nil {
			log.Errorf("err creating request %v", err)
			return
		}

		res, err := doReq(req)

		if err != nil {
			log.Errorf("err sending request %v", err)
		}

		ur := new(protocol.UploadResponse)
		parseResponse(res, ur)

		if res.StatusCode != 201 {
			log.Errorf("Unexpected status code %d", res.StatusCode)
		}
		count()
	}
}
Exemple #2
0
func handle(conn net.Conn) {
	defer conn.Close()

	err := conn.SetDeadline(time.Now().Add(Timeout))

	if err != nil {
		log.Errorf("Error setting deadline on conn")
		return
	}

	buf := getBuf(bufSize)
	defer putBuf(buf)

	n, err := conn.Read(buf)

	if err != nil {
		log.Error("Error reading from conn")
		return
	}

	log.Printf("Got %+q", buf[:n])
	resp, err := parseRequest(buf[:n])

	if err != nil {
		log.Error(err)
		return
	}

	n, err = conn.Write(resp)

	if err != nil || n != len(resp) {
		log.Errorf("Error writing to conn")
		return
	}
}
Exemple #3
0
func (kvstore *KVStore) dial() (redis.Conn, error) {
	conn, err := redis.Dial("tcp", kvstore.cfg.addr)

	if err != nil {
		return nil, err
	}

	if kvstore.cfg.password != "" {
		if _, err := conn.Do("AUTH", kvstore.cfg.password); err != nil {
			log.Errorf("Redis AUTH err: %v", err)
			conn.Close()
			return nil, err
		}
	}

	if kvstore.cfg.db != 0 {
		if _, err := conn.Do("SELECT", kvstore.cfg.db); err != nil {
			log.Errorf("Redis SELECT err: %v", err)
			conn.Close()
			return nil, err
		}
	}

	return conn, nil
}
Exemple #4
0
func handleBatchRemove(req *http.Request, storage blobserver.Storage) (interface{}, error) {
	res := new(protocol.RemoveResponse)
	n := 0
	toRemove := make([]blob.Ref, 0)

	for {
		n++
		if n > maxRemovesPerRequest {
			return nil, newRateLimitError(maxRemovesPerRequest)
		}

		key := fmt.Sprintf("blob%v", n)
		value := req.FormValue(key)
		ref, ok := blob.Parse(value)

		if !ok {
			break
		}

		toRemove = append(toRemove, ref)
	}

	err := storage.RemoveBlobs(toRemove)

	if err != nil {
		log.Errorf("Server error during remove: %v", err)
		return nil, newHTTPError("Server error", http.StatusInternalServerError)
	}

	res.Removed = toRemove
	return res, nil
}
Exemple #5
0
func (sto *swiftStorage) createContainer(name string) (err error) {
	for i := 0; i < 3; i++ {
		if err = sto.createCheckContainer(name); err != nil {
			log.Errorf("create container failed %d, %v", i, err)
		}
	}
	return
}
Exemple #6
0
func handleStat(req *http.Request, storage blobserver.BlobStatter, blobs []string) (interface{}, error) {
	res := new(protocol.StatResponse)
	needStat := map[blob.Ref]bool{}
	n := 0

	for _, value := range blobs {
		n++
		if value == "" {
			n--
			break
		}
		if n > maxStatBlobs {
			return nil, newRateLimitError(maxStatBlobs)
		}
		ref, ok := blob.Parse(value)
		if !ok {
			return nil, newHTTPError("Bogus blobref for value", http.StatusBadRequest)
		}
		needStat[ref] = true
	}

	toStat := make([]blob.Ref, 0, len(needStat))

	for br := range needStat {
		toStat = append(toStat, br)
	}

	log.Printf("Need to stat blob cnt: %d, got %d", len(needStat), len(blobs))
	blobch := make(chan blob.SizedInfoRef)
	errch := make(chan error, 1)

	go func() {
		err := storage.StatBlobs(blobch, toStat)
		close(blobch)
		errch <- err
	}()

	for sb := range blobch {
		res.Stat = append(res.Stat, protocol.RefInfo{
			Ref:  sb.Ref,
			Size: uint32(sb.Size),
			MD5:  sb.MD5,
		})
		delete(needStat, sb.Ref)
	}

	err := <-errch

	if err != nil {
		log.Errorf("Stat error: %v", err)
		return nil, newHTTPError("Server Error", http.StatusInternalServerError)
	}

	return res, nil
}
Exemple #7
0
// createUploadHandler returns the handler that receives multi-part form uploads.
func createUploadHandler(storage blobserver.Storage) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		res, err := handleMultiPartUpload(r, storage)
		if err != nil {
			log.Errorf("upload: %v", err)
			httputil.ServeJSONError(rw, err)
		} else {
			httputil.ReturnJSONCode(rw, http.StatusCreated, res)
		}
	})
}
Exemple #8
0
func RecoveryHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if rec := recover(); rec != nil {
				st := stack(3)
				log.Errorf("Recovered: %s\n%s", rec, st)
				w.WriteHeader(http.StatusInternalServerError)
			}
		}()
		h.ServeHTTP(w, r)
	})
}
Exemple #9
0
func handleRemove(req *http.Request, storage blobserver.Storage) (interface{}, error) {
	res := new(protocol.RemoveResponse)
	vars := mux.Vars(req)
	ref, ok := blob.Parse(vars["blobRef"])
	if !ok {
		return nil, newHTTPError("Invalid blob ref", http.StatusBadRequest)
	}
	toRemove := []blob.Ref{ref}
	err := storage.RemoveBlobs(toRemove)

	if err != nil {
		log.Errorf("Server error during remove: %v", err)
		return nil, newHTTPError("Server error", http.StatusInternalServerError)
	}

	res.Removed = toRemove
	return res, nil
}
Exemple #10
0
func (c *Client) multiStat(paths []string) (Resources, error) {
	results := make(Resources, 0, len(paths))
	values := url.Values{}

	for _, path := range paths {
		fp, err := os.Open(path)

		if err != nil {
			return nil, err
		}

		hasher := md5.New()
		io.Copy(hasher, fp)
		resMD5 := hex.EncodeToString(hasher.Sum(nil))
		err = fp.Close()

		if err != nil {
			return nil, err
		}

		res := &Resource{
			MD5:      resMD5,
			Filename: filepath.Base(path),
			Path:     path,
			Created:  true,
		}

		results = append(results, res)
		values.Add("blob", res.Filename)
	}

	uri := c.absURL("/blob/stat/", values)
	req, err := http.NewRequest("GET", uri, nil)

	if err != nil {
		return results, err
	}

	res, err := http.DefaultClient.Do(req)

	if err != nil {
		return results, err
	}

	sr := new(protocol.StatResponse)

	if err := parseResponse(res, sr); err != nil {
		return nil, err
	}

	for _, si := range sr.Stat {
		cur := results.findByPath(si.Path)

		if cur == nil {
			log.Errorf("unexpected result %v", si)
			continue
		}

		cur.Created = si.MD5 != cur.MD5
		cur.URL = c.CDNBaseURL + si.Path
	}

	return results, nil
}