func (c *Client) doSomeStats() {
c.httpGate.Start()
defer c.httpGate.Done()
var batch map[blob.Ref][]statReq
c.pendStatMu.Lock()
{
if len(c.pendStat) == 0 {
// Lost race. Another batch got these.
c.pendStatMu.Unlock()
return
}
batch = make(map[blob.Ref][]statReq)
for br, reqs := range c.pendStat {
batch[br] = reqs
delete(c.pendStat, br)
if len(batch) == maxStatPerReq {
go c.doSomeStats() // kick off next batch
break
}
}
}
c.pendStatMu.Unlock()
if env.DebugUploads() {
println("doing stat batch of", len(batch))
}
blobs := make([]blob.Ref, 0, len(batch))
for br := range batch {
blobs = append(blobs, br)
}
ourDest := make(chan blob.SizedRef)
errc := make(chan error, 1)
go func() {
// false for not gated, since we already grabbed the
// token at the beginning of this function.
errc <- c.doStat(ourDest, blobs, 0, false)
close(ourDest)
}()
for sb := range ourDest {
for _, req := range batch[sb.Ref] {
req.dest <- sb
}
}
// Copy the doStat's error to all waiters for all blobrefs in this batch.
err := <-errc
for _, reqs := range batch {
for _, req := range reqs {
req.errc <- err
}
}
}
// Upload uploads a blob, as described by the provided UploadHandle parameters.
func (c *Client) Upload(h *UploadHandle) (*PutResult, error) {
errorf := func(msg string, arg ...interface{}) (*PutResult, error) {
err := fmt.Errorf(msg, arg...)
c.log.Print(err.Error())
return nil, err
}
bodyReader, bodySize, err := h.readerAndSize()
if err != nil {
return nil, fmt.Errorf("client: error slurping upload handle to find its length: %v", err)
}
if bodySize > constants.MaxBlobSize {
return nil, errors.New("client: body is bigger then max blob size")
}
c.statsMutex.Lock()
c.stats.UploadRequests.Blobs++
c.stats.UploadRequests.Bytes += bodySize
c.statsMutex.Unlock()
pr := &PutResult{BlobRef: h.BlobRef, Size: uint32(bodySize)}
if c.sto != nil {
// TODO: stat first so we can show skipped?
_, err := blobserver.Receive(c.sto, h.BlobRef, bodyReader)
if err != nil {
return nil, err
}
return pr, nil
}
if !h.Vivify {
if _, ok := c.haveCache.StatBlobCache(h.BlobRef); ok {
pr.Skipped = true
return pr, nil
}
}
blobrefStr := h.BlobRef.String()
// Pre-upload. Check whether the blob already exists on the
// server and if not, the URL to upload it to.
pfx, err := c.prefix()
if err != nil {
return nil, err
}
if !h.SkipStat {
url_ := fmt.Sprintf("%s/camli/stat", pfx)
req := c.newRequest("POST", url_, strings.NewReader("camliversion=1&blob1="+blobrefStr))
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.doReqGated(req)
if err != nil {
return errorf("stat http error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errorf("stat response had http status %d", resp.StatusCode)
}
stat, err := parseStatResponse(resp)
if err != nil {
return nil, err
}
for _, sbr := range stat.HaveMap {
c.haveCache.NoteBlobExists(sbr.Ref, uint32(sbr.Size))
}
_, serverHasIt := stat.HaveMap[blobrefStr]
if env.DebugUploads() {
log.Printf("HTTP Stat(%s) = %v", blobrefStr, serverHasIt)
}
if !h.Vivify && serverHasIt {
pr.Skipped = true
if closer, ok := h.Contents.(io.Closer); ok {
// TODO(bradfitz): I did this
// Close-if-possible thing early on, before I
// knew better. Fix the callers instead, and
// fix the docs.
closer.Close()
}
c.haveCache.NoteBlobExists(h.BlobRef, uint32(bodySize))
return pr, nil
}
}
if env.DebugUploads() {
log.Printf("Uploading: %s (%d bytes)", blobrefStr, bodySize)
}
pipeReader, pipeWriter := io.Pipe()
multipartWriter := multipart.NewWriter(pipeWriter)
copyResult := make(chan error, 1)
go func() {
defer pipeWriter.Close()
part, err := multipartWriter.CreateFormFile(blobrefStr, blobrefStr)
if err != nil {
copyResult <- err
return
}
_, err = io.Copy(part, bodyReader)
if err == nil {
err = multipartWriter.Close()
}
copyResult <- err
}()
// TODO(bradfitz): verbosity levels. make this VLOG(2) or something. it's noisy:
// c.log.Printf("Uploading %s", br)
uploadURL := fmt.Sprintf("%s/camli/upload", pfx)
req := c.newRequest("POST", uploadURL)
req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
if h.Vivify {
req.Header.Add("X-Camlistore-Vivify", "1")
}
req.Body = ioutil.NopCloser(pipeReader)
req.ContentLength = multipartOverhead + bodySize + int64(len(blobrefStr))*2
resp, err := c.doReqGated(req)
if err != nil {
return errorf("upload http error: %v", err)
}
defer resp.Body.Close()
// check error from earlier copy
if err := <-copyResult; err != nil {
return errorf("failed to copy contents into multipart writer: %v", err)
}
// The only valid HTTP responses are 200 and 303.
if resp.StatusCode != 200 && resp.StatusCode != 303 {
return errorf("invalid http response %d in upload response", resp.StatusCode)
}
if resp.StatusCode == 303 {
otherLocation := resp.Header.Get("Location")
if otherLocation == "" {
return errorf("303 without a Location")
}
baseURL, _ := url.Parse(uploadURL)
absURL, err := baseURL.Parse(otherLocation)
if err != nil {
return errorf("303 Location URL relative resolve error: %v", err)
}
otherLocation = absURL.String()
resp, err = http.Get(otherLocation)
if err != nil {
return errorf("error following 303 redirect after upload: %v", err)
}
}
var ures protocol.UploadResponse
if err := httputil.DecodeJSON(resp, &ures); err != nil {
return errorf("error in upload response: %v", err)
}
if ures.ErrorText != "" {
c.log.Printf("Blob server reports error: %s", ures.ErrorText)
}
expectedSize := uint32(bodySize)
for _, sb := range ures.Received {
if sb.Ref != h.BlobRef {
continue
}
if sb.Size != expectedSize {
return errorf("Server got blob %v, but reports wrong length (%v; we sent %d)",
sb.Ref, sb.Size, expectedSize)
}
c.statsMutex.Lock()
c.stats.Uploads.Blobs++
c.stats.Uploads.Bytes += bodySize
c.statsMutex.Unlock()
if pr.Size <= 0 {
pr.Size = sb.Size
}
c.haveCache.NoteBlobExists(pr.BlobRef, pr.Size)
return pr, nil
}
return nil, errors.New("Server didn't receive blob.")
}