// UpdateShareChain reads the schema of b from r, and instructs the client that // all blob refs found in this schema should use b as a preceding chain link, in // all subsequent shared blobs fetches. If the client was not created with // NewFromShareRoot, ErrNotSharing is returned. func (c *Client) UpdateShareChain(b blob.Ref, r io.Reader) error { c.viaMu.Lock() defer c.viaMu.Unlock() if c.via == nil { // Not in sharing mode, so return immediately. return ErrNotSharing } // Slurp 1 MB to find references to other blobrefs for the via path. var buf bytes.Buffer const maxSlurp = 1 << 20 if _, err := io.Copy(&buf, io.LimitReader(r, maxSlurp)); err != nil { return err } // If it looks like a JSON schema blob (starts with '{') if schema.LikelySchemaBlob(buf.Bytes()) { for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) { br, ok := blob.Parse(blobstr) if !ok { log.Printf("Invalid blob ref %q noticed in schema of %v", blobstr, b) continue } c.via[br] = b } } return nil }
func (c *Client) FetchVia(b *blobref.BlobRef, v []*blobref.BlobRef) (io.ReadCloser, int64, error) { pfx, err := c.prefix() if err != nil { return nil, 0, err } url := fmt.Sprintf("%s/camli/%s", pfx, b) if len(v) > 0 { buf := bytes.NewBufferString(url) buf.WriteString("?via=") for i, br := range v { if i != 0 { buf.WriteString(",") } buf.WriteString(br.String()) } url = buf.String() } req := c.newRequest("GET", url) resp, err := c.httpClient.Do(req) if err != nil { return nil, 0, err } if resp.StatusCode != 200 { return nil, 0, errors.New(fmt.Sprintf("Got status code %d from blobserver for %s", resp.StatusCode, b)) } size := resp.ContentLength if size == -1 { return nil, 0, errors.New("blobserver didn't return a Content-Length for blob") } if c.via == nil { // Not in sharing mode, so return immediately. return resp.Body, size, nil } // Slurp 1 MB to find references to other blobrefs for the via path. const maxSlurp = 1 << 20 var buf bytes.Buffer _, err = io.Copy(&buf, io.LimitReader(resp.Body, maxSlurp)) if err != nil { return nil, 0, err } // If it looks like a JSON schema blob (starts with '{') if schema.LikelySchemaBlob(buf.Bytes()) { for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) { c.via[blobstr] = b.String() } } // Read from the multireader, but close the HTTP response body. type rc struct { io.Reader io.Closer } return rc{io.MultiReader(&buf, resp.Body), resp.Body}, size, nil }
func (asr AndroidStatusReceiver) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (blobref.SizedBlobRef, error) { // Sniff the first 1KB of it and don't print the stats if it looks like it was just a schema // blob. We won't update the progress bar for that yet. var buf [1024]byte contents := buf[:0] sb, err := asr.Sr.ReceiveBlob(blob, io.TeeReader(source, writeUntilSliceFull{&contents})) if err == nil && !schema.LikelySchemaBlob(contents) { statBlobUploaded.Incr(1) asr.noteChunkOnServer(sb) } return sb, err }
func (sn *BlobSniffer) bufferIsCamliJSON() bool { buf := sn.header if !schema.LikelySchemaBlob(buf) { return false } blob, err := schema.BlobFromReader(sn.br, bytes.NewReader(buf)) if err != nil { return false } sn.meta = blob return true }
func (c *Client) FetchVia(b blob.Ref, v []blob.Ref) (body io.ReadCloser, size int64, err error) { pfx, err := c.blobPrefix() if err != nil { return nil, 0, err } url := fmt.Sprintf("%s/%s", pfx, b) if len(v) > 0 { buf := bytes.NewBufferString(url) buf.WriteString("?via=") for i, br := range v { if i != 0 { buf.WriteString(",") } buf.WriteString(br.String()) } url = buf.String() } req := c.newRequest("GET", url) resp, err := c.httpClient.Do(req) if err != nil { return nil, 0, err } defer func() { if err != nil { resp.Body.Close() } }() if resp.StatusCode != 200 { return nil, 0, fmt.Errorf("Got status code %d from blobserver for %s", resp.StatusCode, b) } var buf bytes.Buffer var reader io.Reader = io.MultiReader(&buf, resp.Body) var closer io.Closer = resp.Body size = resp.ContentLength if size == -1 { // Might be compressed. Slurp it to memory. n, err := io.CopyN(&buf, resp.Body, blobserver.MaxBlobSize+1) if n > blobserver.MaxBlobSize { return nil, 0, fmt.Errorf("Blob %b over %d bytes; not reading more", b, blobserver.MaxBlobSize) } if err == nil { panic("unexpected") } else if err == io.EOF { size = n reader, closer = &buf, types.NopCloser } else { return nil, 0, fmt.Errorf("Error reading %s: %v", b, err) } } var rc io.ReadCloser = struct { io.Reader io.Closer }{reader, closer} if c.via == nil { // Not in sharing mode, so return immediately. return rc, size, nil } // Slurp 1 MB to find references to other blobrefs for the via path. if buf.Len() == 0 { const maxSlurp = 1 << 20 _, err = io.Copy(&buf, io.LimitReader(resp.Body, maxSlurp)) if err != nil { return nil, 0, err } } // If it looks like a JSON schema blob (starts with '{') if schema.LikelySchemaBlob(buf.Bytes()) { for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) { c.via[blobstr] = b.String() } } return rc, size, nil }