Exemplo n.º 1
0
// UpdateShareChain reads the schema of b from r, and instructs the client that
// all blob refs found in this schema should use b as a preceding chain link, in
// all subsequent shared blobs fetches. If the client was not created with
// NewFromShareRoot, ErrNotSharing is returned.
func (c *Client) UpdateShareChain(b blob.Ref, r io.Reader) error {
	c.viaMu.Lock()
	defer c.viaMu.Unlock()
	if c.via == nil {
		// Not in sharing mode, so return immediately.
		return ErrNotSharing
	}
	// Slurp 1 MB to find references to other blobrefs for the via path.
	var buf bytes.Buffer
	const maxSlurp = 1 << 20
	if _, err := io.Copy(&buf, io.LimitReader(r, maxSlurp)); err != nil {
		return err
	}
	// If it looks like a JSON schema blob (starts with '{')
	if schema.LikelySchemaBlob(buf.Bytes()) {
		for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) {
			br, ok := blob.Parse(blobstr)
			if !ok {
				log.Printf("Invalid blob ref %q noticed in schema of %v", blobstr, b)
				continue
			}
			c.via[br] = b
		}
	}
	return nil
}
Exemplo n.º 2
0
func (c *Client) FetchVia(b *blobref.BlobRef, v []*blobref.BlobRef) (io.ReadCloser, int64, error) {
	pfx, err := c.prefix()
	if err != nil {
		return nil, 0, err
	}
	url := fmt.Sprintf("%s/camli/%s", pfx, b)

	if len(v) > 0 {
		buf := bytes.NewBufferString(url)
		buf.WriteString("?via=")
		for i, br := range v {
			if i != 0 {
				buf.WriteString(",")
			}
			buf.WriteString(br.String())
		}
		url = buf.String()
	}

	req := c.newRequest("GET", url)
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if resp.StatusCode != 200 {
		return nil, 0, errors.New(fmt.Sprintf("Got status code %d from blobserver for %s", resp.StatusCode, b))
	}

	size := resp.ContentLength
	if size == -1 {
		return nil, 0, errors.New("blobserver didn't return a Content-Length for blob")
	}

	if c.via == nil {
		// Not in sharing mode, so return immediately.
		return resp.Body, size, nil
	}

	// Slurp 1 MB to find references to other blobrefs for the via path.
	const maxSlurp = 1 << 20
	var buf bytes.Buffer
	_, err = io.Copy(&buf, io.LimitReader(resp.Body, maxSlurp))
	if err != nil {
		return nil, 0, err
	}
	// If it looks like a JSON schema blob (starts with '{')
	if schema.LikelySchemaBlob(buf.Bytes()) {
		for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) {
			c.via[blobstr] = b.String()
		}
	}
	// Read from the multireader, but close the HTTP response body.
	type rc struct {
		io.Reader
		io.Closer
	}
	return rc{io.MultiReader(&buf, resp.Body), resp.Body}, size, nil
}
Exemplo n.º 3
0
func (asr AndroidStatusReceiver) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (blobref.SizedBlobRef, error) {
	// Sniff the first 1KB of it and don't print the stats if it looks like it was just a schema
	// blob.  We won't update the progress bar for that yet.
	var buf [1024]byte
	contents := buf[:0]
	sb, err := asr.Sr.ReceiveBlob(blob, io.TeeReader(source, writeUntilSliceFull{&contents}))
	if err == nil && !schema.LikelySchemaBlob(contents) {
		statBlobUploaded.Incr(1)
		asr.noteChunkOnServer(sb)
	}
	return sb, err
}
Exemplo n.º 4
0
func (sn *BlobSniffer) bufferIsCamliJSON() bool {
	buf := sn.header
	if !schema.LikelySchemaBlob(buf) {
		return false
	}
	blob, err := schema.BlobFromReader(sn.br, bytes.NewReader(buf))
	if err != nil {
		return false
	}
	sn.meta = blob
	return true
}
Exemplo n.º 5
0
func (c *Client) FetchVia(b blob.Ref, v []blob.Ref) (body io.ReadCloser, size int64, err error) {
	pfx, err := c.blobPrefix()
	if err != nil {
		return nil, 0, err
	}
	url := fmt.Sprintf("%s/%s", pfx, b)

	if len(v) > 0 {
		buf := bytes.NewBufferString(url)
		buf.WriteString("?via=")
		for i, br := range v {
			if i != 0 {
				buf.WriteString(",")
			}
			buf.WriteString(br.String())
		}
		url = buf.String()
	}

	req := c.newRequest("GET", url)
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer func() {
		if err != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode != 200 {
		return nil, 0, fmt.Errorf("Got status code %d from blobserver for %s", resp.StatusCode, b)
	}

	var buf bytes.Buffer
	var reader io.Reader = io.MultiReader(&buf, resp.Body)
	var closer io.Closer = resp.Body
	size = resp.ContentLength
	if size == -1 {
		// Might be compressed. Slurp it to memory.
		n, err := io.CopyN(&buf, resp.Body, blobserver.MaxBlobSize+1)
		if n > blobserver.MaxBlobSize {
			return nil, 0, fmt.Errorf("Blob %b over %d bytes; not reading more", b, blobserver.MaxBlobSize)
		}
		if err == nil {
			panic("unexpected")
		} else if err == io.EOF {
			size = n
			reader, closer = &buf, types.NopCloser
		} else {
			return nil, 0, fmt.Errorf("Error reading %s: %v", b, err)
		}
	}

	var rc io.ReadCloser = struct {
		io.Reader
		io.Closer
	}{reader, closer}

	if c.via == nil {
		// Not in sharing mode, so return immediately.
		return rc, size, nil
	}

	// Slurp 1 MB to find references to other blobrefs for the via path.
	if buf.Len() == 0 {
		const maxSlurp = 1 << 20
		_, err = io.Copy(&buf, io.LimitReader(resp.Body, maxSlurp))
		if err != nil {
			return nil, 0, err
		}
	}
	// If it looks like a JSON schema blob (starts with '{')
	if schema.LikelySchemaBlob(buf.Bytes()) {
		for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) {
			c.via[blobstr] = b.String()
		}
	}
	return rc, size, nil
}