Example #1
0
// bytesHaveSchemaLink reports whether bb is a valid Camlistore schema
// blob and has target somewhere in a schema field used to represent a
// Merkle-tree-ish file or directory.
func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool {
	// Fast path for no:
	if !bytes.Contains(bb, []byte(target.String())) {
		return false
	}
	b, err := schema.BlobFromReader(br, bytes.NewReader(bb))
	if err != nil {
		return false
	}
	typ := b.Type()
	switch typ {
	case "file", "blob":
		for _, bp := range b.ByteParts() {
			if bp.BlobRef.Valid() {
				return bp.BlobRef == target
			}
			if bp.BytesRef.Valid() {
				return bp.BytesRef == target
			}
		}
	case "directory":
		if d, ok := b.DirectoryEntries(); ok {
			return d == target
		}
	case "static-set":
		for _, m := range b.StaticSetMembers() {
			if m == target {
				return true
			}
		}
	}
	return false
}
Example #2
0
// SetShareTarget sets the target of share claim.
// It panics if bb isn't a "share" claim type.
func (bb *Builder) SetShareTarget(t blob.Ref) *Builder {
	if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
		panic("called SetShareTarget on non-share")
	}
	bb.m["target"] = t.String()
	return bb
}
Example #3
0
func (sto *driveStorage) ReceiveBlob(b blob.Ref, source io.Reader) (blob.SizedRef, error) {
	file, err := sto.service.Upsert(b.String(), source)
	if err != nil {
		return blob.SizedRef{Ref: b, Size: 0}, err
	}
	return blob.SizedRef{Ref: b, Size: file.FileSize}, err
}
Example #4
0
func (s *Storage) Fetch(br blob.Ref) (rc io.ReadCloser, size uint32, err error) {
	if s.cache != nil {
		if rc, size, err = s.cache.Fetch(br); err == nil {
			return
		}
	}
	// TODO(mpl): use context from caller, once one is available (issue 733)
	r, err := s.client.Bucket(s.bucket).Object(s.dirPrefix + br.String()).NewReader(context.TODO())
	if err == storage.ErrObjectNotExist {
		return nil, 0, os.ErrNotExist
	}
	if err != nil {
		return nil, 0, err
	}
	if r.Size() >= 1<<32 {
		r.Close()
		return nil, 0, errors.New("object larger than a uint32")
	}
	size = uint32(r.Size())
	if size > constants.MaxBlobSize {
		r.Close()
		return nil, size, errors.New("object too big")
	}
	return r, size, nil
}
Example #5
0
func (h *SimpleBlobHub) NotifyBlobReceived(br blob.Ref) {
	h.l.Lock()
	defer h.l.Unlock()

	// Callback channels to notify, nil until non-empty
	var notify []chan blob.Ref

	// Append global listeners
	for ch, _ := range h.listeners {
		notify = append(notify, ch)
	}

	// Append blob-specific listeners
	if h.blobListeners != nil {
		blobstr := br.String()
		if set, ok := h.blobListeners[blobstr]; ok {
			for ch, _ := range set {
				notify = append(notify, ch)
			}
		}
	}

	// Run in a separate Goroutine so NotifyBlobReceived doesn't block
	// callers if callbacks are slow.
	go func() {
		for _, ch := range notify {
			ch <- br
		}
	}()
}
Example #6
0
// Given a blobref and a few hex characters of the digest of the next hop, return the complete
// blobref of the prefix, if that's a valid next hop.
func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.Ref, err error) {
	// TODO: this is a linear scan right now. this should be
	// optimized to use a new database table of members so this is
	// a quick lookup.  in the meantime it should be in memcached
	// at least.
	if len(prefix) < 8 {
		return blob.Ref{}, fmt.Errorf("Member prefix %q too small", prefix)
	}
	dr := sh.NewDescribeRequest()
	dr.Describe(parent, 1)
	res, err := dr.Result()
	if err != nil {
		return
	}
	des, ok := res[parent.String()]
	if !ok {
		return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent)
	}
	if des.Permanode != nil {
		if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) {
			return cr, nil
		}
		for _, member := range des.Members() {
			if strings.HasPrefix(member.BlobRef.Digest(), prefix) {
				return member.BlobRef, nil
			}
		}
	}
	return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent)
}
Example #7
0
func (s *Storage) Fetch(br blob.Ref) (rc io.ReadCloser, size uint32, err error) {
	if s.cache != nil {
		if rc, size, err = s.cache.Fetch(br); err == nil {
			return
		}
	}
	r, fi, err := s.cl.DownloadFileByName(s.b.Name, s.dirPrefix+br.String())
	if err, ok := err.(*b2.Error); ok && err.Status == 404 {
		return nil, 0, os.ErrNotExist
	}
	if err != nil {
		return nil, 0, err
	}

	if br.HashName() == "sha1" && fi.ContentSHA1 != br.Digest() {
		return nil, 0, errors.New("b2: remote ContentSHA1 mismatch")
	}

	if fi.ContentLength >= 1<<32 {
		r.Close()
		return nil, 0, errors.New("object larger than a uint32")
	}
	size = uint32(fi.ContentLength)
	if size > constants.MaxBlobSize {
		r.Close()
		return nil, size, errors.New("object too big")
	}
	return r, size, nil
}
Example #8
0
func (s *Storage) Fetch(ref blob.Ref) (file io.ReadCloser, size uint32, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if s.lru != nil {
		s.lru.Get(ref.String()) // force to head
	}
	if s.m == nil {
		err = os.ErrNotExist
		return
	}
	b, ok := s.m[ref]
	if !ok {
		err = os.ErrNotExist
		return
	}
	size = uint32(len(b))
	atomic.AddInt64(&s.blobsFetched, 1)
	atomic.AddInt64(&s.bytesFetched, int64(len(b)))

	return struct {
		*io.SectionReader
		io.Closer
	}{
		io.NewSectionReader(bytes.NewReader(b), 0, int64(size)),
		types.NopCloser,
	}, size, nil
}
Example #9
0
// populateMutationMap populates keys & values that will be committed
// into the returned map.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) {
	// TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when
	// e.g in populateClaim we detect a bogus claim (which does not yield an error)?
	mm := &mutationMap{
		kv: map[string]string{
			"have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()),
			"meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()),
		},
	}

	if blob, ok := sniffer.SchemaBlob(); ok {
		switch blob.Type() {
		case "claim":
			if err := ix.populateClaim(blob, mm); err != nil {
				return nil, err
			}
		case "file":
			if err := ix.populateFile(fetcher, blob, mm); err != nil {
				return nil, err
			}
		case "directory":
			if err := ix.populateDir(fetcher, blob, mm); err != nil {
				return nil, err
			}
		}
	}

	return mm, nil
}
Example #10
0
func (c *SQLiteHaveCache) StatBlobCache(br blob.Ref) (size int64, ok bool) {
	if !br.Valid() {
		return
	}
	// TODO(mpl): is it enough that we know it's a valid blobref to avoid any injection risk ?
	query := blobSizeQuery + fmt.Sprintf("'%v';\n", br.String())
	c.mu.Lock()
	defer c.mu.Unlock()
	err := c.startSQLiteChild()
	if err != nil {
		log.Fatalf("Could not start sqlite child process: %v", err)
	}
	_, err = c.w.Write([]byte(query))
	if err != nil {
		log.Fatalf("failed to query have cache: %v", err)
	}
	out, err := c.r.ReadString('\n')
	if err != nil {
		log.Fatalf("failed to read have cache query result: %v", err)
	}
	out = strings.TrimRight(out, "\n")
	if out == noResult {
		return
	}
	size, err = strconv.ParseInt(out, 10, 64)
	if err != nil {
		log.Fatalf("Bogus blob size in %v table: %v", haveTableName, err)
	}
	return size, true
}
Example #11
0
func (fr *FileReader) getSuperset(br blob.Ref) (*superset, error) {
	if root := fr.rootReader(); root != fr {
		return root.getSuperset(br)
	}
	brStr := br.String()
	ssi, err := fr.sfg.Do(brStr, func() (interface{}, error) {
		fr.ssmmu.Lock()
		ss, ok := fr.ssm[br]
		fr.ssmmu.Unlock()
		if ok {
			return ss, nil
		}
		rc, _, err := fr.fetcher.Fetch(br)
		if err != nil {
			return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
		}
		defer rc.Close()
		ss, err = parseSuperset(rc)
		if err != nil {
			return nil, err
		}
		ss.BlobRef = br
		fr.ssmmu.Lock()
		defer fr.ssmmu.Unlock()
		fr.ssm[br] = ss
		return ss, nil
	})
	if err != nil {
		return nil, err
	}
	return ssi.(*superset), nil
}
Example #12
0
func (x *Index) GetImageInfo(fileRef blob.Ref) (*search.ImageInfo, error) {
	// it might be that the key does not exist because image.DecodeConfig failed earlier
	// (because of unsupported JPEG features like progressive mode).
	key := keyImageSize.Key(fileRef.String())
	dim, err := x.s.Get(key)
	if err == ErrNotFound {
		err = os.ErrNotExist
	}
	if err != nil {
		return nil, err
	}
	valPart := strings.Split(dim, "|")
	if len(valPart) != 2 {
		return nil, fmt.Errorf("index: bogus key %q = %q", key, dim)
	}
	width, err := strconv.Atoi(valPart[0])
	if err != nil {
		return nil, fmt.Errorf("index: bogus integer at position 0 in key %q: %q", key, valPart[0])
	}
	height, err := strconv.Atoi(valPart[1])
	if err != nil {
		return nil, fmt.Errorf("index: bogus integer at position 1 in key %q: %q", key, valPart[1])
	}

	imgInfo := &search.ImageInfo{
		Width:  width,
		Height: height,
	}
	return imgInfo, nil
}
Example #13
0
func (ph *publishHandler) deepDescribe(br blob.Ref) (*search.DescribeResponse, error) {
	res, err := ph.cl.Query(&search.SearchQuery{
		Constraint: &search.Constraint{
			BlobRefPrefix: br.String(),
			CamliType:     "permanode",
		},
		Describe: &search.DescribeRequest{
			ThumbnailSize: 1000,
			Depth:         1,
			Rules: []*search.DescribeRule{
				{
					Attrs: []string{"camliContent", "camliContentImage", "camliMember", "camliPath:*"},
				},
			},
		},
		Limit: -1,
	})
	if err != nil {
		return nil, fmt.Errorf("Could not deep describe %v: %v", br, err)
	}
	if res == nil || res.Describe == nil {
		return nil, fmt.Errorf("no describe result for %v", br)
	}
	return res.Describe, nil
}
Example #14
0
func (c *FlatHaveCache) NoteBlobExists(br blob.Ref, size int64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if size < 0 {
		panic("negative size")
	}
	k := br.String()
	if c.m[k] == size {
		// dup
		return
	}
	c.m[k] = size

	if c.af == nil {
		var err error
		c.af, err = os.OpenFile(c.filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			log.Printf("opening have-cache for append: %v", err)
			return
		}
	}
	// TODO: flocking. see leveldb-go.
	c.af.Seek(0, os.SEEK_END)
	c.af.Write([]byte(fmt.Sprintf("%s %d\n", k, size)))
}
Example #15
0
func (tf *Fetcher) Fetch(ref blob.Ref) (file io.ReadCloser, size uint32, err error) {
	if tf.FetchErr != nil {
		if err = tf.FetchErr(); err != nil {
			return
		}
	}
	tf.mu.RLock()
	defer tf.mu.RUnlock()
	if tf.m == nil {
		err = os.ErrNotExist
		return
	}
	tb, ok := tf.m[ref.String()]
	if !ok {
		err = os.ErrNotExist
		return
	}
	size = uint32(len(tb.Contents))
	return struct {
		*io.SectionReader
		io.Closer
	}{
		io.NewSectionReader(strings.NewReader(tb.Contents), 0, int64(size)),
		types.NopCloser,
	}, size, nil
}
Example #16
0
// if not found, err == nil.
func (s *storage) getMetaRow(br blob.Ref) (meta, error) {
	v, err := s.meta.Get(blobMetaPrefix + br.String())
	if err == sorted.ErrNotFound {
		return meta{}, nil
	}
	return parseMetaRow([]byte(v))
}
Example #17
0
func (m *thumbMeta) Put(key string, br blob.Ref) error {
	m.mem.Add(key, br)
	if m.kv != nil {
		return m.kv.Set(key, br.String())
	}
	return nil
}
Example #18
0
func (tf *Fetcher) Fetch(ref blob.Ref) (file types.ReadSeekCloser, size int64, err error) {
	if tf.FetchErr != nil {
		if err = tf.FetchErr(); err != nil {
			return
		}
	}
	tf.l.Lock()
	defer tf.l.Unlock()
	if tf.m == nil {
		err = os.ErrNotExist
		return
	}
	tb, ok := tf.m[ref.String()]
	if !ok {
		err = os.ErrNotExist
		return
	}
	size = int64(len(tb.Contents))
	return struct {
		*io.SectionReader
		io.Closer
	}{
		io.NewSectionReader(strings.NewReader(tb.Contents), 0, size),
		dummyCloser,
	}, size, nil
}
Example #19
0
// ScaledCached reads the scaled version of the image in file,
// if it is in cache. On success, the image format is returned.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string, err error) {
	name := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
	br, err := ih.sc.Get(name)
	if err != nil {
		return format, fmt.Errorf("%v: %v", name, err)
	}
	fr, err := ih.cached(br)
	if err != nil {
		return format, fmt.Errorf("No cache hit for %v: %v", br, err)
	}
	defer fr.Close()
	_, err = io.Copy(buf, fr)
	if err != nil {
		return format, fmt.Errorf("error reading cached thumbnail %v: %v", name, err)
	}
	mime := magic.MIMEType(buf.Bytes())
	if mime == "" {
		return format, fmt.Errorf("error with cached thumbnail %v: unknown mime type", name)
	}
	pieces := strings.Split(mime, "/")
	if len(pieces) < 2 {
		return format, fmt.Errorf("error with cached thumbnail %v: bogus mime type", name)
	}
	if pieces[0] != "image" {
		return format, fmt.Errorf("error with cached thumbnail %v: not an image", name)
	}
	return pieces[1], nil
}
Example #20
0
func (ph *publishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) {
	if suffix == "" {
		return root, nil
	}
	// TODO: verify it's optimized: http://camlistore.org/issue/405
	result, err := ph.cl.Query(&search.SearchQuery{
		Limit: 1,
		Constraint: &search.Constraint{
			Permanode: &search.PermanodeConstraint{
				SkipHidden: true,
				Relation: &search.RelationConstraint{
					Relation: "parent",
					EdgeType: "camliPath:" + suffix,
					Any: &search.Constraint{
						BlobRefPrefix: root.String(),
					},
				},
			},
		},
	})
	if err != nil {
		return blob.Ref{}, err
	}
	if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() {
		return blob.Ref{}, os.ErrNotExist
	}
	return result.Blobs[0].Blob, nil
}
Example #21
0
// SearchExistingFileSchema does a search query looking for an
// existing file with entire contents of wholeRef, then does a HEAD
// request to verify the file still exists on the server.  If so,
// it returns that file schema's blobref.
//
// May return (zero, nil) on ENOENT. A non-nil error is only returned
// if there were problems searching.
func (c *Client) SearchExistingFileSchema(wholeRef blob.Ref) (blob.Ref, error) {
	sr, err := c.SearchRoot()
	if err != nil {
		return blob.Ref{}, err
	}
	url := sr + "camli/search/files?wholedigest=" + wholeRef.String()
	req := c.newRequest("GET", url)
	res, err := c.doReqGated(req)
	if err != nil {
		return blob.Ref{}, err
	}
	if res.StatusCode != 200 {
		body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
		res.Body.Close()
		return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
	}
	var ress struct {
		Files []blob.Ref `json:"files"`
	}
	if err := httputil.DecodeJSON(res, &ress); err != nil {
		return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
	}
	if len(ress.Files) == 0 {
		return blob.Ref{}, nil
	}
	for _, f := range ress.Files {
		if c.FileHasContents(f, wholeRef) {
			return f, nil
		}
	}
	return blob.Ref{}, nil
}
Example #22
0
func (ph *publishHandler) describeMembers(br blob.Ref) (*search.SearchResult, error) {
	res, err := ph.cl.Query(&search.SearchQuery{
		Constraint: &search.Constraint{
			Permanode: &search.PermanodeConstraint{
				Relation: &search.RelationConstraint{
					Relation: "parent",
					Any: &search.Constraint{
						BlobRefPrefix: br.String(),
					},
				},
			},
			CamliType: "permanode",
		},
		Describe: &search.DescribeRequest{
			Depth: 1,
			Rules: []*search.DescribeRule{
				{
					Attrs: []string{"camliContent", "camliContentImage"},
				},
			},
		},
		Limit: -1,
	})
	if err != nil {
		return nil, fmt.Errorf("Could not describe members of %v: %v", br, err)
	}
	return res, nil
}
Example #23
0
func (x *Index) GetBlobMeta(br blob.Ref) (camtypes.BlobMeta, error) {
	if x.corpus != nil {
		return x.corpus.GetBlobMeta(br)
	}
	key := "meta:" + br.String()
	meta, err := x.s.Get(key)
	if err == sorted.ErrNotFound {
		err = os.ErrNotExist
	}
	if err != nil {
		return camtypes.BlobMeta{}, err
	}
	pos := strings.Index(meta, "|")
	if pos < 0 {
		panic(fmt.Sprintf("Bogus index row for key %q: got value %q", key, meta))
	}
	size, err := strconv.ParseUint(meta[:pos], 10, 32)
	if err != nil {
		return camtypes.BlobMeta{}, err
	}
	mime := meta[pos+1:]
	return camtypes.BlobMeta{
		Ref:       br,
		Size:      uint32(size),
		CamliType: camliTypeFromMIME(mime),
	}, nil
}
Example #24
0
// populateMutation populates keys & values into the provided BatchMutation.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutation(br blob.Ref, sniffer *BlobSniffer, bm BatchMutation) error {
	bm.Set("have:"+br.String(), fmt.Sprintf("%d", sniffer.Size()))
	bm.Set("meta:"+br.String(), fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()))

	if blob, ok := sniffer.SchemaBlob(); ok {
		switch blob.Type() {
		case "claim":
			if err := ix.populateClaim(blob, bm); err != nil {
				return err
			}
		case "permanode":
			//if err := mi.populatePermanode(blobRef, camli, bm); err != nil {
			//return err
			//}
		case "file":
			if err := ix.populateFile(blob, bm); err != nil {
				return err
			}
		case "directory":
			if err := ix.populateDir(blob, bm); err != nil {
				return err
			}
		}
	}
	return nil
}
Example #25
0
func (s *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
	var buf bytes.Buffer
	size, err := io.Copy(&buf, source)
	if err != nil {
		return blob.SizedRef{}, err
	}

	b := bytes.NewReader(buf.Bytes())
	fi, err := s.b.Upload(b, s.dirPrefix+br.String(), "")
	if err != nil {
		return blob.SizedRef{}, err
	}

	if int64(fi.ContentLength) != size {
		return blob.SizedRef{}, fmt.Errorf("b2: expected ContentLength %d, got %d", size, fi.ContentLength)
	}
	if br.HashName() == "sha1" && fi.ContentSHA1 != br.Digest() {
		return blob.SizedRef{}, fmt.Errorf("b2: expected ContentSHA1 %s, got %s", br.Digest(), fi.ContentSHA1)
	}

	if s.cache != nil {
		// NoHash because it's already verified if we read it without
		// errors from the source, and uploaded it without mismatch.
		blobserver.ReceiveNoHash(s.cache, br, &buf)
	}
	return blob.SizedRef{Ref: br, Size: uint32(size)}, nil
}
Example #26
0
// ScaledCached reads the scaled version of the image in file,
// if it is in cache and writes it to buf.
//
// On successful read and population of buf, the returned format is non-empty.
// Almost all errors are not interesting. Real errors will be logged.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) {
	key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
	br, err := ih.thumbMeta.Get(key)
	if err == errCacheMiss {
		return
	}
	if err != nil {
		log.Printf("Warning: thumbnail cachekey(%q)->meta lookup error: %v", key, err)
		return
	}
	fr, err := ih.cached(br)
	if err != nil {
		return
	}
	defer fr.Close()
	_, err = io.Copy(buf, fr)
	if err != nil {
		return
	}
	mime := magic.MIMEType(buf.Bytes())
	if format = strings.TrimPrefix(mime, "image/"); format == mime {
		log.Printf("Warning: unescaped MIME type %q of %v file for thumbnail %q", mime, br, key)
		return
	}
	return format
}
Example #27
0
func (fi *FakeIndex) AddClaim(owner, permanode blob.Ref, claimType, attr, value string) {
	fi.lk.Lock()
	defer fi.lk.Unlock()
	date := fi.nextDate()

	claim := &search.Claim{
		Permanode: permanode,
		Signer:    blob.Ref{},
		BlobRef:   blob.Ref{},
		Date:      date,
		Type:      claimType,
		Attr:      attr,
		Value:     value,
	}
	key := permanode.String() + "/" + owner.String()
	fi.ownerClaims[key] = append(fi.ownerClaims[key], claim)

	if claimType == "set-attribute" && strings.HasPrefix(attr, "camliPath:") {
		suffix := attr[len("camliPath:"):]
		path := &search.Path{
			Target: blob.MustParse(value),
			Suffix: suffix,
		}
		fi.path[fmt.Sprintf("%s\x00%s\x00%s", owner, permanode, suffix)] = path
	}
}
Example #28
0
func (c *Client) FetchVia(b blob.Ref, v []blob.Ref) (io.ReadCloser, int64, error) {
	pfx, err := c.blobPrefix()
	if err != nil {
		return nil, 0, err
	}
	url := fmt.Sprintf("%s/%s", pfx, b)

	if len(v) > 0 {
		buf := bytes.NewBufferString(url)
		buf.WriteString("?via=")
		for i, br := range v {
			if i != 0 {
				buf.WriteString(",")
			}
			buf.WriteString(br.String())
		}
		url = buf.String()
	}

	req := c.newRequest("GET", url)
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if resp.StatusCode != 200 {
		return nil, 0, errors.New(fmt.Sprintf("Got status code %d from blobserver for %s", resp.StatusCode, b))
	}

	size := resp.ContentLength
	if size == -1 {
		return nil, 0, errors.New("blobserver didn't return a Content-Length for blob")
	}

	if c.via == nil {
		// Not in sharing mode, so return immediately.
		return resp.Body, size, nil
	}

	// Slurp 1 MB to find references to other blobrefs for the via path.
	const maxSlurp = 1 << 20
	var buf bytes.Buffer
	_, err = io.Copy(&buf, io.LimitReader(resp.Body, maxSlurp))
	if err != nil {
		return nil, 0, err
	}
	// If it looks like a JSON schema blob (starts with '{')
	if schema.LikelySchemaBlob(buf.Bytes()) {
		for _, blobstr := range blobsRx.FindAllString(buf.String(), -1) {
			c.via[blobstr] = b.String()
		}
	}
	// Read from the multireader, but close the HTTP response body.
	type rc struct {
		io.Reader
		io.Closer
	}
	return rc{io.MultiReader(&buf, resp.Body), resp.Body}, size, nil
}
Example #29
0
// uploadFilePermanode creates and uploads the planned permanode (with sum as a
// fixed key) associated with the file blobref fileRef.
// It also sets the optional tags for this permanode.
func (up *Uploader) uploadFilePermanode(sum string, fileRef blob.Ref, claimTime time.Time) error {
	// Use a fixed time value for signing; not using modtime
	// so two identical files don't have different modtimes?
	// TODO(bradfitz): consider this more?
	permaNodeSigTime := time.Unix(0, 0)
	permaNode, err := up.UploadPlannedPermanode(sum, permaNodeSigTime)
	if err != nil {
		return fmt.Errorf("Error uploading planned permanode: %v", err)
	}
	handleResult("node-permanode", permaNode, nil)

	contentAttr := schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", fileRef.String())
	contentAttr.SetClaimDate(claimTime)
	signer, err := up.Signer()
	if err != nil {
		return err
	}
	signed, err := contentAttr.SignAt(signer, claimTime)
	if err != nil {
		return fmt.Errorf("Failed to sign content claim: %v", err)
	}
	put, err := up.uploadString(signed)
	if err != nil {
		return fmt.Errorf("Error uploading permanode's attribute: %v", err)
	}

	handleResult("node-permanode-contentattr", put, nil)
	if tags := up.fileOpts.tags(); len(tags) > 0 {
		errch := make(chan error)
		for _, tag := range tags {
			go func(tag string) {
				m := schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag)
				m.SetClaimDate(claimTime)
				signed, err := m.SignAt(signer, claimTime)
				if err != nil {
					errch <- fmt.Errorf("Failed to sign tag claim: %v", err)
					return
				}
				put, err := up.uploadString(signed)
				if err != nil {
					errch <- fmt.Errorf("Error uploading permanode's tag attribute %v: %v", tag, err)
					return
				}
				handleResult("node-permanode-tag", put, nil)
				errch <- nil
			}(tag)
		}

		for range tags {
			if e := <-errch; e != nil && err == nil {
				err = e
			}
		}
		if err != nil {
			return err
		}
	}
	return nil
}
Example #30
0
// Serves details of accounts at http://host/importer/twitter/sha1-23098429382934
func (h *Host) serveImporterAccount(w http.ResponseWriter, r *http.Request, imp *importer, acctRef blob.Ref) {
	ia, err := imp.account(acctRef)
	if err != nil {
		http.Error(w, "Unknown or invalid importer account "+acctRef.String()+": "+err.Error(), 400)
		return
	}
	ia.ServeHTTP(w, r)
}