func (s *Storage) Fetch(br blob.Ref) (rc io.ReadCloser, size uint32, err error) {
if s.cache != nil {
if rc, size, err = s.cache.Fetch(br); err == nil {
return
}
}
// TODO(mpl): use context from caller, once one is available (issue 733)
r, err := s.client.Bucket(s.bucket).Object(s.dirPrefix + br.String()).NewReader(context.TODO())
if err == storage.ErrObjectNotExist {
return nil, 0, os.ErrNotExist
}
if err != nil {
return nil, 0, err
}
if r.Size() >= 1<<32 {
r.Close()
return nil, 0, errors.New("object larger than a uint32")
}
size = uint32(r.Size())
if size > constants.MaxBlobSize {
r.Close()
return nil, size, errors.New("object too big")
}
return r, size, nil
}
// populateMutationMap populates keys & values that will be committed
// into the returned map.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutationMap(fetcher *missTrackFetcher, br blob.Ref, sniffer *BlobSniffer) (*mutationMap, error) {
// TODO(mpl): shouldn't we remove these two from the map (so they don't get committed) when
// e.g in populateClaim we detect a bogus claim (which does not yield an error)?
mm := &mutationMap{
kv: map[string]string{
"have:" + br.String(): fmt.Sprintf("%d", sniffer.Size()),
"meta:" + br.String(): fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()),
},
}
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
if err := ix.populateClaim(blob, mm); err != nil {
return nil, err
}
case "file":
if err := ix.populateFile(fetcher, blob, mm); err != nil {
return nil, err
}
case "directory":
if err := ix.populateDir(fetcher, blob, mm); err != nil {
return nil, err
}
}
}
return mm, nil
}
// ScaledCached reads the scaled version of the image in file,
// if it is in cache and writes it to buf.
//
// On successful read and population of buf, the returned format is non-empty.
// Almost all errors are not interesting. Real errors will be logged.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) {
key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
br, err := ih.thumbMeta.Get(key)
if err == errCacheMiss {
return
}
if err != nil {
log.Printf("Warning: thumbnail cachekey(%q)->meta lookup error: %v", key, err)
return
}
fr, err := ih.cached(br)
if err != nil {
return
}
defer fr.Close()
_, err = io.Copy(buf, fr)
if err != nil {
return
}
mime := magic.MIMEType(buf.Bytes())
if format = strings.TrimPrefix(mime, "image/"); format == mime {
log.Printf("Warning: unescaped MIME type %q of %v file for thumbnail %q", mime, br, key)
return
}
return format
}
// NewFileReader returns a new FileReader reading the contents of fileBlobRef,
// fetching blobs from fetcher. The fileBlobRef must be of a "bytes" or "file"
// schema blob.
//
// The caller should call Close on the FileReader when done reading.
func NewFileReader(fetcher blob.Fetcher, fileBlobRef blob.Ref) (*FileReader, error) {
// TODO(bradfitz): rename this into bytes reader? but for now it's still
// named FileReader, but can also read a "bytes" schema.
if !fileBlobRef.Valid() {
return nil, errors.New("schema/filereader: NewFileReader blobref invalid")
}
rc, _, err := fetcher.Fetch(fileBlobRef)
if err != nil {
return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
}
defer rc.Close()
ss, err := parseSuperset(rc)
if err != nil {
return nil, fmt.Errorf("schema/filereader: decoding file schema blob: %v", err)
}
ss.BlobRef = fileBlobRef
if ss.Type != "file" && ss.Type != "bytes" {
return nil, fmt.Errorf("schema/filereader: expected \"file\" or \"bytes\" schema blob, got %q", ss.Type)
}
fr, err := ss.NewFileReader(fetcher)
if err != nil {
return nil, fmt.Errorf("schema/filereader: creating FileReader for %s: %v", fileBlobRef, err)
}
return fr, nil
}
func (s *Storage) Fetch(ref blob.Ref) (file io.ReadCloser, size uint32, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
if s.lru != nil {
s.lru.Get(ref.String()) // force to head
}
if s.m == nil {
err = os.ErrNotExist
return
}
b, ok := s.m[ref]
if !ok {
err = os.ErrNotExist
return
}
size = uint32(len(b))
atomic.AddInt64(&s.blobsFetched, 1)
atomic.AddInt64(&s.bytesFetched, int64(len(b)))
return struct {
*io.SectionReader
io.Closer
}{
io.NewSectionReader(bytes.NewReader(b), 0, int64(size)),
types.NopCloser,
}, size, nil
}
func (h *SimpleBlobHub) NotifyBlobReceived(br blob.Ref) {
h.l.Lock()
defer h.l.Unlock()
// Callback channels to notify, nil until non-empty
var notify []chan blob.Ref
// Append global listeners
for ch, _ := range h.listeners {
notify = append(notify, ch)
}
// Append blob-specific listeners
if h.blobListeners != nil {
blobstr := br.String()
if set, ok := h.blobListeners[blobstr]; ok {
for ch, _ := range set {
notify = append(notify, ch)
}
}
}
// Run in a separate Goroutine so NotifyBlobReceived doesn't block
// callers if callbacks are slow.
go func() {
for _, ch := range notify {
ch <- br
}
}()
}
// ScaledCached reads the scaled version of the image in file,
// if it is in cache. On success, the image format is returned.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string, err error) {
name := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
br, err := ih.sc.Get(name)
if err != nil {
return format, fmt.Errorf("%v: %v", name, err)
}
fr, err := ih.cached(br)
if err != nil {
return format, fmt.Errorf("No cache hit for %v: %v", br, err)
}
defer fr.Close()
_, err = io.Copy(buf, fr)
if err != nil {
return format, fmt.Errorf("error reading cached thumbnail %v: %v", name, err)
}
mime := magic.MIMEType(buf.Bytes())
if mime == "" {
return format, fmt.Errorf("error with cached thumbnail %v: unknown mime type", name)
}
pieces := strings.Split(mime, "/")
if len(pieces) < 2 {
return format, fmt.Errorf("error with cached thumbnail %v: bogus mime type", name)
}
if pieces[0] != "image" {
return format, fmt.Errorf("error with cached thumbnail %v: not an image", name)
}
return pieces[1], nil
}
func (x *Index) GetImageInfo(fileRef blob.Ref) (*search.ImageInfo, error) {
// it might be that the key does not exist because image.DecodeConfig failed earlier
// (because of unsupported JPEG features like progressive mode).
key := keyImageSize.Key(fileRef.String())
dim, err := x.s.Get(key)
if err == ErrNotFound {
err = os.ErrNotExist
}
if err != nil {
return nil, err
}
valPart := strings.Split(dim, "|")
if len(valPart) != 2 {
return nil, fmt.Errorf("index: bogus key %q = %q", key, dim)
}
width, err := strconv.Atoi(valPart[0])
if err != nil {
return nil, fmt.Errorf("index: bogus integer at position 0 in key %q: %q", key, valPart[0])
}
height, err := strconv.Atoi(valPart[1])
if err != nil {
return nil, fmt.Errorf("index: bogus integer at position 1 in key %q: %q", key, valPart[1])
}
imgInfo := &search.ImageInfo{
Width: width,
Height: height,
}
return imgInfo, nil
}
func (ph *publishHandler) deepDescribe(br blob.Ref) (*search.DescribeResponse, error) {
res, err := ph.cl.Query(&search.SearchQuery{
Constraint: &search.Constraint{
BlobRefPrefix: br.String(),
CamliType: "permanode",
},
Describe: &search.DescribeRequest{
ThumbnailSize: 1000,
Depth: 1,
Rules: []*search.DescribeRule{
{
Attrs: []string{"camliContent", "camliContentImage", "camliMember", "camliPath:*"},
},
},
},
Limit: -1,
})
if err != nil {
return nil, fmt.Errorf("Could not deep describe %v: %v", br, err)
}
if res == nil || res.Describe == nil {
return nil, fmt.Errorf("no describe result for %v", br)
}
return res.Describe, nil
}
func (c *FlatHaveCache) NoteBlobExists(br blob.Ref, size int64) {
c.mu.Lock()
defer c.mu.Unlock()
if size < 0 {
panic("negative size")
}
k := br.String()
if c.m[k] == size {
// dup
return
}
c.m[k] = size
if c.af == nil {
var err error
c.af, err = os.OpenFile(c.filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
log.Printf("opening have-cache for append: %v", err)
return
}
}
// TODO: flocking. see leveldb-go.
c.af.Seek(0, os.SEEK_END)
c.af.Write([]byte(fmt.Sprintf("%s %d\n", k, size)))
}
func (scv *statCacheValue) unmarshalBinary(data []byte) error {
if scv == nil {
return errors.New("Can't unmarshalBinary into a nil stat cache value")
}
if scv.Fingerprint != "" {
return errors.New("Can't unmarshalBinary into a non empty stat cache value")
}
parts := bytes.SplitN(data, pipe, 3)
if len(parts) != 3 {
return fmt.Errorf("Bogus stat cache value; was expecting fingerprint|blobSize|blobRef, got %q", data)
}
fingerprint := string(parts[0])
buf := bytes.NewReader(parts[1])
var size int32
err := binary.Read(buf, binary.BigEndian, &size)
if err != nil {
return fmt.Errorf("Could not decode blob size from stat cache value part %q: %v", parts[1], err)
}
br := new(blob.Ref)
if err := br.UnmarshalBinary(parts[2]); err != nil {
return fmt.Errorf("Could not unmarshalBinary for %q: %v", parts[2], err)
}
scv.Fingerprint = statFingerprint(fingerprint)
scv.Result = client.PutResult{
BlobRef: *br,
Size: uint32(size),
Skipped: true,
}
return nil
}
func (tf *Fetcher) Fetch(ref blob.Ref) (file io.ReadCloser, size uint32, err error) {
if tf.FetchErr != nil {
if err = tf.FetchErr(); err != nil {
return
}
}
tf.mu.RLock()
defer tf.mu.RUnlock()
if tf.m == nil {
err = os.ErrNotExist
return
}
tb, ok := tf.m[ref.String()]
if !ok {
err = os.ErrNotExist
return
}
size = uint32(len(tb.Contents))
return struct {
*io.SectionReader
io.Closer
}{
io.NewSectionReader(strings.NewReader(tb.Contents), 0, int64(size)),
types.NopCloser,
}, size, nil
}
// if not found, err == nil.
func (s *storage) getMetaRow(br blob.Ref) (meta, error) {
v, err := s.meta.Get(blobMetaPrefix + br.String())
if err == sorted.ErrNotFound {
return meta{}, nil
}
return parseMetaRow([]byte(v))
}
func (ds *DiskStorage) blobDirectory(partition string, b blob.Ref) string {
d := b.Digest()
if len(d) < 6 {
d = d + "______"
}
return filepath.Join(ds.PartitionRoot(partition), b.HashName(), d[0:3], d[3:6])
}
// SetShareTarget sets the target of share claim.
// It panics if bb isn't a "share" claim type.
func (bb *Builder) SetShareTarget(t blob.Ref) *Builder {
if bb.Type() != "claim" || bb.ClaimType() != ShareClaim {
panic("called SetShareTarget on non-share")
}
bb.m["target"] = t.String()
return bb
}
func (c *SQLiteHaveCache) StatBlobCache(br blob.Ref) (size int64, ok bool) {
if !br.Valid() {
return
}
// TODO(mpl): is it enough that we know it's a valid blobref to avoid any injection risk ?
query := blobSizeQuery + fmt.Sprintf("'%v';\n", br.String())
c.mu.Lock()
defer c.mu.Unlock()
err := c.startSQLiteChild()
if err != nil {
log.Fatalf("Could not start sqlite child process: %v", err)
}
_, err = c.w.Write([]byte(query))
if err != nil {
log.Fatalf("failed to query have cache: %v", err)
}
out, err := c.r.ReadString('\n')
if err != nil {
log.Fatalf("failed to read have cache query result: %v", err)
}
out = strings.TrimRight(out, "\n")
if out == noResult {
return
}
size, err = strconv.ParseInt(out, 10, 64)
if err != nil {
log.Fatalf("Bogus blob size in %v table: %v", haveTableName, err)
}
return size, true
}
// populateMutation populates keys & values into the provided BatchMutation.
//
// the blobref can be trusted at this point (it's been fully consumed
// and verified to match), and the sniffer has been populated.
func (ix *Index) populateMutation(br blob.Ref, sniffer *BlobSniffer, bm BatchMutation) error {
bm.Set("have:"+br.String(), fmt.Sprintf("%d", sniffer.Size()))
bm.Set("meta:"+br.String(), fmt.Sprintf("%d|%s", sniffer.Size(), sniffer.MIMEType()))
if blob, ok := sniffer.SchemaBlob(); ok {
switch blob.Type() {
case "claim":
if err := ix.populateClaim(blob, bm); err != nil {
return err
}
case "permanode":
//if err := mi.populatePermanode(blobRef, camli, bm); err != nil {
//return err
//}
case "file":
if err := ix.populateFile(blob, bm); err != nil {
return err
}
case "directory":
if err := ix.populateDir(blob, bm); err != nil {
return err
}
}
}
return nil
}
func (tf *Fetcher) Fetch(ref blob.Ref) (file types.ReadSeekCloser, size int64, err error) {
if tf.FetchErr != nil {
if err = tf.FetchErr(); err != nil {
return
}
}
tf.l.Lock()
defer tf.l.Unlock()
if tf.m == nil {
err = os.ErrNotExist
return
}
tb, ok := tf.m[ref.String()]
if !ok {
err = os.ErrNotExist
return
}
size = int64(len(tb.Contents))
return struct {
*io.SectionReader
io.Closer
}{
io.NewSectionReader(strings.NewReader(tb.Contents), 0, size),
dummyCloser,
}, size, nil
}
func (m *thumbMeta) Put(key string, br blob.Ref) error {
m.mem.Add(key, br)
if m.kv != nil {
return m.kv.Set(key, br.String())
}
return nil
}
func (ph *publishHandler) lookupPathTarget(root blob.Ref, suffix string) (blob.Ref, error) {
if suffix == "" {
return root, nil
}
// TODO: verify it's optimized: http://camlistore.org/issue/405
result, err := ph.cl.Query(&search.SearchQuery{
Limit: 1,
Constraint: &search.Constraint{
Permanode: &search.PermanodeConstraint{
SkipHidden: true,
Relation: &search.RelationConstraint{
Relation: "parent",
EdgeType: "camliPath:" + suffix,
Any: &search.Constraint{
BlobRefPrefix: root.String(),
},
},
},
},
})
if err != nil {
return blob.Ref{}, err
}
if len(result.Blobs) == 0 || !result.Blobs[0].Blob.Valid() {
return blob.Ref{}, os.ErrNotExist
}
return result.Blobs[0].Blob, nil
}
func (fr *FileReader) getSuperset(br blob.Ref) (*superset, error) {
if root := fr.rootReader(); root != fr {
return root.getSuperset(br)
}
brStr := br.String()
ssi, err := fr.sfg.Do(brStr, func() (interface{}, error) {
fr.ssmmu.Lock()
ss, ok := fr.ssm[br]
fr.ssmmu.Unlock()
if ok {
return ss, nil
}
rc, _, err := fr.fetcher.Fetch(br)
if err != nil {
return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
}
defer rc.Close()
ss, err = parseSuperset(rc)
if err != nil {
return nil, err
}
ss.BlobRef = br
fr.ssmmu.Lock()
defer fr.ssmmu.Unlock()
fr.ssm[br] = ss
return ss, nil
})
if err != nil {
return nil, err
}
return ssi.(*superset), nil
}
func (ph *publishHandler) describeMembers(br blob.Ref) (*search.SearchResult, error) {
res, err := ph.cl.Query(&search.SearchQuery{
Constraint: &search.Constraint{
Permanode: &search.PermanodeConstraint{
Relation: &search.RelationConstraint{
Relation: "parent",
Any: &search.Constraint{
BlobRefPrefix: br.String(),
},
},
},
CamliType: "permanode",
},
Describe: &search.DescribeRequest{
Depth: 1,
Rules: []*search.DescribeRule{
{
Attrs: []string{"camliContent", "camliContentImage"},
},
},
},
Limit: -1,
})
if err != nil {
return nil, fmt.Errorf("Could not describe members of %v: %v", br, err)
}
return res, nil
}
// SearchExistingFileSchema does a search query looking for an
// existing file with entire contents of wholeRef, then does a HEAD
// request to verify the file still exists on the server. If so,
// it returns that file schema's blobref.
//
// May return (zero, nil) on ENOENT. A non-nil error is only returned
// if there were problems searching.
func (c *Client) SearchExistingFileSchema(wholeRef blob.Ref) (blob.Ref, error) {
sr, err := c.SearchRoot()
if err != nil {
return blob.Ref{}, err
}
url := sr + "camli/search/files?wholedigest=" + wholeRef.String()
req := c.newRequest("GET", url)
res, err := c.doReqGated(req)
if err != nil {
return blob.Ref{}, err
}
if res.StatusCode != 200 {
body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
res.Body.Close()
return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
}
var ress struct {
Files []blob.Ref `json:"files"`
}
if err := httputil.DecodeJSON(res, &ress); err != nil {
return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
}
if len(ress.Files) == 0 {
return blob.Ref{}, nil
}
for _, f := range ress.Files {
if c.FileHasContents(f, wholeRef) {
return f, nil
}
}
return blob.Ref{}, nil
}
func (ds *DiskStorage) blobDirectory(b blob.Ref) string {
d := b.Digest()
if len(d) < 4 {
d = d + "____"
}
return filepath.Join(ds.root, b.HashName(), d[0:2], d[2:4])
}
func (x *Index) GetBlobMeta(br blob.Ref) (camtypes.BlobMeta, error) {
if x.corpus != nil {
return x.corpus.GetBlobMeta(br)
}
key := "meta:" + br.String()
meta, err := x.s.Get(key)
if err == sorted.ErrNotFound {
err = os.ErrNotExist
}
if err != nil {
return camtypes.BlobMeta{}, err
}
pos := strings.Index(meta, "|")
if pos < 0 {
panic(fmt.Sprintf("Bogus index row for key %q: got value %q", key, meta))
}
size, err := strconv.ParseUint(meta[:pos], 10, 32)
if err != nil {
return camtypes.BlobMeta{}, err
}
mime := meta[pos+1:]
return camtypes.BlobMeta{
Ref: br,
Size: uint32(size),
CamliType: camliTypeFromMIME(mime),
}, nil
}
// Given a blobref and a few hex characters of the digest of the next hop, return the complete
// blobref of the prefix, if that's a valid next hop.
func (sh *Handler) ResolvePrefixHop(parent blob.Ref, prefix string) (child blob.Ref, err error) {
// TODO: this is a linear scan right now. this should be
// optimized to use a new database table of members so this is
// a quick lookup. in the meantime it should be in memcached
// at least.
if len(prefix) < 8 {
return blob.Ref{}, fmt.Errorf("Member prefix %q too small", prefix)
}
dr := sh.NewDescribeRequest()
dr.Describe(parent, 1)
res, err := dr.Result()
if err != nil {
return
}
des, ok := res[parent.String()]
if !ok {
return blob.Ref{}, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent)
}
if des.Permanode != nil {
if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) {
return cr, nil
}
for _, member := range des.Members() {
if strings.HasPrefix(member.BlobRef.Digest(), prefix) {
return member.BlobRef, nil
}
}
}
return blob.Ref{}, fmt.Errorf("Member prefix %q not found in %q", prefix, parent)
}
func (fi *FakeIndex) AddClaim(owner, permanode blob.Ref, claimType, attr, value string) {
fi.lk.Lock()
defer fi.lk.Unlock()
date := fi.nextDate()
claim := &search.Claim{
Permanode: permanode,
Signer: blob.Ref{},
BlobRef: blob.Ref{},
Date: date,
Type: claimType,
Attr: attr,
Value: value,
}
key := permanode.String() + "/" + owner.String()
fi.ownerClaims[key] = append(fi.ownerClaims[key], claim)
if claimType == "set-attribute" && strings.HasPrefix(attr, "camliPath:") {
suffix := attr[len("camliPath:"):]
path := &search.Path{
Target: blob.MustParse(value),
Suffix: suffix,
}
fi.path[fmt.Sprintf("%s\x00%s\x00%s", owner, permanode, suffix)] = path
}
}
func (n *node) addEdge(dst blob.Ref) {
if !dst.Valid() {
return
}
n.g.startLoadNode(dst)
n.edges = append(n.edges, dst)
}
func (sto *driveStorage) ReceiveBlob(b blob.Ref, source io.Reader) (blob.SizedRef, error) {
file, err := sto.service.Upsert(b.String(), source)
if err != nil {
return blob.SizedRef{Ref: b, Size: 0}, err
}
return blob.SizedRef{Ref: b, Size: file.FileSize}, err
}
// bytesHaveSchemaLink reports whether bb is a valid Camlistore schema
// blob and has target somewhere in a schema field used to represent a
// Merkle-tree-ish file or directory.
func bytesHaveSchemaLink(br blob.Ref, bb []byte, target blob.Ref) bool {
// Fast path for no:
if !bytes.Contains(bb, []byte(target.String())) {
return false
}
b, err := schema.BlobFromReader(br, bytes.NewReader(bb))
if err != nil {
return false
}
typ := b.Type()
switch typ {
case "file", "blob":
for _, bp := range b.ByteParts() {
if bp.BlobRef.Valid() {
return bp.BlobRef == target
}
if bp.BytesRef.Valid() {
return bp.BytesRef == target
}
}
case "directory":
if d, ok := b.DirectoryEntries(); ok {
return d == target
}
case "static-set":
for _, m := range b.StaticSetMembers() {
if m == target {
return true
}
}
}
return false
}