func (mi *Indexer) GetOwnerClaims(permanode, owner *blobref.BlobRef) (claims search.ClaimList, err os.Error) { claims = make(search.ClaimList, 0) // TODO: ignore rows where unverified = 'N' rs, err := mi.db.Query("SELECT blobref, date, claim, attr, value FROM claims WHERE permanode = ? AND signer = ?", permanode.String(), owner.String()) if err != nil { return } defer rs.Close() var row claimsRow for rs.Next() { err = rs.Scan(&row.blobref, &row.date, &row.claim, &row.attr, &row.value) if err != nil { return } t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(row.date)) if err != nil { log.Printf("Skipping; error parsing time %q: %v", row.date, err) continue } claims = append(claims, &search.Claim{ BlobRef: blobref.Parse(row.blobref), Signer: owner, Permanode: permanode, Type: row.claim, Date: t, Attr: row.attr, Value: row.value, }) } return }
func (c *FlatHaveCache) NoteBlobExists(br *blobref.BlobRef) { c.mu.Lock() defer c.mu.Unlock() k := br.String() c.m[k] = true c.dirty[k] = true }
func (fi *FakeIndex) AddClaim(owner, permanode *blobref.BlobRef, claimType, attr, value string) { fi.lk.Lock() defer fi.lk.Unlock() date := fi.nextDate() claim := &search.Claim{ Permanode: permanode, Signer: nil, BlobRef: nil, Date: date, Type: claimType, Attr: attr, Value: value, } key := permanode.String() + "/" + owner.String() fi.ownerClaims[key] = append(fi.ownerClaims[key], claim) if claimType == "set-attribute" && strings.HasPrefix(attr, "camliPath:") { suffix := attr[len("camliPath:"):] path := &search.Path{ Target: blobref.MustParse(value), Suffix: suffix, } fi.path[fmt.Sprintf("%s\x00%s\x00%s", owner, permanode, suffix)] = path } }
// Given a blobref and a few hex characters of the digest of the next hop, return the complete // blobref of the prefix, if that's a valid next hop. func (sh *Handler) ResolvePrefixHop(parent *blobref.BlobRef, prefix string) (child *blobref.BlobRef, err os.Error) { // TODO: this is a linear scan right now. this should be // optimized to use a new database table of members so this is // a quick lookup. in the meantime it should be in memcached // at least. if len(prefix) < 8 { return nil, fmt.Errorf("Member prefix %q too small", prefix) } dr := sh.NewDescribeRequest() dr.Describe(parent, 1) res, err := dr.Result() if err != nil { return } des, ok := res[parent.String()] if !ok { return nil, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent) } if des.Permanode != nil { if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) { return cr, nil } for _, member := range des.Members() { if strings.HasPrefix(member.BlobRef.Digest(), prefix) { return member.BlobRef, nil } } } return nil, fmt.Errorf("Member prefix %q not found in %q", prefix, parent) }
func (mi *Indexer) PathsLookup(signer, base *blobref.BlobRef, suffix string) (paths []*search.Path, err os.Error) { keyId, err := mi.keyIdOfSigner(signer) if err != nil { return } rs, err := mi.db.Query("SELECT claimref, claimdate, targetref FROM path "+ "WHERE keyid=? AND baseref=? AND suffix=?", keyId, base.String(), suffix) if err != nil { return } defer rs.Close() var claimref, claimdate, targetref string for rs.Next() { if err = rs.Scan(&claimref, &claimdate, &targetref); err != nil { return } t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(claimdate)) if err != nil { log.Printf("Skipping bogus path row with bad time: %q", claimref) continue } _ = t // TODO: use this? paths = append(paths, &search.Path{ Claim: blobref.Parse(claimref), ClaimDate: claimdate, Base: base, Target: blobref.Parse(targetref), Suffix: suffix, }) } return }
func NewShareRef(authType string, target *blobref.BlobRef, transitive bool) map[string]interface{} { m := newCamliMap(1, "share") m["authType"] = authType m["target"] = target.String() m["transitive"] = transitive return m }
func (mi *Indexer) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) { client, err := mi.getConnection() if err != nil { return } defer func() { if err == nil || err == os.ENOENT { mi.releaseConnection(client) } else { client.Close() } }() err = client.Query(fmt.Sprintf("SELECT type, size FROM blobs WHERE blobref=%q", blob.String())) if err != nil { return } result, err := client.StoreResult() if err != nil { return } defer client.FreeResult() row := result.FetchRow() if row == nil { err = os.ENOENT return } //log.Printf("got row: %#v (2 is %T)", row, row[1]) mime, _ = row[0].(string) size, _ = row[1].(int64) return }
func (mi *Indexer) ExistingFileSchemas(bytesRef *blobref.BlobRef) (files []*blobref.BlobRef, err os.Error) { client, err := mi.getConnection() if err != nil { return } defer func() { if err == nil { mi.releaseConnection(client) } else { client.Close() } }() err = client.Query(fmt.Sprintf("SELECT fileschemaref FROM files WHERE bytesref=%q", bytesRef.String())) if err != nil { return } result, err := client.StoreResult() if err != nil { return } defer client.FreeResult() for { row := result.FetchRow() if row == nil { break } files = append(files, blobref.Parse(row[0].(string))) } return }
func (mi *Indexer) populatePermanode(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) { err = execSQL(client, "INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+ "VALUES (?, 'Y', ?, '')", blobRef.String(), camli.Signer) return }
func (h *SimpleBlobHub) NotifyBlobReceived(blob *blobref.BlobRef) { h.l.Lock() defer h.l.Unlock() // Callback channels to notify, nil until non-empty var notify []chan *blobref.BlobRef // Append global listeners for ch, _ := range h.listeners { notify = append(notify, ch) } // Append blob-specific listeners if h.blobListeners != nil { blobstr := blob.String() if set, ok := h.blobListeners[blobstr]; ok { for ch, _ := range set { notify = append(notify, ch) } } } // Run in a separate Goroutine so NotifyBlobReceived doesn't block // callers if callbacks are slow. go func() { for _, ch := range notify { ch <- blob } }() }
func (fs *CamliFileSystem) fetchSchemaSuperset(br *blobref.BlobRef) (*schema.Superset, os.Error) { blobStr := br.String() if ss, ok := fs.blobToSchema.Get(blobStr); ok { return ss.(*schema.Superset), nil } log.Printf("schema cache MISS on %q", blobStr) rsc, _, err := fs.fetcher.Fetch(br) if err != nil { return nil, err } defer rsc.Close() jd := json.NewDecoder(rsc) ss := new(schema.Superset) err = jd.Decode(ss) if err != nil { log.Printf("Error parsing %s as schema blob: %v", br, err) return nil, os.EINVAL } if ss.Type == "" { log.Printf("blob %s is JSON but lacks camliType", br) return nil, os.EINVAL } ss.BlobRef = br fs.blobToSchema.Add(blobStr, ss) return ss, nil }
func NewClaim(permaNode *blobref.BlobRef, claimType string) map[string]interface{} { m := newCamliMap(1, "claim") m["permaNode"] = permaNode.String() m["claimType"] = claimType m["claimDate"] = RFC3339FromNanos(time.Nanoseconds()) return m }
func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) { sniffer := new(blobSniffer) hash := blobRef.Hash() var written int64 written, err = io.Copy(io.MultiWriter(hash, sniffer), source) log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err) if err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } sniffer.Parse() mimeType := sniffer.MimeType() log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated()) var client *mysql.Client if client, err = mi.getConnection(); err != nil { return } defer mi.releaseConnection(client) var stmt *mysql.Statement if stmt, err = client.Prepare("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)"); err != nil { log.Printf("mysqlindexer: prepare error: %v", err) return } if err = stmt.BindParams(blobRef.String(), written, mimeType); err != nil { log.Printf("mysqlindexer: bind error: %v", err) return } if err = stmt.Execute(); err != nil { log.Printf("mysqlindexer: execute error: %v", err) return } if camli := sniffer.camli; camli != nil { switch camli.Type { case "claim": if err = mi.populateClaim(client, blobRef, camli, sniffer); err != nil { return } case "permanode": if err = mi.populatePermanode(client, blobRef, camli); err != nil { return } case "file": if err = mi.populateFile(client, blobRef, camli); err != nil { return } } } retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written} return }
func (mi *Indexer) populatePermanode(blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) { err = mi.db.Execute( "INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+ "VALUES (?, 'Y', ?, '') "+ "ON DUPLICATE KEY UPDATE unverified = 'Y', signer = ?", blobRef.String(), camli.Signer, camli.Signer) return }
func (dr *DescribeRequest) DescribeSync(br *blobref.BlobRef) (*DescribedBlob, os.Error) { dr.Describe(br, 1) res, err := dr.Result() if err != nil { return nil, err } return res[br.String()], nil }
func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset, sniffer *blobSniffer) (err os.Error) { pnBlobref := blobref.Parse(camli.Permanode) if pnBlobref == nil { // Skip bogus claim with malformed permanode. return } verifiedKeyId := "" if rawJson, err := sniffer.Body(); err == nil { vr := jsonsign.NewVerificationRequest(rawJson, mi.KeyFetcher) if vr.Verify() { verifiedKeyId = vr.SignerKeyId log.Printf("mysqlindex: verified claim %s from %s", blobRef, verifiedKeyId) if err = execSQL(client, "INSERT IGNORE INTO signerkeyid (blobref, keyid) "+ "VALUES (?, ?)", vr.CamliSigner.String(), verifiedKeyId); err != nil { return } } else { log.Printf("mysqlindex: verification failure on claim %s: %v", blobRef, vr.Err) } } if err = execSQL(client, "INSERT IGNORE INTO claims (blobref, signer, verifiedkeyid, date, unverified, claim, permanode, attr, value) "+ "VALUES (?, ?, ?, ?, 'Y', ?, ?, ?, ?)", blobRef.String(), camli.Signer, verifiedKeyId, camli.ClaimDate, camli.ClaimType, camli.Permanode, camli.Attribute, camli.Value); err != nil { return } if verifiedKeyId != "" { // TODO: limit this to only certain attributes (for now, just "camliRoot") once search handler // is working and the UI permits setting camliRoot. if err = execSQL(client, "INSERT IGNORE INTO signerattrvalue (keyid, attr, value, claimdate, blobref, permanode) "+ "VALUES (?, ?, ?, ?, ?, ?)", verifiedKeyId, camli.Attribute, camli.Value, camli.ClaimDate, blobRef.String(), camli.Permanode); err != nil { return } } // And update the lastmod on the permanode row. if err = execSQL(client, "INSERT IGNORE INTO permanodes (blobref) VALUES (?)", pnBlobref.String()); err != nil { return } if err = execSQL(client, "UPDATE permanodes SET lastmod=? WHERE blobref=? AND ? > lastmod", camli.ClaimDate, pnBlobref.String(), camli.ClaimDate); err != nil { return } return nil }
func (dr *DescribeRequest) addError(br *blobref.BlobRef, err os.Error) { if err == nil { return } dr.lk.Lock() defer dr.lk.Unlock() // TODO: append? meh. dr.errs[br.String()] = err }
func (fi *FakeIndex) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) { fi.lk.Lock() defer fi.lk.Unlock() bs := blob.String() mime, ok := fi.mimeType[bs] if !ok { return "", 0, os.ENOENT } return mime, fi.size[bs], nil }
func (sr *statsStatReceiver) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err os.Error) { n, err := io.Copy(ioutil.Discard, source) if err != nil { return } sr.lock() defer sr.mu.Unlock() sr.have[blob.String()] = n return blobref.SizedBlobRef{blob, n}, nil }
func (mi *Indexer) GetOwnerClaims(permanode, owner *blobref.BlobRef) (claims search.ClaimList, reterr os.Error) { claims = make(search.ClaimList, 0) client, err := mi.getConnection() if err != nil { reterr = err return } defer mi.releaseConnection(client) // TODO: ignore rows where unverified = 'N' stmt, err := client.Prepare("SELECT blobref, date, claim, attr, value FROM claims WHERE permanode = ? AND signer = ?") if err != nil { reterr = err return } err = stmt.BindParams(permanode.String(), owner.String()) if err != nil { reterr = err return } err = stmt.Execute() if err != nil { reterr = err return } var row claimsRow stmt.BindResult(&row.blobref, &row.date, &row.claim, &row.attr, &row.value) defer stmt.Close() for { done, err := stmt.Fetch() if err != nil { reterr = err return } if done { break } t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(row.date)) if err != nil { log.Printf("Skipping; error parsing time %q: %v", row.date, err) continue } claims = append(claims, &search.Claim{ BlobRef: blobref.Parse(row.blobref), Signer: owner, Permanode: permanode, Type: row.claim, Date: t, Attr: row.attr, Value: row.value, }) } return }
func (dr *DescribeRequest) describedBlob(b *blobref.BlobRef) *DescribedBlob { dr.lk.Lock() defer dr.lk.Unlock() bs := b.String() if des, ok := dr.m[bs]; ok { return des } des := &DescribedBlob{Request: dr, BlobRef: b} dr.m[bs] = des return des }
func (b *DescribedBlob) PeerBlob(br *blobref.BlobRef) *DescribedBlob { if b.Request == nil { return &DescribedBlob{BlobRef: br, Stub: true} } b.Request.lk.Lock() defer b.Request.lk.Unlock() if peer, ok := b.Request.m[br.String()]; ok { return peer } return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true} }
func (mi *Indexer) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) { rs, err := mi.db.Query("SELECT type, size FROM blobs WHERE blobref=?", blob.String()) if err != nil { return } defer rs.Close() if !rs.Next() { err = os.ENOENT return } err = rs.Scan(&mime, &size) return }
func (h *SimpleBlobHub) RegisterBlobListener(blob *blobref.BlobRef, ch chan *blobref.BlobRef) { h.l.Lock() defer h.l.Unlock() if h.blobListeners == nil { h.blobListeners = make(map[string]map[chan *blobref.BlobRef]bool) } bstr := blob.String() _, ok := h.blobListeners[bstr] if !ok { h.blobListeners[bstr] = make(map[chan *blobref.BlobRef]bool) } h.blobListeners[bstr][ch] = true }
func (mi *Indexer) keyIdOfSigner(signer *blobref.BlobRef) (keyid string, err os.Error) { rs, err := mi.db.Query("SELECT keyid FROM signerkeyid WHERE blobref=?", signer.String()) if err != nil { return } defer rs.Close() if !rs.Next() { return "", fmt.Errorf("mysqlindexer: failed to find keyid of signer %q", signer.String()) } err = rs.Scan(&keyid) return }
func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err os.Error) { if sto.ctx == nil { err = errNoContext return } var b bytes.Buffer hash := br.Hash() written, err := io.Copy(io.MultiWriter(hash, &b), in) if err != nil { return } if !br.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } mimeType := "application/octet-stream" bw, err := blobstore.Create(sto.ctx, mimeType) if err != nil { return } written, err = io.Copy(bw, &b) if err != nil { // TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it. return } err = bw.Close() if err != nil { // TODO(bradfitz): try to clean up; see if we can find the key, delete it. return } bkey, err := bw.Key() if err != nil { return } var ent blobEnt ent.BlobRefStr = br.String() ent.Size = written ent.BlobKey = bkey dkey := datastore.NewKey(sto.ctx, blobKind, br.String(), 0, nil) _, err = datastore.Put(sto.ctx, dkey, &ent) if err != nil { blobstore.Delete(sto.ctx, bkey) // TODO: insert into task queue on error to try later? return } return blobref.SizedBlobRef{br, written}, nil }
func (mi *Indexer) GetFileInfo(fileRef *blobref.BlobRef) (*search.FileInfo, os.Error) { rs, err := mi.db.Query("SELECT size, filename, mime FROM bytesfiles WHERE schemaref=?", fileRef.String()) if err != nil { return nil, err } defer rs.Close() if !rs.Next() { return nil, os.ENOENT } var fi search.FileInfo err = rs.Scan(&fi.Size, &fi.FileName, &fi.MimeType) return &fi, err }
func (tf *Fetcher) Fetch(ref *blobref.BlobRef) (file blobref.ReadSeekCloser, size int64, err os.Error) { tf.l.Lock() defer tf.l.Unlock() if tf.m == nil { err = os.ENOENT return } tb, ok := tf.m[ref.String()] if !ok { err = os.ENOENT return } file = &strReader{tb.Contents, 0} size = int64(len(tb.Contents)) return }
func (mi *Indexer) ExistingFileSchemas(wholeDigest *blobref.BlobRef) (files []*blobref.BlobRef, err os.Error) { rs, err := mi.db.Query("SELECT schemaref FROM bytesfiles WHERE wholedigest=?", wholeDigest.String()) if err != nil { return } defer rs.Close() ref := "" for rs.Next() { if err := rs.Scan(&ref); err != nil { return nil, err } files = append(files, blobref.Parse(ref)) } return }
func (h *SimpleBlobHub) UnregisterBlobListener(blob *blobref.BlobRef, ch chan *blobref.BlobRef) { h.l.Lock() defer h.l.Unlock() if h.blobListeners == nil { panic("blobhub: UnregisterBlobListener called without RegisterBlobListener") } bstr := blob.String() set, ok := h.blobListeners[bstr] if !ok { panic("blobhub: UnregisterBlobListener called without RegisterBlobListener for " + bstr) } set[ch] = false, false if len(set) == 0 { h.blobListeners[bstr] = nil, false } }