func (mi *Indexer) GetOwnerClaims(permanode, owner *blobref.BlobRef) (claims search.ClaimList, err os.Error) {
claims = make(search.ClaimList, 0)
// TODO: ignore rows where unverified = 'N'
rs, err := mi.db.Query("SELECT blobref, date, claim, attr, value FROM claims WHERE permanode = ? AND signer = ?",
permanode.String(), owner.String())
if err != nil {
return
}
defer rs.Close()
var row claimsRow
for rs.Next() {
err = rs.Scan(&row.blobref, &row.date, &row.claim, &row.attr, &row.value)
if err != nil {
return
}
t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(row.date))
if err != nil {
log.Printf("Skipping; error parsing time %q: %v", row.date, err)
continue
}
claims = append(claims, &search.Claim{
BlobRef: blobref.Parse(row.blobref),
Signer: owner,
Permanode: permanode,
Type: row.claim,
Date: t,
Attr: row.attr,
Value: row.value,
})
}
return
}
func (c *FlatHaveCache) NoteBlobExists(br *blobref.BlobRef) {
c.mu.Lock()
defer c.mu.Unlock()
k := br.String()
c.m[k] = true
c.dirty[k] = true
}
func (fi *FakeIndex) AddClaim(owner, permanode *blobref.BlobRef, claimType, attr, value string) {
fi.lk.Lock()
defer fi.lk.Unlock()
date := fi.nextDate()
claim := &search.Claim{
Permanode: permanode,
Signer: nil,
BlobRef: nil,
Date: date,
Type: claimType,
Attr: attr,
Value: value,
}
key := permanode.String() + "/" + owner.String()
fi.ownerClaims[key] = append(fi.ownerClaims[key], claim)
if claimType == "set-attribute" && strings.HasPrefix(attr, "camliPath:") {
suffix := attr[len("camliPath:"):]
path := &search.Path{
Target: blobref.MustParse(value),
Suffix: suffix,
}
fi.path[fmt.Sprintf("%s\x00%s\x00%s", owner, permanode, suffix)] = path
}
}
// Given a blobref and a few hex characters of the digest of the next hop, return the complete
// blobref of the prefix, if that's a valid next hop.
func (sh *Handler) ResolvePrefixHop(parent *blobref.BlobRef, prefix string) (child *blobref.BlobRef, err os.Error) {
// TODO: this is a linear scan right now. this should be
// optimized to use a new database table of members so this is
// a quick lookup. in the meantime it should be in memcached
// at least.
if len(prefix) < 8 {
return nil, fmt.Errorf("Member prefix %q too small", prefix)
}
dr := sh.NewDescribeRequest()
dr.Describe(parent, 1)
res, err := dr.Result()
if err != nil {
return
}
des, ok := res[parent.String()]
if !ok {
return nil, fmt.Errorf("Failed to describe member %q in parent %q", prefix, parent)
}
if des.Permanode != nil {
if cr, ok := des.ContentRef(); ok && strings.HasPrefix(cr.Digest(), prefix) {
return cr, nil
}
for _, member := range des.Members() {
if strings.HasPrefix(member.BlobRef.Digest(), prefix) {
return member.BlobRef, nil
}
}
}
return nil, fmt.Errorf("Member prefix %q not found in %q", prefix, parent)
}
func (mi *Indexer) PathsLookup(signer, base *blobref.BlobRef, suffix string) (paths []*search.Path, err os.Error) {
keyId, err := mi.keyIdOfSigner(signer)
if err != nil {
return
}
rs, err := mi.db.Query("SELECT claimref, claimdate, targetref FROM path "+
"WHERE keyid=? AND baseref=? AND suffix=?",
keyId, base.String(), suffix)
if err != nil {
return
}
defer rs.Close()
var claimref, claimdate, targetref string
for rs.Next() {
if err = rs.Scan(&claimref, &claimdate, &targetref); err != nil {
return
}
t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(claimdate))
if err != nil {
log.Printf("Skipping bogus path row with bad time: %q", claimref)
continue
}
_ = t // TODO: use this?
paths = append(paths, &search.Path{
Claim: blobref.Parse(claimref),
ClaimDate: claimdate,
Base: base,
Target: blobref.Parse(targetref),
Suffix: suffix,
})
}
return
}
func NewShareRef(authType string, target *blobref.BlobRef, transitive bool) map[string]interface{} {
m := newCamliMap(1, "share")
m["authType"] = authType
m["target"] = target.String()
m["transitive"] = transitive
return m
}
func (mi *Indexer) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) {
client, err := mi.getConnection()
if err != nil {
return
}
defer func() {
if err == nil || err == os.ENOENT {
mi.releaseConnection(client)
} else {
client.Close()
}
}()
err = client.Query(fmt.Sprintf("SELECT type, size FROM blobs WHERE blobref=%q", blob.String()))
if err != nil {
return
}
result, err := client.StoreResult()
if err != nil {
return
}
defer client.FreeResult()
row := result.FetchRow()
if row == nil {
err = os.ENOENT
return
}
//log.Printf("got row: %#v (2 is %T)", row, row[1])
mime, _ = row[0].(string)
size, _ = row[1].(int64)
return
}
func (mi *Indexer) ExistingFileSchemas(bytesRef *blobref.BlobRef) (files []*blobref.BlobRef, err os.Error) {
client, err := mi.getConnection()
if err != nil {
return
}
defer func() {
if err == nil {
mi.releaseConnection(client)
} else {
client.Close()
}
}()
err = client.Query(fmt.Sprintf("SELECT fileschemaref FROM files WHERE bytesref=%q", bytesRef.String()))
if err != nil {
return
}
result, err := client.StoreResult()
if err != nil {
return
}
defer client.FreeResult()
for {
row := result.FetchRow()
if row == nil {
break
}
files = append(files, blobref.Parse(row[0].(string)))
}
return
}
func (mi *Indexer) populatePermanode(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
err = execSQL(client,
"INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+
"VALUES (?, 'Y', ?, '')",
blobRef.String(), camli.Signer)
return
}
func (h *SimpleBlobHub) NotifyBlobReceived(blob *blobref.BlobRef) {
h.l.Lock()
defer h.l.Unlock()
// Callback channels to notify, nil until non-empty
var notify []chan *blobref.BlobRef
// Append global listeners
for ch, _ := range h.listeners {
notify = append(notify, ch)
}
// Append blob-specific listeners
if h.blobListeners != nil {
blobstr := blob.String()
if set, ok := h.blobListeners[blobstr]; ok {
for ch, _ := range set {
notify = append(notify, ch)
}
}
}
// Run in a separate Goroutine so NotifyBlobReceived doesn't block
// callers if callbacks are slow.
go func() {
for _, ch := range notify {
ch <- blob
}
}()
}
func (fs *CamliFileSystem) fetchSchemaSuperset(br *blobref.BlobRef) (*schema.Superset, os.Error) {
blobStr := br.String()
if ss, ok := fs.blobToSchema.Get(blobStr); ok {
return ss.(*schema.Superset), nil
}
log.Printf("schema cache MISS on %q", blobStr)
rsc, _, err := fs.fetcher.Fetch(br)
if err != nil {
return nil, err
}
defer rsc.Close()
jd := json.NewDecoder(rsc)
ss := new(schema.Superset)
err = jd.Decode(ss)
if err != nil {
log.Printf("Error parsing %s as schema blob: %v", br, err)
return nil, os.EINVAL
}
if ss.Type == "" {
log.Printf("blob %s is JSON but lacks camliType", br)
return nil, os.EINVAL
}
ss.BlobRef = br
fs.blobToSchema.Add(blobStr, ss)
return ss, nil
}
func NewClaim(permaNode *blobref.BlobRef, claimType string) map[string]interface{} {
m := newCamliMap(1, "claim")
m["permaNode"] = permaNode.String()
m["claimType"] = claimType
m["claimDate"] = RFC3339FromNanos(time.Nanoseconds())
return m
}
func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) {
sniffer := new(blobSniffer)
hash := blobRef.Hash()
var written int64
written, err = io.Copy(io.MultiWriter(hash, sniffer), source)
log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err)
if err != nil {
return
}
if !blobRef.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
sniffer.Parse()
mimeType := sniffer.MimeType()
log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated())
var client *mysql.Client
if client, err = mi.getConnection(); err != nil {
return
}
defer mi.releaseConnection(client)
var stmt *mysql.Statement
if stmt, err = client.Prepare("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)"); err != nil {
log.Printf("mysqlindexer: prepare error: %v", err)
return
}
if err = stmt.BindParams(blobRef.String(), written, mimeType); err != nil {
log.Printf("mysqlindexer: bind error: %v", err)
return
}
if err = stmt.Execute(); err != nil {
log.Printf("mysqlindexer: execute error: %v", err)
return
}
if camli := sniffer.camli; camli != nil {
switch camli.Type {
case "claim":
if err = mi.populateClaim(client, blobRef, camli, sniffer); err != nil {
return
}
case "permanode":
if err = mi.populatePermanode(client, blobRef, camli); err != nil {
return
}
case "file":
if err = mi.populateFile(client, blobRef, camli); err != nil {
return
}
}
}
retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written}
return
}
func (mi *Indexer) populatePermanode(blobRef *blobref.BlobRef, camli *schema.Superset) (err os.Error) {
err = mi.db.Execute(
"INSERT IGNORE INTO permanodes (blobref, unverified, signer, lastmod) "+
"VALUES (?, 'Y', ?, '') "+
"ON DUPLICATE KEY UPDATE unverified = 'Y', signer = ?",
blobRef.String(), camli.Signer, camli.Signer)
return
}
func (dr *DescribeRequest) DescribeSync(br *blobref.BlobRef) (*DescribedBlob, os.Error) {
dr.Describe(br, 1)
res, err := dr.Result()
if err != nil {
return nil, err
}
return res[br.String()], nil
}
func (mi *Indexer) populateClaim(client *mysql.Client, blobRef *blobref.BlobRef, camli *schema.Superset, sniffer *blobSniffer) (err os.Error) {
pnBlobref := blobref.Parse(camli.Permanode)
if pnBlobref == nil {
// Skip bogus claim with malformed permanode.
return
}
verifiedKeyId := ""
if rawJson, err := sniffer.Body(); err == nil {
vr := jsonsign.NewVerificationRequest(rawJson, mi.KeyFetcher)
if vr.Verify() {
verifiedKeyId = vr.SignerKeyId
log.Printf("mysqlindex: verified claim %s from %s", blobRef, verifiedKeyId)
if err = execSQL(client, "INSERT IGNORE INTO signerkeyid (blobref, keyid) "+
"VALUES (?, ?)", vr.CamliSigner.String(), verifiedKeyId); err != nil {
return
}
} else {
log.Printf("mysqlindex: verification failure on claim %s: %v", blobRef, vr.Err)
}
}
if err = execSQL(client,
"INSERT IGNORE INTO claims (blobref, signer, verifiedkeyid, date, unverified, claim, permanode, attr, value) "+
"VALUES (?, ?, ?, ?, 'Y', ?, ?, ?, ?)",
blobRef.String(), camli.Signer, verifiedKeyId, camli.ClaimDate,
camli.ClaimType, camli.Permanode,
camli.Attribute, camli.Value); err != nil {
return
}
if verifiedKeyId != "" {
// TODO: limit this to only certain attributes (for now, just "camliRoot") once search handler
// is working and the UI permits setting camliRoot.
if err = execSQL(client, "INSERT IGNORE INTO signerattrvalue (keyid, attr, value, claimdate, blobref, permanode) "+
"VALUES (?, ?, ?, ?, ?, ?)",
verifiedKeyId, camli.Attribute, camli.Value,
camli.ClaimDate, blobRef.String(), camli.Permanode); err != nil {
return
}
}
// And update the lastmod on the permanode row.
if err = execSQL(client,
"INSERT IGNORE INTO permanodes (blobref) VALUES (?)",
pnBlobref.String()); err != nil {
return
}
if err = execSQL(client,
"UPDATE permanodes SET lastmod=? WHERE blobref=? AND ? > lastmod",
camli.ClaimDate, pnBlobref.String(), camli.ClaimDate); err != nil {
return
}
return nil
}
func (dr *DescribeRequest) addError(br *blobref.BlobRef, err os.Error) {
if err == nil {
return
}
dr.lk.Lock()
defer dr.lk.Unlock()
// TODO: append? meh.
dr.errs[br.String()] = err
}
func (fi *FakeIndex) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) {
fi.lk.Lock()
defer fi.lk.Unlock()
bs := blob.String()
mime, ok := fi.mimeType[bs]
if !ok {
return "", 0, os.ENOENT
}
return mime, fi.size[bs], nil
}
func (sr *statsStatReceiver) ReceiveBlob(blob *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err os.Error) {
n, err := io.Copy(ioutil.Discard, source)
if err != nil {
return
}
sr.lock()
defer sr.mu.Unlock()
sr.have[blob.String()] = n
return blobref.SizedBlobRef{blob, n}, nil
}
func (mi *Indexer) GetOwnerClaims(permanode, owner *blobref.BlobRef) (claims search.ClaimList, reterr os.Error) {
claims = make(search.ClaimList, 0)
client, err := mi.getConnection()
if err != nil {
reterr = err
return
}
defer mi.releaseConnection(client)
// TODO: ignore rows where unverified = 'N'
stmt, err := client.Prepare("SELECT blobref, date, claim, attr, value FROM claims WHERE permanode = ? AND signer = ?")
if err != nil {
reterr = err
return
}
err = stmt.BindParams(permanode.String(), owner.String())
if err != nil {
reterr = err
return
}
err = stmt.Execute()
if err != nil {
reterr = err
return
}
var row claimsRow
stmt.BindResult(&row.blobref, &row.date, &row.claim, &row.attr, &row.value)
defer stmt.Close()
for {
done, err := stmt.Fetch()
if err != nil {
reterr = err
return
}
if done {
break
}
t, err := time.Parse(time.RFC3339, trimRFC3339Subseconds(row.date))
if err != nil {
log.Printf("Skipping; error parsing time %q: %v", row.date, err)
continue
}
claims = append(claims, &search.Claim{
BlobRef: blobref.Parse(row.blobref),
Signer: owner,
Permanode: permanode,
Type: row.claim,
Date: t,
Attr: row.attr,
Value: row.value,
})
}
return
}
func (dr *DescribeRequest) describedBlob(b *blobref.BlobRef) *DescribedBlob {
dr.lk.Lock()
defer dr.lk.Unlock()
bs := b.String()
if des, ok := dr.m[bs]; ok {
return des
}
des := &DescribedBlob{Request: dr, BlobRef: b}
dr.m[bs] = des
return des
}
func (b *DescribedBlob) PeerBlob(br *blobref.BlobRef) *DescribedBlob {
if b.Request == nil {
return &DescribedBlob{BlobRef: br, Stub: true}
}
b.Request.lk.Lock()
defer b.Request.lk.Unlock()
if peer, ok := b.Request.m[br.String()]; ok {
return peer
}
return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true}
}
func (mi *Indexer) GetBlobMimeType(blob *blobref.BlobRef) (mime string, size int64, err os.Error) {
rs, err := mi.db.Query("SELECT type, size FROM blobs WHERE blobref=?", blob.String())
if err != nil {
return
}
defer rs.Close()
if !rs.Next() {
err = os.ENOENT
return
}
err = rs.Scan(&mime, &size)
return
}
func (h *SimpleBlobHub) RegisterBlobListener(blob *blobref.BlobRef, ch chan *blobref.BlobRef) {
h.l.Lock()
defer h.l.Unlock()
if h.blobListeners == nil {
h.blobListeners = make(map[string]map[chan *blobref.BlobRef]bool)
}
bstr := blob.String()
_, ok := h.blobListeners[bstr]
if !ok {
h.blobListeners[bstr] = make(map[chan *blobref.BlobRef]bool)
}
h.blobListeners[bstr][ch] = true
}
func (mi *Indexer) keyIdOfSigner(signer *blobref.BlobRef) (keyid string, err os.Error) {
rs, err := mi.db.Query("SELECT keyid FROM signerkeyid WHERE blobref=?", signer.String())
if err != nil {
return
}
defer rs.Close()
if !rs.Next() {
return "", fmt.Errorf("mysqlindexer: failed to find keyid of signer %q", signer.String())
}
err = rs.Scan(&keyid)
return
}
func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err os.Error) {
if sto.ctx == nil {
err = errNoContext
return
}
var b bytes.Buffer
hash := br.Hash()
written, err := io.Copy(io.MultiWriter(hash, &b), in)
if err != nil {
return
}
if !br.HashMatches(hash) {
err = blobserver.ErrCorruptBlob
return
}
mimeType := "application/octet-stream"
bw, err := blobstore.Create(sto.ctx, mimeType)
if err != nil {
return
}
written, err = io.Copy(bw, &b)
if err != nil {
// TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it.
return
}
err = bw.Close()
if err != nil {
// TODO(bradfitz): try to clean up; see if we can find the key, delete it.
return
}
bkey, err := bw.Key()
if err != nil {
return
}
var ent blobEnt
ent.BlobRefStr = br.String()
ent.Size = written
ent.BlobKey = bkey
dkey := datastore.NewKey(sto.ctx, blobKind, br.String(), 0, nil)
_, err = datastore.Put(sto.ctx, dkey, &ent)
if err != nil {
blobstore.Delete(sto.ctx, bkey) // TODO: insert into task queue on error to try later?
return
}
return blobref.SizedBlobRef{br, written}, nil
}
func (mi *Indexer) GetFileInfo(fileRef *blobref.BlobRef) (*search.FileInfo, os.Error) {
rs, err := mi.db.Query("SELECT size, filename, mime FROM bytesfiles WHERE schemaref=?",
fileRef.String())
if err != nil {
return nil, err
}
defer rs.Close()
if !rs.Next() {
return nil, os.ENOENT
}
var fi search.FileInfo
err = rs.Scan(&fi.Size, &fi.FileName, &fi.MimeType)
return &fi, err
}
func (tf *Fetcher) Fetch(ref *blobref.BlobRef) (file blobref.ReadSeekCloser, size int64, err os.Error) {
tf.l.Lock()
defer tf.l.Unlock()
if tf.m == nil {
err = os.ENOENT
return
}
tb, ok := tf.m[ref.String()]
if !ok {
err = os.ENOENT
return
}
file = &strReader{tb.Contents, 0}
size = int64(len(tb.Contents))
return
}
func (mi *Indexer) ExistingFileSchemas(wholeDigest *blobref.BlobRef) (files []*blobref.BlobRef, err os.Error) {
rs, err := mi.db.Query("SELECT schemaref FROM bytesfiles WHERE wholedigest=?", wholeDigest.String())
if err != nil {
return
}
defer rs.Close()
ref := ""
for rs.Next() {
if err := rs.Scan(&ref); err != nil {
return nil, err
}
files = append(files, blobref.Parse(ref))
}
return
}
func (h *SimpleBlobHub) UnregisterBlobListener(blob *blobref.BlobRef, ch chan *blobref.BlobRef) {
h.l.Lock()
defer h.l.Unlock()
if h.blobListeners == nil {
panic("blobhub: UnregisterBlobListener called without RegisterBlobListener")
}
bstr := blob.String()
set, ok := h.blobListeners[bstr]
if !ok {
panic("blobhub: UnregisterBlobListener called without RegisterBlobListener for " + bstr)
}
set[ch] = false, false
if len(set) == 0 {
h.blobListeners[bstr] = nil, false
}
}