// restoreInvariants sorts claims by date and
// recalculates latest attributes.
func (pm *PermanodeMeta) restoreInvariants() {
sort.Sort(camtypes.ClaimPtrsByDate(pm.Claims))
pm.Attrs = make(map[string][]string)
for _, cl := range pm.Claims {
pm.appendAttrClaim(cl)
}
}
func (c *Corpus) mergeClaimRow(k, v []byte) error {
// TODO: update kvClaim to take []byte instead of string
cl, ok := kvClaim(string(k), string(v), c.blobParse)
if !ok || !cl.Permanode.Valid() {
return fmt.Errorf("bogus claim row: %q -> %q", k, v)
}
cl.Type = c.str(cl.Type)
cl.Attr = c.str(cl.Attr)
cl.Value = c.str(cl.Value) // less likely to intern, but some (tags) do
pn := c.br(cl.Permanode)
pm, ok := c.permanodes[pn]
if !ok {
pm = new(PermanodeMeta)
c.permanodes[pn] = pm
}
pm.Claims = append(pm.Claims, &cl)
if !c.building {
// Unless we're still starting up (at which we sort at
// the end instead), keep this sorted.
sort.Sort(camtypes.ClaimPtrsByDate(pm.Claims))
}
if vbr, ok := blob.Parse(cl.Value); ok {
c.claimBack[vbr] = append(c.claimBack[vbr], &cl)
}
return nil
}
// restoreInvariants sorts claims by date and
// recalculates latest attributes.
func (pm *PermanodeMeta) restoreInvariants() {
sort.Sort(camtypes.ClaimPtrsByDate(pm.Claims))
pm.attr = make(attrValues)
pm.signer = make(map[blob.Ref]attrValues)
for _, cl := range pm.Claims {
pm.appendAttrClaim(cl)
}
}
// fixupLastClaim fixes invariants on the assumption
// that the all but the last element in Claims are sorted by date
// and the last element is the only one not yet included in Attrs.
func (pm *PermanodeMeta) fixupLastClaim() {
if pm.attr != nil {
n := len(pm.Claims)
if n < 2 || camtypes.ClaimPtrsByDate(pm.Claims).Less(n-2, n-1) {
// already sorted, update Attrs from new Claim
pm.appendAttrClaim(pm.Claims[n-1])
return
}
}
pm.restoreInvariants()
}
func (c *Corpus) scanFromStorage(s sorted.KeyValue) error {
c.building = true
var ms0 *runtime.MemStats
if logCorpusStats {
ms0 = memstats()
log.Printf("Slurping corpus to memory from index...")
log.Printf("Slurping corpus to memory from index... (1/6: meta rows)")
}
// We do the "meta" rows first, before the prefixes below, because it
// populates the blobs map (used for blobref interning) and the camBlobs
// map (used for hinting the size of other maps)
if err := c.scanPrefix(s, "meta:"); err != nil {
return err
}
c.files = make(map[blob.Ref]camtypes.FileInfo, len(c.camBlobs["file"]))
c.permanodes = make(map[blob.Ref]*PermanodeMeta, len(c.camBlobs["permanode"]))
cpu0 := osutil.CPUUsage()
prefixes := []string{
"signerkeyid:",
"claim|",
"fileinfo|",
"filetimes|",
"imagesize|",
"wholetofile|",
"exifgps|",
}
var grp syncutil.Group
for i, prefix := range prefixes {
if logCorpusStats {
log.Printf("Slurping corpus to memory from index... (%d/%d: prefix %q)", i+2, len(prefixes)+1,
prefix[:len(prefix)-1])
}
prefix := prefix
grp.Go(func() error { return c.scanPrefix(s, prefix) })
}
if err := grp.Err(); err != nil {
return err
}
// Post-load optimizations and restoration of invariants.
for _, pm := range c.permanodes {
// Restore invariants violated during building:
sort.Sort(camtypes.ClaimPtrsByDate(pm.Claims))
// And intern some stuff.
for _, cl := range pm.Claims {
cl.BlobRef = c.br(cl.BlobRef)
cl.Signer = c.br(cl.Signer)
cl.Permanode = c.br(cl.Permanode)
cl.Target = c.br(cl.Target)
}
}
c.brOfStr = nil // drop this now.
c.building = false
// log.V(1).Printf("interned blob.Ref = %d", c.brInterns)
if logCorpusStats {
cpu := osutil.CPUUsage() - cpu0
ms1 := memstats()
memUsed := ms1.Alloc - ms0.Alloc
if ms1.Alloc < ms0.Alloc {
memUsed = 0
}
log.Printf("Corpus stats: %.3f MiB mem: %d blobs (%.3f GiB) (%d schema (%d permanode, %d file (%d image), ...)",
float64(memUsed)/(1<<20),
len(c.blobs),
float64(c.sumBlobBytes)/(1<<30),
c.numSchemaBlobsLocked(),
len(c.permanodes),
len(c.files),
len(c.imageInfo))
log.Printf("Corpus scanning CPU usage: %v", cpu)
}
return nil
}
