func kvClaim(k, v string, blobParse func(string) (blob.Ref, bool)) (c camtypes.Claim, ok bool) {
const nKeyPart = 5
const nValPart = 4
var keya [nKeyPart]string
var vala [nValPart]string
keyPart := strutil.AppendSplitN(keya[:0], k, "|", -1)
valPart := strutil.AppendSplitN(vala[:0], v, "|", -1)
if len(keyPart) < nKeyPart || len(valPart) < nValPart {
return
}
signerRef, ok := blobParse(valPart[3])
if !ok {
return
}
permaNode, ok := blobParse(keyPart[1])
if !ok {
return
}
claimRef, ok := blobParse(keyPart[4])
if !ok {
return
}
date, err := time.Parse(time.RFC3339, keyPart[3])
if err != nil {
return
}
return camtypes.Claim{
BlobRef: claimRef,
Signer: signerRef,
Permanode: permaNode,
Date: date,
Type: urld(valPart[0]),
Attr: urld(valPart[1]),
Value: urld(valPart[2]),
}, true
}
func (c *Corpus) mergeFileTimesRow(k, v []byte) error {
if len(v) == 0 {
return nil
}
// "filetimes|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b" "1970-01-01T00%3A02%3A03Z"
pipe := bytes.IndexByte(k, '|')
if pipe < 0 {
return fmt.Errorf("unexpected fileinfo key %q", k)
}
br, ok := blob.ParseBytes(k[pipe+1:])
if !ok {
return fmt.Errorf("unexpected filetimes blobref in key %q", k)
}
c.ss = strutil.AppendSplitN(c.ss[:0], urld(string(v)), ",", -1)
times := c.ss
c.mutateFileInfo(br, func(fi *camtypes.FileInfo) {
updateFileInfoTimes(fi, times)
})
return nil
}
func (c *Corpus) mergeFileInfoRow(k, v []byte) error {
// fileinfo|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b" "5|some-stuff.txt|"
pipe := bytes.IndexByte(k, '|')
if pipe < 0 {
return fmt.Errorf("unexpected fileinfo key %q", k)
}
br, ok := blob.ParseBytes(k[pipe+1:])
if !ok {
return fmt.Errorf("unexpected fileinfo blobref in key %q", k)
}
// TODO: could at least use strutil.ParseUintBytes to not stringify and retain
// the length bytes of v.
c.ss = strutil.AppendSplitN(c.ss[:0], string(v), "|", 4)
if len(c.ss) != 3 && len(c.ss) != 4 {
return fmt.Errorf("unexpected fileinfo value %q", v)
}
size, err := strconv.ParseInt(c.ss[0], 10, 64)
if err != nil {
return fmt.Errorf("unexpected fileinfo value %q", v)
}
var wholeRef blob.Ref
if len(c.ss) == 4 && c.ss[3] != "" { // checking for "" because of special files such as symlinks.
var ok bool
wholeRef, ok = blob.Parse(urld(c.ss[3]))
if !ok {
return fmt.Errorf("invalid wholeRef blobref in value %q for fileinfo key %q", v, k)
}
}
c.mutateFileInfo(br, func(fi *camtypes.FileInfo) {
fi.Size = size
fi.FileName = c.str(urld(c.ss[1]))
fi.MIMEType = c.str(urld(c.ss[2]))
fi.WholeRef = wholeRef
})
return nil
}
// fixMissingWholeRef appends the wholeRef to all the keyFileInfo rows values. It should
// only be called to upgrade a version 4 index schema to version 5.
func (x *Index) fixMissingWholeRef(fetcher blob.Fetcher) (err error) {
// We did that check from the caller, but double-check again to prevent from misuse
// of that function.
if x.schemaVersion() != 4 || requiredSchemaVersion != 5 {
panic("fixMissingWholeRef should only be used when upgrading from v4 to v5 of the index schema")
}
log.Println("index: fixing the missing wholeRef in the fileInfo rows...")
defer func() {
if err != nil {
log.Printf("index: fixing the fileInfo rows failed: %v", err)
return
}
log.Print("index: successfully fixed wholeRef in FileInfo rows.")
}()
// first build a reverted keyWholeToFileRef map, so we can get the wholeRef from the fileRef easily.
fileRefToWholeRef := make(map[blob.Ref]blob.Ref)
it := x.queryPrefix(keyWholeToFileRef)
var keyA [3]string
for it.Next() {
keyPart := strutil.AppendSplitN(keyA[:0], it.Key(), "|", 3)
if len(keyPart) != 3 {
return fmt.Errorf("bogus keyWholeToFileRef key: got %q, wanted \"wholetofile|wholeRef|fileRef\"", it.Key())
}
wholeRef, ok1 := blob.Parse(keyPart[1])
fileRef, ok2 := blob.Parse(keyPart[2])
if !ok1 || !ok2 {
return fmt.Errorf("bogus part in keyWholeToFileRef key: %q", it.Key())
}
fileRefToWholeRef[fileRef] = wholeRef
}
if err := it.Close(); err != nil {
return err
}
var fixedEntries, missedEntries int
t := time.NewTicker(5 * time.Second)
defer t.Stop()
// We record the mutations and set them all after the iteration because of the sqlite locking:
// since BeginBatch takes a lock, and Find too, we would deadlock at queryPrefix if we
// started a batch mutation before.
mutations := make(map[string]string)
keyPrefix := keyFileInfo.name + "|"
it = x.queryPrefix(keyFileInfo)
defer it.Close()
var valA [3]string
for it.Next() {
select {
case <-t.C:
log.Printf("Recorded %d missing wholeRef that we'll try to fix, and %d that we can't fix.", fixedEntries, missedEntries)
default:
}
br, ok := blob.ParseBytes(it.KeyBytes()[len(keyPrefix):])
if !ok {
return fmt.Errorf("invalid blobRef %q", it.KeyBytes()[len(keyPrefix):])
}
wholeRef, ok := fileRefToWholeRef[br]
if !ok {
missedEntries++
log.Printf("WARNING: wholeRef for %v not found in index. You should probably rebuild the whole index.", br)
continue
}
valPart := strutil.AppendSplitN(valA[:0], it.Value(), "|", 3)
// The old format we're fixing should be: size|filename|mimetype
if len(valPart) != 3 {
return fmt.Errorf("bogus keyFileInfo value: got %q, wanted \"size|filename|mimetype\"", it.Value())
}
size_s, filename, mimetype := valPart[0], valPart[1], urld(valPart[2])
if strings.Contains(mimetype, "|") {
// I think this can only happen for people migrating from a commit at least as recent as
// 8229c1985079681a652cb65551b4e80a10d135aa, when wholeRef was introduced to keyFileInfo
// but there was no migration code yet.
// For the "production" migrations between 0.8 and 0.9, the index should not have any wholeRef
// in the keyFileInfo entries. So if something goes wrong and is somehow linked to that happening,
// I'd like to know about it, hence the logging.
log.Printf("%v: %v already has a wholeRef, not fixing it", it.Key(), it.Value())
continue
}
size, err := strconv.Atoi(size_s)
if err != nil {
return fmt.Errorf("bogus size in keyFileInfo value %v: %v", it.Value(), err)
}
mutations[keyFileInfo.Key(br)] = keyFileInfo.Val(size, filename, mimetype, wholeRef)
fixedEntries++
}
if err := it.Close(); err != nil {
return err
}
log.Printf("Starting to commit the missing wholeRef fixes (%d entries) now, this can take a while.", fixedEntries)
bm := x.s.BeginBatch()
for k, v := range mutations {
bm.Set(k, v)
}
bm.Set(keySchemaVersion.name, "5")
if err := x.s.CommitBatch(bm); err != nil {
return err
}
if missedEntries > 0 {
log.Printf("Some missing wholeRef entries were not fixed (%d), you should do a full reindex.", missedEntries)
}
return nil
}