func TestRepositoryIncrementalIndex(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) repository.IndexFull = func(*repository.Index) bool { return true } // add 15 packs for j := 0; j < 5; j++ { // add 3 packs, write intermediate index for i := 0; i < 3; i++ { saveRandomDataBlobs(t, repo, 5, 1<<15) OK(t, repo.Flush()) } OK(t, repo.SaveFullIndex()) } // add another 5 packs for i := 0; i < 5; i++ { saveRandomDataBlobs(t, repo, 5, 1<<15) OK(t, repo.Flush()) } // save final index OK(t, repo.SaveIndex()) type packEntry struct { id backend.ID indexes []*repository.Index } packEntries := make(map[backend.ID]map[backend.ID]struct{}) for id := range repo.List(backend.Index, nil) { idx, err := repository.LoadIndex(repo, id.String()) OK(t, err) for pb := range idx.Each(nil) { if _, ok := packEntries[pb.PackID]; !ok { packEntries[pb.PackID] = make(map[backend.ID]struct{}) } packEntries[pb.PackID][id] = struct{}{} } } for packID, ids := range packEntries { if len(ids) > 1 { t.Errorf("pack %v listed in %d indexes\n", packID, len(ids)) } } }
func TestConvertIndex(t *testing.T) { WithTestEnvironment(t, oldIndexTestRepo, func(repodir string) { repo := OpenLocalRepo(t, repodir) old := make(map[backend.ID]*repository.Index) for id := range repo.List(backend.Index, nil) { idx, err := repository.LoadIndex(repo, id.String()) OK(t, err) old[id] = idx } OK(t, repository.ConvertIndexes(repo)) for id := range repo.List(backend.Index, nil) { idx, err := repository.LoadIndexWithDecoder(repo, id.String(), repository.DecodeIndex) OK(t, err) Assert(t, len(idx.Supersedes()) == 1, "Expected index %v to supersed exactly one index, got %v", id, idx.Supersedes()) oldIndexID := idx.Supersedes()[0] oldIndex, ok := old[oldIndexID] Assert(t, ok, "Index %v superseds %v, but that wasn't found in the old index map", id.Str(), oldIndexID.Str()) Assert(t, idx.Count(pack.Data) == oldIndex.Count(pack.Data), "Index %v count blobs %v: %v != %v", id.Str(), pack.Data, idx.Count(pack.Data), oldIndex.Count(pack.Data)) Assert(t, idx.Count(pack.Tree) == oldIndex.Count(pack.Tree), "Index %v count blobs %v: %v != %v", id.Str(), pack.Tree, idx.Count(pack.Tree), oldIndex.Count(pack.Tree)) for packedBlob := range idx.Each(nil) { packID, tpe, offset, length, err := oldIndex.Lookup(packedBlob.ID) OK(t, err) Assert(t, packID == packedBlob.PackID, "Check blob %v: pack ID %v != %v", packedBlob.ID, packID, packedBlob.PackID) Assert(t, tpe == packedBlob.Type, "Check blob %v: Type %v != %v", packedBlob.ID, tpe, packedBlob.Type) Assert(t, offset == packedBlob.Offset, "Check blob %v: Type %v != %v", packedBlob.ID, offset, packedBlob.Offset) Assert(t, length == packedBlob.Length, "Check blob %v: Type %v != %v", packedBlob.ID, length, packedBlob.Length) } } }) }
func (cmd CmdDump) DumpIndexes() error { done := make(chan struct{}) defer close(done) for id := range cmd.repo.List(backend.Index, done) { fmt.Printf("index_id: %v\n", id) idx, err := repository.LoadIndex(cmd.repo, id.String()) if err != nil { return err } err = idx.Dump(os.Stdout) if err != nil { return err } } return nil }
func (cmd CmdRebuildIndex) RebuildIndex() error { debug.Log("RebuildIndex.RebuildIndex", "start") done := make(chan struct{}) defer close(done) indexIDs := backend.NewIDSet() for id := range cmd.repo.List(backend.Index, done) { indexIDs.Insert(id) } cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs)) debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs)) combinedIndex := repository.NewIndex() packsDone := backend.NewIDSet() type Blob struct { id backend.ID tpe pack.BlobType } blobsDone := make(map[Blob]struct{}) i := 0 for indexID := range indexIDs { cmd.global.Printf(" loading index %v\n", i) debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) idx, err := repository.LoadIndex(cmd.repo, indexID.String()) if err != nil { return err } debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str()) for packedBlob := range idx.Each(done) { packsDone.Insert(packedBlob.PackID) b := Blob{ id: packedBlob.ID, tpe: packedBlob.Type, } if _, ok := blobsDone[b]; ok { continue } blobsDone[b] = struct{}{} combinedIndex.Store(packedBlob) } combinedIndex.AddToSupersedes(indexID) if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } i++ } var err error if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("removing %d old indexes\n", len(indexIDs)) for id := range indexIDs { debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str()) err := cmd.repo.Backend().Remove(backend.Index, id.String()) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err) return err } } cmd.global.Printf("checking for additional packs\n") newPacks := 0 for packID := range cmd.repo.List(backend.Data, done) { if packsDone.Has(packID) { continue } debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str()) newPacks++ rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err) return err } var readSeeker io.ReadSeeker if r, ok := rd.(io.ReadSeeker); ok { debug.Log("RebuildIndex.RebuildIndex", "reader is seekable") readSeeker = r } else { debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram") buf, err := ioutil.ReadAll(rd) if err != nil { return err } readSeeker = bytes.NewReader(buf) } up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str()) return err } for _, blob := range up.Entries { debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob) combinedIndex.Store(repository.PackedBlob{ Type: blob.Type, ID: blob.ID, PackID: packID, Offset: blob.Offset, Length: blob.Length, }) } err = rd.Close() debug.Log("RebuildIndex.RebuildIndex", "error closing reader for pack %v: %v", packID.Str(), err) if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } } if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("added %d packs to the index\n", newPacks) debug.Log("RebuildIndex.RebuildIndex", "done") return nil }
// LoadIndex loads all index files. func (c *Checker) LoadIndex() error { debug.Log("LoadIndex", "Start") type indexRes struct { Index *repository.Index ID string } indexCh := make(chan indexRes) worker := func(id string, done <-chan struct{}) error { debug.Log("LoadIndex", "worker got index %v", id) idx, err := repository.LoadIndex(c.repo, id) if err != nil { return err } select { case indexCh <- indexRes{Index: idx, ID: id}: case <-done: } return nil } var perr error go func() { defer close(indexCh) debug.Log("LoadIndex", "start loading indexes in parallel") perr = repository.FilesInParallel(c.repo.Backend(), backend.Index, defaultParallelism, worker) debug.Log("LoadIndex", "loading indexes finished, error: %v", perr) }() done := make(chan struct{}) defer close(done) for res := range indexCh { debug.Log("LoadIndex", "process index %v", res.ID) id, err := backend.ParseID(res.ID) if err != nil { return err } c.indexes[id] = res.Index c.masterIndex.Merge(res.Index) debug.Log("LoadIndex", "process blobs") cnt := 0 for blob := range res.Index.Each(done) { c.packs[blob.PackID] = struct{}{} c.blobs[blob.ID] = struct{}{} c.blobRefs.M[blob.ID] = 0 cnt++ } debug.Log("LoadIndex", "%d blobs processed", cnt) } debug.Log("LoadIndex", "done, error %v", perr) c.repo.SetIndex(c.masterIndex) return perr }
func (cmd CmdRebuildIndex) RebuildIndex() error { debug.Log("RebuildIndex.RebuildIndex", "start") done := make(chan struct{}) defer close(done) indexIDs := backend.NewIDSet() for id := range cmd.repo.List(backend.Index, done) { indexIDs.Insert(id) } cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs)) debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs)) combinedIndex := repository.NewIndex() packsDone := backend.NewIDSet() type Blob struct { id backend.ID tpe pack.BlobType } blobsDone := make(map[Blob]struct{}) i := 0 for indexID := range indexIDs { cmd.global.Printf(" loading index %v\n", i) debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) idx, err := repository.LoadIndex(cmd.repo, indexID.String()) if err != nil { return err } debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str()) for packedBlob := range idx.Each(done) { packsDone.Insert(packedBlob.PackID) b := Blob{ id: packedBlob.ID, tpe: packedBlob.Type, } if _, ok := blobsDone[b]; ok { continue } blobsDone[b] = struct{}{} combinedIndex.Store(packedBlob) } combinedIndex.AddToSupersedes(indexID) if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } i++ } var err error if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("removing %d old indexes\n", len(indexIDs)) for id := range indexIDs { debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str()) err := cmd.repo.Backend().Remove(backend.Index, id.String()) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err) return err } } cmd.global.Printf("checking for additional packs\n") newPacks := 0 var buf []byte for packID := range cmd.repo.List(backend.Data, done) { if packsDone.Has(packID) { continue } debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str()) newPacks++ var err error h := backend.Handle{Type: backend.Data, Name: packID.String()} buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str()) return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err) } hash := backend.Hash(buf) if !hash.Equal(packID) { debug.Log("RebuildIndex.RebuildIndex", "Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) return fmt.Errorf("Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) } up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf)) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str()) return err } for _, blob := range up.Entries { debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob) combinedIndex.Store(repository.PackedBlob{ Type: blob.Type, ID: blob.ID, PackID: packID, Offset: blob.Offset, Length: blob.Length, }) } if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } } if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("added %d packs to the index\n", newPacks) debug.Log("RebuildIndex.RebuildIndex", "done") return nil }