func TestIndexSize(t *testing.T) { idx := repository.NewIndex() packs := 200 blobs := 100 for i := 0; i < packs; i++ { packID := restic.NewRandomID() pos := uint(0) for j := 0; j < blobs; j++ { id := restic.NewRandomID() length := uint(i*100 + j) idx.Store(restic.PackedBlob{ Blob: restic.Blob{ Type: restic.DataBlob, ID: id, Offset: pos, Length: length, }, PackID: packID, }) pos += length } } wr := bytes.NewBuffer(nil) err := idx.Encode(wr) OK(t, err) t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len()) }
func (cmd CmdRebuildIndex) storeIndex(index *repository.Index) (*repository.Index, error) { debug.Log("RebuildIndex.RebuildIndex", "saving index") cmd.global.Printf(" saving new index\n") id, err := repository.SaveIndex(cmd.repo, index) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) return nil, err } debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str()) index = repository.NewIndex() return index, nil }
func TestIndexPacks(t *testing.T) { idx := repository.NewIndex() packs := backend.NewIDSet() for i := 0; i < 20; i++ { packID := randomID() idx.Store(repository.PackedBlob{ Type: pack.Data, ID: randomID(), PackID: packID, Offset: 0, Length: 23, }) packs.Insert(packID) } idxPacks := idx.Packs() Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index") }
func (cmd CmdRebuildIndex) rebuildIndex() error { debug.Log("RebuildIndex.RebuildIndex", "start rebuilding index") packs := loadBlobsFromPacks(cmd.repo) cmd.global.Verbosef("loaded blobs from %d packs\n", len(packs)) idx := repository.NewIndex() for packID, entries := range packs { for _, entry := range entries { pb := repository.PackedBlob{ ID: entry.ID, Type: entry.Type, Length: entry.Length, Offset: entry.Offset, PackID: packID, } idx.Store(pb) } } oldIndexes := listIndexIDs(cmd.repo) idx.AddToSupersedes(oldIndexes...) cmd.global.Printf(" saving new index\n") id, err := repository.SaveIndex(cmd.repo, idx) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) return err } debug.Log("RebuildIndex.RebuildIndex", "new index saved as %v", id.Str()) for _, indexID := range oldIndexes { err := cmd.repo.Backend().Remove(backend.Index, indexID.String()) if err != nil { cmd.global.Warnf("unable to remove index %v: %v\n", indexID.Str(), err) } } return nil }
func TestIndexSerialize(t *testing.T) { type testEntry struct { id restic.ID pack restic.ID tpe restic.BlobType offset, length uint } tests := []testEntry{} idx := repository.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() pos := uint(0) for j := 0; j < 20; j++ { id := restic.NewRandomID() length := uint(i*100 + j) idx.Store(restic.PackedBlob{ Blob: restic.Blob{ Type: restic.DataBlob, ID: id, Offset: pos, Length: length, }, PackID: packID, }) tests = append(tests, testEntry{ id: id, pack: packID, tpe: restic.DataBlob, offset: pos, length: length, }) pos += length } } wr := bytes.NewBuffer(nil) err := idx.Encode(wr) OK(t, err) idx2, err := repository.DecodeIndex(wr) OK(t, err) Assert(t, idx2 != nil, "nil returned for decoded index") wr2 := bytes.NewBuffer(nil) err = idx2.Encode(wr2) OK(t, err) for _, testBlob := range tests { list, err := idx.Lookup(testBlob.id, testBlob.tpe) OK(t, err) if len(list) != 1 { t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) } result := list[0] Equals(t, testBlob.pack, result.PackID) Equals(t, testBlob.tpe, result.Type) Equals(t, testBlob.offset, result.Offset) Equals(t, testBlob.length, result.Length) list2, err := idx2.Lookup(testBlob.id, testBlob.tpe) OK(t, err) if len(list2) != 1 { t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list2), list2) } result2 := list2[0] Equals(t, testBlob.pack, result2.PackID) Equals(t, testBlob.tpe, result2.Type) Equals(t, testBlob.offset, result2.Offset) Equals(t, testBlob.length, result2.Length) } // add more blobs to idx newtests := []testEntry{} for i := 0; i < 10; i++ { packID := restic.NewRandomID() pos := uint(0) for j := 0; j < 10; j++ { id := restic.NewRandomID() length := uint(i*100 + j) idx.Store(restic.PackedBlob{ Blob: restic.Blob{ Type: restic.DataBlob, ID: id, Offset: pos, Length: length, }, PackID: packID, }) newtests = append(newtests, testEntry{ id: id, pack: packID, tpe: restic.DataBlob, offset: pos, length: length, }) pos += length } } // serialize idx, unserialize to idx3 wr3 := bytes.NewBuffer(nil) err = idx.Finalize(wr3) OK(t, err) Assert(t, idx.Final(), "index not final after encoding") id := restic.NewRandomID() OK(t, idx.SetID(id)) id2, err := idx.ID() Assert(t, id2.Equal(id), "wrong ID returned: want %v, got %v", id, id2) idx3, err := repository.DecodeIndex(wr3) OK(t, err) Assert(t, idx3 != nil, "nil returned for decoded index") Assert(t, idx3.Final(), "decoded index is not final") // all new blobs must be in the index for _, testBlob := range newtests { list, err := idx3.Lookup(testBlob.id, testBlob.tpe) OK(t, err) if len(list) != 1 { t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) } blob := list[0] Equals(t, testBlob.pack, blob.PackID) Equals(t, testBlob.tpe, blob.Type) Equals(t, testBlob.offset, blob.Offset) Equals(t, testBlob.length, blob.Length) } }
func (cmd CmdRebuildIndex) RebuildIndex() error { debug.Log("RebuildIndex.RebuildIndex", "start") done := make(chan struct{}) defer close(done) indexIDs := backend.NewIDSet() for id := range cmd.repo.List(backend.Index, done) { indexIDs.Insert(id) } cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs)) debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs)) combinedIndex := repository.NewIndex() packsDone := backend.NewIDSet() type Blob struct { id backend.ID tpe pack.BlobType } blobsDone := make(map[Blob]struct{}) i := 0 for indexID := range indexIDs { cmd.global.Printf(" loading index %v\n", i) debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) idx, err := repository.LoadIndex(cmd.repo, indexID.String()) if err != nil { return err } debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str()) for packedBlob := range idx.Each(done) { packsDone.Insert(packedBlob.PackID) b := Blob{ id: packedBlob.ID, tpe: packedBlob.Type, } if _, ok := blobsDone[b]; ok { continue } blobsDone[b] = struct{}{} combinedIndex.Store(packedBlob) } combinedIndex.AddToSupersedes(indexID) if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } i++ } var err error if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("removing %d old indexes\n", len(indexIDs)) for id := range indexIDs { debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str()) err := cmd.repo.Backend().Remove(backend.Index, id.String()) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err) return err } } cmd.global.Printf("checking for additional packs\n") newPacks := 0 var buf []byte for packID := range cmd.repo.List(backend.Data, done) { if packsDone.Has(packID) { continue } debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str()) newPacks++ var err error h := backend.Handle{Type: backend.Data, Name: packID.String()} buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str()) return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err) } hash := backend.Hash(buf) if !hash.Equal(packID) { debug.Log("RebuildIndex.RebuildIndex", "Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) return fmt.Errorf("Pack ID does not match, want %v, got %v", packID.Str(), hash.Str()) } up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf)) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str()) return err } for _, blob := range up.Entries { debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob) combinedIndex.Store(repository.PackedBlob{ Type: blob.Type, ID: blob.ID, PackID: packID, Offset: blob.Offset, Length: blob.Length, }) } if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } } if combinedIndex.Length() > 0 { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { return err } } cmd.global.Printf("added %d packs to the index\n", newPacks) debug.Log("RebuildIndex.RebuildIndex", "done") return nil }