func TestIndexSize(t *testing.T) {
idx := repository.NewIndex()
packs := 200
blobs := 100
for i := 0; i < packs; i++ {
packID := randomID()
pos := uint(0)
for j := 0; j < blobs; j++ {
id := randomID()
length := uint(i*100 + j)
idx.Store(pack.Data, id, packID, pos, length)
pos += length
}
}
wr := bytes.NewBuffer(nil)
err := idx.Encode(wr)
OK(t, err)
t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len())
}
func BenchmarkLoadIndex(b *testing.B) {
WithTestEnvironment(b, repoFixture, func(repodir string) {
repo := OpenLocalRepo(b, repodir)
b.ResetTimer()
for i := 0; i < b.N; i++ {
repo.SetIndex(repository.NewIndex())
OK(b, repo.LoadIndex())
}
})
}
// New returns a new checker which runs on repo.
func New(repo *repository.Repository) *Checker {
c := &Checker{
packs: make(map[backend.ID]struct{}),
blobs: make(map[backend.ID]struct{}),
masterIndex: repository.NewIndex(),
indexes: make(map[backend.ID]*repository.Index),
repo: repo,
}
c.blobRefs.M = make(map[backend.ID]uint)
return c
}
func TestIndexPacks(t *testing.T) {
idx := repository.NewIndex()
packs := backend.NewIDSet()
for i := 0; i < 20; i++ {
packID := randomID()
idx.Store(pack.Data, randomID(), packID, 0, 23)
packs.Insert(packID)
}
idxPacks := idx.Packs()
Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index")
}
func (cmd CmdRebuildIndex) storeIndex(index *repository.Index) (*repository.Index, error) {
debug.Log("RebuildIndex.RebuildIndex", "saving index")
cmd.global.Printf(" saving new index\n")
id, err := repository.SaveIndex(cmd.repo, index)
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
return nil, err
}
debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str())
index = repository.NewIndex()
return index, nil
}
func TestIndexPacks(t *testing.T) {
idx := repository.NewIndex()
packs := backend.NewIDSet()
for i := 0; i < 20; i++ {
packID := randomID()
idx.Store(repository.PackedBlob{
Type: pack.Data,
ID: randomID(),
PackID: packID,
Offset: 0,
Length: 23,
})
packs.Insert(packID)
}
idxPacks := idx.Packs()
Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index")
}
func (cmd CmdRebuildIndex) RebuildIndex() error {
debug.Log("RebuildIndex.RebuildIndex", "start")
done := make(chan struct{})
defer close(done)
indexIDs := backend.NewIDSet()
for id := range cmd.repo.List(backend.Index, done) {
indexIDs.Insert(id)
}
cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs))
debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
combinedIndex := repository.NewIndex()
packsDone := backend.NewIDSet()
type Blob struct {
id backend.ID
tpe pack.BlobType
}
blobsDone := make(map[Blob]struct{})
i := 0
for indexID := range indexIDs {
cmd.global.Printf(" loading index %v\n", i)
debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
idx, err := repository.LoadIndex(cmd.repo, indexID.String())
if err != nil {
return err
}
debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())
for packedBlob := range idx.Each(done) {
packsDone.Insert(packedBlob.PackID)
b := Blob{
id: packedBlob.ID,
tpe: packedBlob.Type,
}
if _, ok := blobsDone[b]; ok {
continue
}
blobsDone[b] = struct{}{}
combinedIndex.Store(packedBlob)
}
combinedIndex.AddToSupersedes(indexID)
if repository.IndexFull(combinedIndex) {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
i++
}
var err error
if combinedIndex.Length() > 0 {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
cmd.global.Printf("removing %d old indexes\n", len(indexIDs))
for id := range indexIDs {
debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())
err := cmd.repo.Backend().Remove(backend.Index, id.String())
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
return err
}
}
cmd.global.Printf("checking for additional packs\n")
newPacks := 0
for packID := range cmd.repo.List(backend.Data, done) {
if packsDone.Has(packID) {
continue
}
debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
newPacks++
rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0)
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err)
return err
}
var readSeeker io.ReadSeeker
if r, ok := rd.(io.ReadSeeker); ok {
debug.Log("RebuildIndex.RebuildIndex", "reader is seekable")
readSeeker = r
} else {
debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram")
buf, err := ioutil.ReadAll(rd)
if err != nil {
return err
}
readSeeker = bytes.NewReader(buf)
}
up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker)
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
return err
}
for _, blob := range up.Entries {
debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
combinedIndex.Store(repository.PackedBlob{
Type: blob.Type,
ID: blob.ID,
PackID: packID,
Offset: blob.Offset,
Length: blob.Length,
})
}
err = rd.Close()
debug.Log("RebuildIndex.RebuildIndex", "error closing reader for pack %v: %v", packID.Str(), err)
if repository.IndexFull(combinedIndex) {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
}
if combinedIndex.Length() > 0 {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
cmd.global.Printf("added %d packs to the index\n", newPacks)
debug.Log("RebuildIndex.RebuildIndex", "done")
return nil
}
func TestIndexSerialize(t *testing.T) {
type testEntry struct {
id backend.ID
pack backend.ID
tpe pack.BlobType
offset, length uint
}
tests := []testEntry{}
idx := repository.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
packID := randomID()
pos := uint(0)
for j := 0; j < 20; j++ {
id := randomID()
length := uint(i*100 + j)
idx.Store(pack.Data, id, packID, pos, length)
tests = append(tests, testEntry{
id: id,
pack: packID,
tpe: pack.Data,
offset: pos,
length: length,
})
pos += length
}
}
wr := bytes.NewBuffer(nil)
err := idx.Encode(wr)
OK(t, err)
idx2, err := repository.DecodeIndex(wr)
OK(t, err)
Assert(t, idx2 != nil,
"nil returned for decoded index")
wr2 := bytes.NewBuffer(nil)
err = idx2.Encode(wr2)
OK(t, err)
for _, testBlob := range tests {
packID, tpe, offset, length, err := idx.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, packID)
Equals(t, testBlob.tpe, tpe)
Equals(t, testBlob.offset, offset)
Equals(t, testBlob.length, length)
packID, tpe, offset, length, err = idx2.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, packID)
Equals(t, testBlob.tpe, tpe)
Equals(t, testBlob.offset, offset)
Equals(t, testBlob.length, length)
}
// add more blobs to idx2
newtests := []testEntry{}
for i := 0; i < 10; i++ {
packID := randomID()
pos := uint(0)
for j := 0; j < 10; j++ {
id := randomID()
length := uint(i*100 + j)
idx2.Store(pack.Data, id, packID, pos, length)
newtests = append(newtests, testEntry{
id: id,
pack: packID,
tpe: pack.Data,
offset: pos,
length: length,
})
pos += length
}
}
// serialize idx2, unserialize to idx3
wr3 := bytes.NewBuffer(nil)
err = idx2.Encode(wr3)
OK(t, err)
idx3, err := repository.DecodeIndex(wr3)
OK(t, err)
Assert(t, idx3 != nil,
"nil returned for decoded index")
// all old blobs must not be present in the index
for _, testBlob := range tests {
_, _, _, _, err := idx3.Lookup(testBlob.id)
Assert(t, err != nil,
"found old id %v in serialized index", testBlob.id.Str())
}
// all new blobs must be in the index
for _, testBlob := range newtests {
packID, tpe, offset, length, err := idx3.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, packID)
Equals(t, testBlob.tpe, tpe)
Equals(t, testBlob.offset, offset)
Equals(t, testBlob.length, length)
}
}
func TestIndexSerialize(t *testing.T) {
type testEntry struct {
id backend.ID
pack backend.ID
tpe pack.BlobType
offset, length uint
}
tests := []testEntry{}
idx := repository.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
packID := randomID()
pos := uint(0)
for j := 0; j < 20; j++ {
id := randomID()
length := uint(i*100 + j)
idx.Store(repository.PackedBlob{
Type: pack.Data,
ID: id,
PackID: packID,
Offset: pos,
Length: length,
})
tests = append(tests, testEntry{
id: id,
pack: packID,
tpe: pack.Data,
offset: pos,
length: length,
})
pos += length
}
}
wr := bytes.NewBuffer(nil)
err := idx.Encode(wr)
OK(t, err)
idx2, err := repository.DecodeIndex(wr)
OK(t, err)
Assert(t, idx2 != nil,
"nil returned for decoded index")
wr2 := bytes.NewBuffer(nil)
err = idx2.Encode(wr2)
OK(t, err)
for _, testBlob := range tests {
result, err := idx.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, result.PackID)
Equals(t, testBlob.tpe, result.Type)
Equals(t, testBlob.offset, result.Offset)
Equals(t, testBlob.length, result.Length)
result2, err := idx2.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, result2.PackID)
Equals(t, testBlob.tpe, result2.Type)
Equals(t, testBlob.offset, result2.Offset)
Equals(t, testBlob.length, result2.Length)
}
// add more blobs to idx
newtests := []testEntry{}
for i := 0; i < 10; i++ {
packID := randomID()
pos := uint(0)
for j := 0; j < 10; j++ {
id := randomID()
length := uint(i*100 + j)
idx.Store(repository.PackedBlob{
Type: pack.Data,
ID: id,
PackID: packID,
Offset: pos,
Length: length,
})
newtests = append(newtests, testEntry{
id: id,
pack: packID,
tpe: pack.Data,
offset: pos,
length: length,
})
pos += length
}
}
// serialize idx, unserialize to idx3
wr3 := bytes.NewBuffer(nil)
err = idx.Finalize(wr3)
OK(t, err)
Assert(t, idx.Final(),
"index not final after encoding")
id := randomID()
OK(t, idx.SetID(id))
id2, err := idx.ID()
Assert(t, id2.Equal(id),
"wrong ID returned: want %v, got %v", id, id2)
idx3, err := repository.DecodeIndex(wr3)
OK(t, err)
Assert(t, idx3 != nil,
"nil returned for decoded index")
Assert(t, idx3.Final(),
"decoded index is not final")
// all new blobs must be in the index
for _, testBlob := range newtests {
blob, err := idx3.Lookup(testBlob.id)
OK(t, err)
Equals(t, testBlob.pack, blob.PackID)
Equals(t, testBlob.tpe, blob.Type)
Equals(t, testBlob.offset, blob.Offset)
Equals(t, testBlob.length, blob.Length)
}
}
func (cmd CmdRebuildIndex) RebuildIndex() error {
debug.Log("RebuildIndex.RebuildIndex", "start")
done := make(chan struct{})
defer close(done)
indexIDs := backend.NewIDSet()
for id := range cmd.repo.List(backend.Index, done) {
indexIDs.Insert(id)
}
cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs))
debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
combinedIndex := repository.NewIndex()
packsDone := backend.NewIDSet()
type Blob struct {
id backend.ID
tpe pack.BlobType
}
blobsDone := make(map[Blob]struct{})
i := 0
for indexID := range indexIDs {
cmd.global.Printf(" loading index %v\n", i)
debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
idx, err := repository.LoadIndex(cmd.repo, indexID.String())
if err != nil {
return err
}
debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())
for packedBlob := range idx.Each(done) {
packsDone.Insert(packedBlob.PackID)
b := Blob{
id: packedBlob.ID,
tpe: packedBlob.Type,
}
if _, ok := blobsDone[b]; ok {
continue
}
blobsDone[b] = struct{}{}
combinedIndex.Store(packedBlob)
}
combinedIndex.AddToSupersedes(indexID)
if repository.IndexFull(combinedIndex) {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
i++
}
var err error
if combinedIndex.Length() > 0 {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
cmd.global.Printf("removing %d old indexes\n", len(indexIDs))
for id := range indexIDs {
debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())
err := cmd.repo.Backend().Remove(backend.Index, id.String())
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
return err
}
}
cmd.global.Printf("checking for additional packs\n")
newPacks := 0
var buf []byte
for packID := range cmd.repo.List(backend.Data, done) {
if packsDone.Has(packID) {
continue
}
debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
newPacks++
var err error
h := backend.Handle{Type: backend.Data, Name: packID.String()}
buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf)
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str())
return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err)
}
hash := backend.Hash(buf)
if !hash.Equal(packID) {
debug.Log("RebuildIndex.RebuildIndex", "Pack ID does not match, want %v, got %v", packID.Str(), hash.Str())
return fmt.Errorf("Pack ID does not match, want %v, got %v", packID.Str(), hash.Str())
}
up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf))
if err != nil {
debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
return err
}
for _, blob := range up.Entries {
debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
combinedIndex.Store(repository.PackedBlob{
Type: blob.Type,
ID: blob.ID,
PackID: packID,
Offset: blob.Offset,
Length: blob.Length,
})
}
if repository.IndexFull(combinedIndex) {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
}
if combinedIndex.Length() > 0 {
combinedIndex, err = cmd.storeIndex(combinedIndex)
if err != nil {
return err
}
}
cmd.global.Printf("added %d packs to the index\n", newPacks)
debug.Log("RebuildIndex.RebuildIndex", "done")
return nil
}