func TestStreamSkipRemovedBlobs(t *testing.T) {
// Note: This is the only streaming test that makes use of the
// index (for RemoveBlobs() to succeed). The others do create
// an indexed storage but they do not use the index to stream
// (nor should they use it). The streaming in this test is
// done by reading the underlying diskpacks.
s, cleanup := newTempDiskpacked(t)
defer cleanup()
uploadTestBlobs(t, s, testPack1)
ref, ok := blob.Parse(testPack1[0].digest)
if !ok {
t.Fatalf("blob.Parse: %s", testPack1[0].digest)
}
err := s.RemoveBlobs([]blob.Ref{ref})
if err != nil {
t.Fatalf("RemoveBlobs: %v", err)
}
diskpackedSto := s.(*storage)
expected := len(testPack1) - 1 // We've deleted 1
storagetest.TestStreamer(t, diskpackedSto, storagetest.WantN(expected))
}
func populateLoose(t *testing.T, s *storage) (wants []storagetest.StreamerTestOpt) {
const nBlobs = 10
for i := 0; i < nBlobs; i++ {
(&test.Blob{strconv.Itoa(i)}).MustUpload(t, s)
}
return append(wants, storagetest.WantN(nBlobs))
}
func TestStreamer(t *testing.T) {
s := new(memory.Storage)
phrases := []string{"foo", "bar", "baz", "quux"}
for _, str := range phrases {
(&test.Blob{str}).MustUpload(t, s)
}
storagetest.TestStreamer(t, s, storagetest.WantN(len(phrases)))
}
func TestStreamBlobs(t *testing.T) {
small := new(test.Fetcher)
s := &storage{
small: small,
large: new(test.Fetcher),
meta: sorted.NewMemoryKeyValue(),
log: test.NewLogger(t, "blobpacked: "),
}
s.init()
all := map[blob.Ref]bool{}
const nBlobs = 10
for i := 0; i < nBlobs; i++ {
b := &test.Blob{strconv.Itoa(i)}
b.MustUpload(t, small)
all[b.BlobRef()] = true
}
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
token := "" // beginning
got := map[blob.Ref]bool{}
dest := make(chan blobserver.BlobAndToken, 16)
done := make(chan bool)
go func() {
defer close(done)
for bt := range dest {
got[bt.Blob.Ref()] = true
}
}()
err := s.StreamBlobs(ctx, dest, token)
if err != nil {
t.Fatalf("StreamBlobs = %v", err)
}
<-done
if !reflect.DeepEqual(got, all) {
t.Errorf("Got blobs %v; want %v", got, all)
}
storagetest.TestStreamer(t, s, storagetest.WantN(nBlobs))
}
// Tests that we can correctly switch over to the next pack if we
// still need to stream more blobs when a pack reaches EOF.
func TestStreamMultiplePacks(t *testing.T) {
s, clean := newTestStorage(t, pack{testPack1}, pack{testPack2})
defer clean()
storagetest.TestStreamer(t, s, storagetest.WantN(len(testPack1)+len(testPack2)))
}