func testEnumerate(t *testing.T, sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) {
var after string
var n = 1000
for _, opt := range opts {
switch v := opt.(type) {
case string:
after = v
case int:
n = v
default:
panic("bad option of type " + fmt.Sprint("%T", v))
}
}
want := append([]blob.SizedRef(nil), wantUnsorted...)
sort.Sort(blob.SizedByRef(want))
sbc := make(chan blob.SizedRef, 10)
var got []blob.SizedRef
var grp syncutil.Group
sawEnd := make(chan bool, 1)
grp.Go(func() error {
if err := sto.EnumerateBlobs(context.New(), sbc, after, n); err != nil {
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n)
}
return nil
})
grp.Go(func() error {
for sb := range sbc {
if !sb.Valid() {
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
}
got = append(got, sb)
}
sawEnd <- true
return nil
})
grp.Go(func() error {
select {
case <-sawEnd:
return nil
case <-time.After(10 * time.Second):
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
}
})
if err := grp.Err(); err != nil {
t.Fatalf("Enumerate error: %v", err)
return
}
if len(got) == 0 && len(want) == 0 {
return
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
len(got), len(want), got, want)
}
}
func TestOpt(t *testing.T, opt Opts) {
sto, cleanup := opt.New(t)
defer func() {
if t.Failed() {
t.Logf("test %T FAILED, skipping cleanup!", sto)
} else {
if cleanup != nil {
cleanup()
}
}
}()
r := &run{
t: t,
opt: opt,
sto: sto,
}
t.Logf("Testing blobserver storage %T", sto)
t.Logf("Testing Enumerate for empty")
r.testEnumerate(nil)
var blobs []*test.Blob
var blobRefs []blob.Ref
var blobSizedRefs []blob.SizedRef
contents := []string{"foo", "quux", "asdf", "qwerty", "0123456789"}
if !testing.Short() {
for i := 0; i < 95; i++ {
contents = append(contents, "foo-"+strconv.Itoa(i))
}
}
t.Logf("Testing receive")
for i, x := range contents {
b1 := &test.Blob{x}
if testing.Short() {
t.Logf("blob[%d] = %s: %q", i, b1.BlobRef(), x)
}
b1s, err := sto.ReceiveBlob(b1.BlobRef(), b1.Reader())
if err != nil {
t.Fatalf("ReceiveBlob of %s: %v", b1, err)
}
if b1s != b1.SizedRef() {
t.Fatalf("Received %v; want %v", b1s, b1.SizedRef())
}
blobs = append(blobs, b1)
blobRefs = append(blobRefs, b1.BlobRef())
blobSizedRefs = append(blobSizedRefs, b1.SizedRef())
switch len(blobSizedRefs) {
case 1, 5, 100:
t.Logf("Testing Enumerate for %d blobs", len(blobSizedRefs))
r.testEnumerate(blobSizedRefs)
}
}
t.Logf("Testing Fetch")
for i, b2 := range blobs {
rc, size, err := sto.Fetch(b2.BlobRef())
if err != nil {
t.Fatalf("error fetching %d. %s: %v", i, b2, err)
}
defer rc.Close()
testSizedBlob(t, rc, b2.BlobRef(), int64(size))
}
t.Logf("Testing Stat")
dest := make(chan blob.SizedRef)
errc := make(chan error, 1)
go func() {
errc <- sto.StatBlobs(dest, blobRefs)
}()
testStat(t, dest, blobSizedRefs)
if err := <-errc; err != nil {
t.Fatalf("error stating blobs %s: %v", blobRefs, err)
}
// Enumerate tests.
sort.Sort(blob.SizedByRef(blobSizedRefs))
t.Logf("Testing Enumerate on all")
r.testEnumerate(blobSizedRefs)
t.Logf("Testing Enumerate 'limit' param")
r.testEnumerate(blobSizedRefs[:3], 3)
// Enumerate 'after'
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' param; after %q", after)
r.testEnumerate(blobSizedRefs[3:], after)
}
// Enumerate 'after' + limit
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' + 'limit' param; after %q, limit 1", after)
r.testEnumerate(blobSizedRefs[3:4], after, 1)
}
// Enumerate 'after' with prefix of a blobref + limit
{
after := "a"
t.Logf("Testing Enumerate 'after' + 'limit' param; after %q, limit 1", after)
r.testEnumerate(blobSizedRefs[:1], after, 1)
}
r.testRemove(blobRefs)
r.testSubFetcher()
}
func CheckEnumerate(sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) error {
var after string
var n = 1000
for _, opt := range opts {
switch v := opt.(type) {
case string:
after = v
case int:
n = v
default:
panic("bad option of type " + fmt.Sprintf("%T", v))
}
}
want := append([]blob.SizedRef(nil), wantUnsorted...)
sort.Sort(blob.SizedByRef(want))
sbc := make(chan blob.SizedRef, 10)
var got []blob.SizedRef
var grp syncutil.Group
sawEnd := make(chan bool, 1)
grp.Go(func() error {
ctx := context.New()
defer ctx.Cancel()
if err := sto.EnumerateBlobs(ctx, sbc, after, n); err != nil {
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n, err)
}
return nil
})
grp.Go(func() error {
var lastRef blob.Ref
for sb := range sbc {
if !sb.Valid() {
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
}
got = append(got, sb)
if lastRef.Valid() && sb.Ref.Less(lastRef) {
return fmt.Errorf("blobs appearing out of order")
}
lastRef = sb.Ref
}
sawEnd <- true
return nil
})
grp.Go(func() error {
select {
case <-sawEnd:
return nil
case <-time.After(10 * time.Second):
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
}
})
if err := grp.Err(); err != nil {
return fmt.Errorf("Enumerate error: %v", err)
}
if len(got) == 0 && len(want) == 0 {
return nil
}
var gotSet = map[blob.SizedRef]bool{}
for _, sb := range got {
if gotSet[sb] {
return fmt.Errorf("duplicate blob %v returned in enumerate", sb)
}
gotSet[sb] = true
}
if !reflect.DeepEqual(got, want) {
return fmt.Errorf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
len(got), len(want), got, want)
}
return nil
}
func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup func())) {
sto, cleanup := fn(t)
defer func() {
if t.Failed() {
t.Logf("test %T FAILED, skipping cleanup!", sto)
} else {
cleanup()
}
}()
t.Logf("Testing blobserver storage %T", sto)
t.Logf("Testing Enumerate for empty")
testEnumerate(t, sto, nil)
var blobs []*test.Blob
var blobRefs []blob.Ref
var blobSizedRefs []blob.SizedRef
contents := []string{"foo", "quux", "asdf", "qwerty", "0123456789"}
if !testing.Short() {
for i := 0; i < 95; i++ {
contents = append(contents, "foo-"+strconv.Itoa(i))
}
}
t.Logf("Testing receive")
for _, x := range contents {
b1 := &test.Blob{x}
b1s, err := sto.ReceiveBlob(b1.BlobRef(), b1.Reader())
if err != nil {
t.Fatalf("ReceiveBlob of %s: %v", b1, err)
}
if b1s != b1.SizedRef() {
t.Fatal("Received %v; want %v", b1s, b1.SizedRef())
}
blobs = append(blobs, b1)
blobRefs = append(blobRefs, b1.BlobRef())
blobSizedRefs = append(blobSizedRefs, b1.SizedRef())
switch len(blobSizedRefs) {
case 1, 5, 100:
t.Logf("Testing Enumerate for %d blobs", len(blobSizedRefs))
testEnumerate(t, sto, blobSizedRefs)
}
}
b1 := blobs[0]
// finish here if you want to examine the test directory
//t.Fatalf("FINISH")
t.Logf("Testing FetchStreaming")
for i, b2 := range blobs {
rc, size, err := sto.FetchStreaming(b2.BlobRef())
if err != nil {
t.Fatalf("error fetching %d. %s: %v", i, b2, err)
}
defer rc.Close()
testSizedBlob(t, rc, b2.BlobRef(), size)
}
if fetcher, ok := sto.(fetcher); ok {
rsc, size, err := fetcher.Fetch(b1.BlobRef())
if err != nil {
t.Fatalf("error fetching %s: %v", b1, err)
}
defer rsc.Close()
n, err := rsc.Seek(0, 0)
if err != nil {
t.Fatalf("error seeking in %s: %v", rsc, err)
}
if n != 0 {
t.Fatalf("after seeking to 0, we are at %d!", n)
}
testSizedBlob(t, rsc, b1.BlobRef(), size)
}
t.Logf("Testing Stat")
dest := make(chan blob.SizedRef)
go func() {
if err := sto.StatBlobs(dest, blobRefs); err != nil {
t.Fatalf("error stating blobs %s: %v", blobRefs, err)
}
}()
testStat(t, dest, blobSizedRefs)
// Enumerate tests.
sort.Sort(blob.SizedByRef(blobSizedRefs))
t.Logf("Testing Enumerate on all")
testEnumerate(t, sto, blobSizedRefs)
t.Logf("Testing Enumerate 'limit' param")
testEnumerate(t, sto, blobSizedRefs[:3], 3)
// Enumerate 'after'
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' param; after %q", after)
testEnumerate(t, sto, blobSizedRefs[3:], after)
}
// Enumerate 'after' + limit
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' + 'limit' param; after %q, limit 1", after)
testEnumerate(t, sto, blobSizedRefs[3:4], after, 1)
}
t.Logf("Testing Remove")
if err := sto.RemoveBlobs(blobRefs); err != nil {
if strings.Contains(err.Error(), "not implemented") {
t.Logf("RemoveBlob %s: %v", b1, err)
} else {
t.Fatalf("RemoveBlob %s: %v", b1, err)
}
}
}