func TestEnumerate(t *testing.T) {
ds := NewStorage(t)
defer cleanUp(ds)
// For test simplicity foo, bar, and baz all have ascending
// sha1s and lengths.
foo := &test.Blob{"foo"} // 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
bar := &test.Blob{"baar"} // b23361951dde70cb3eca44c0c674181673a129dc
baz := &test.Blob{"bazzz"} // e0eb17003ce1c2812ca8f19089fff44ca32b3710
foo.MustUpload(t, ds)
bar.MustUpload(t, ds)
baz.MustUpload(t, ds)
limit := 5000
ch := make(chan blob.SizedRef)
errCh := make(chan error)
go func() {
errCh <- ds.EnumerateBlobs(context.New(), ch, "", limit)
}()
var (
sb blob.SizedRef
ok bool
)
sb, ok = <-ch
Assert(t, ok, "got 1st blob")
ExpectInt(t, 3, int(sb.Size), "1st blob size")
sb, ok = <-ch
Assert(t, ok, "got 2nd blob")
ExpectInt(t, 4, int(sb.Size), "2nd blob size")
sb, ok = <-ch
Assert(t, ok, "got 3rd blob")
ExpectInt(t, 5, int(sb.Size), "3rd blob size")
sb, ok = <-ch
Assert(t, !ok, "got channel close")
ExpectNil(t, <-errCh, "EnumerateBlobs return value")
// Now again, but skipping foo's blob
ch = make(chan blob.SizedRef)
go func() {
errCh <- ds.EnumerateBlobs(context.New(),
ch,
foo.BlobRef().String(),
limit)
}()
sb, ok = <-ch
Assert(t, ok, "got 1st blob, skipping foo")
ExpectInt(t, 4, int(sb.Size), "blob size")
sb, ok = <-ch
Assert(t, ok, "got 2nd blob, skipping foo")
ExpectInt(t, 5, int(sb.Size), "blob size")
sb, ok = <-ch
Assert(t, !ok, "got final nil")
ExpectNil(t, <-errCh, "EnumerateBlobs return value")
}
func (ia *importerAcct) start() {
ia.mu.Lock()
defer ia.mu.Unlock()
if ia.current != nil {
return
}
ctx := context.New()
rc := &RunContext{
Context: ctx,
Host: ia.im.host,
ia: ia,
}
ia.current = rc
ia.stopped = false
ia.lastRunStart = time.Now()
go func() {
log.Printf("Starting importer %s: %s", ia.im.name, ia.AccountLinkSummary())
err := ia.im.impl.Run(rc)
if err != nil {
log.Printf("Importer %s error: %v", ia.im.name, err)
} else {
log.Printf("Importer %s finished.", ia.im.name)
}
ia.mu.Lock()
defer ia.mu.Unlock()
ia.current = nil
ia.stopped = false
ia.lastRunDone = time.Now()
ia.lastRunErr = err
}()
}
func TestGetUserID(t *testing.T) {
im := &imp{
credsVal: &oauth.Credentials{
Token: "foo",
Secret: "bar",
},
}
ctx := context.New()
ctx.SetHTTPClient(&http.Client{
Transport: newFakeTransport(map[string]func() *http.Response{
apiURL + userInfoAPIPath: fileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
}),
})
inf, err := im.getUserInfo(ctx)
if err != nil {
t.Fatal(err)
}
want := userInfo{
ID: "2325935334",
ScreenName: "lejatorn",
Name: "Mathieu Lonjaret",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func (ia *importerAcct) start() {
ia.mu.Lock()
defer ia.mu.Unlock()
if ia.current != nil {
return
}
rc := &RunContext{
// TODO: context plumbing
Context: context.New(context.WithHTTPClient(ia.im.host.HTTPClient())),
Host: ia.im.host,
ia: ia,
}
ia.current = rc
ia.stopped = false
ia.lastRunStart = time.Now()
go func() {
log.Printf("Starting %v: %s", ia, ia.AccountLinkSummary())
err := ia.im.impl.Run(rc)
if err != nil {
log.Printf("%v error: %v", ia, err)
} else {
log.Printf("%v finished.", ia)
}
ia.mu.Lock()
defer ia.mu.Unlock()
ia.current = nil
ia.stopped = false
ia.lastRunDone = time.Now()
ia.lastRunErr = err
go ia.maybeStart()
}()
}
func checkEnumerate(idx *index.Index, want []blob.SizedRef, args *enumArgs) error {
if args == nil {
args = &enumArgs{}
}
if args.ctx == nil {
args.ctx = context.New()
}
if args.dest == nil {
args.dest = make(chan blob.SizedRef)
}
if args.limit == 0 {
args.limit = 5000
}
errCh := make(chan error)
go func() {
errCh <- idx.EnumerateBlobs(args.ctx, args.dest, args.after, args.limit)
}()
for k, sbr := range want {
got, ok := <-args.dest
if !ok {
return fmt.Errorf("could not enumerate blob %d", k)
}
if got != sbr {
return fmt.Errorf("enumeration %d: got %v, wanted %v", k, got, sbr)
}
}
_, ok := <-args.dest
if ok {
return errors.New("chan was not closed after enumeration")
}
return <-errCh
}
func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
defer func() {
sh.mu.Lock()
if err != nil {
errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
sh.logf("%s", errs)
sh.vshardErrs = append(sh.vshardErrs, errs)
} else {
sh.vshardDone++
}
sh.mu.Unlock()
}()
ctx := context.New()
defer ctx.Cancel()
src, serrc := sh.startValidatePrefix(ctx, pfx, false)
dst, derrc := sh.startValidatePrefix(ctx, pfx, true)
missing := make(chan blob.SizedRef, 8)
go blobserver.ListMissingDestinationBlobs(missing, func(blob.Ref) {}, src, dst)
for sb := range missing {
sh.mu.Lock()
sh.vmissing++
sh.mu.Unlock()
// TODO: stats for missing blobs found.
sh.enqueue(sb)
}
if err := <-serrc; err != nil {
return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
}
if err := <-derrc; err != nil {
return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
}
return nil
}
func TestS3(t *testing.T) {
if *bucket == "" || *key == "" || *secret == "" {
t.Skip("Skipping test because at least one of -s3_key, -s3_secret, or -s3_bucket flags has not been provided.")
}
if !strings.HasPrefix(*bucket, "camlistore-") || !strings.HasSuffix(*bucket, "-test") {
t.Fatalf("bogus bucket name %q; must begin with 'camlistore-' and end in '-test'", *bucket)
}
storagetest.Test(t, func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
sto, err := newFromConfig(nil, jsonconfig.Obj{
"aws_access_key": *key,
"aws_secret_access_key": *secret,
"bucket": *bucket,
})
if err != nil {
t.Fatalf("newFromConfig error: %v", err)
}
if !testing.Short() {
log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
}
clearBucket := func() {
var all []blob.Ref
blobserver.EnumerateAll(context.New(), sto, func(sb blob.SizedRef) error {
t.Logf("Deleting: %v", sb.Ref)
all = append(all, sb.Ref)
return nil
})
if err := sto.RemoveBlobs(all); err != nil {
t.Fatalf("Error removing blobs during cleanup: %v", err)
}
}
clearBucket()
return sto, clearBucket
})
}
func testEnumerate(t *testing.T, sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) {
var after string
var n = 1000
for _, opt := range opts {
switch v := opt.(type) {
case string:
after = v
case int:
n = v
default:
panic("bad option of type " + fmt.Sprint("%T", v))
}
}
want := append([]blob.SizedRef(nil), wantUnsorted...)
sort.Sort(blob.SizedByRef(want))
sbc := make(chan blob.SizedRef, 10)
var got []blob.SizedRef
var grp syncutil.Group
sawEnd := make(chan bool, 1)
grp.Go(func() error {
if err := sto.EnumerateBlobs(context.New(), sbc, after, n); err != nil {
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n)
}
return nil
})
grp.Go(func() error {
for sb := range sbc {
if !sb.Valid() {
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
}
got = append(got, sb)
}
sawEnd <- true
return nil
})
grp.Go(func() error {
select {
case <-sawEnd:
return nil
case <-time.After(10 * time.Second):
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
}
})
if err := grp.Err(); err != nil {
t.Fatalf("Enumerate error: %v", err)
return
}
if len(got) == 0 && len(want) == 0 {
return
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
len(got), len(want), got, want)
}
}
func TestEnumerateIsSorted(t *testing.T) {
ds := NewStorage(t)
defer cleanUp(ds)
const blobsToMake = 250
t.Logf("Uploading test blobs...")
for i := 0; i < blobsToMake; i++ {
blob := &test.Blob{fmt.Sprintf("blob-%d", i)}
blob.MustUpload(t, ds)
}
// Make some fake blobs in other partitions to confuse the
// enumerate code.
// TODO(bradfitz): remove this eventually.
fakeDir := ds.root + "/partition/queue-indexer/sha1/1f0/710"
ExpectNil(t, os.MkdirAll(fakeDir, 0755), "creating fakeDir")
ExpectNil(t, ioutil.WriteFile(fakeDir+"/sha1-1f07105465650aa243cfc1b1bbb1c68ea95c6812.dat",
[]byte("fake file"), 0644), "writing fake blob")
// And the same for a "cache" directory, used by the default configuration.
fakeDir = ds.root + "/cache/sha1/1f0/710"
ExpectNil(t, os.MkdirAll(fakeDir, 0755), "creating cache fakeDir")
ExpectNil(t, ioutil.WriteFile(fakeDir+"/sha1-1f07105465650aa243cfc1b1bbb1c68ea95c6812.dat",
[]byte("fake file"), 0644), "writing fake blob")
var tests = []struct {
limit int
after string
}{
{200, ""},
{blobsToMake, ""},
{200, "sha1-2"},
{200, "sha1-3"},
{200, "sha1-4"},
{200, "sha1-5"},
{200, "sha1-e"},
{200, "sha1-f"},
{200, "sha1-ff"},
}
for _, test := range tests {
limit := test.limit
ch := make(chan blob.SizedRef)
errCh := make(chan error)
go func() {
errCh <- ds.EnumerateBlobs(context.New(), ch, test.after, limit)
}()
got := make([]blob.SizedRef, 0, blobsToMake)
for sb := range ch {
got = append(got, sb)
}
if err := <-errCh; err != nil {
t.Errorf("case %+v; enumerate error: %v", test, err)
continue
}
if !sort.IsSorted(SortedSizedBlobs(got)) {
t.Errorf("case %+v: expected sorted; got: %q", test, got)
}
}
}
func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
defer func() {
sh.mu.Lock()
if err != nil {
errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
sh.logf("%s", errs)
sh.vshardErrs = append(sh.vshardErrs, errs)
} else {
sh.vshardDone++
}
sh.mu.Unlock()
}()
ctx := context.New()
defer ctx.Cancel()
src, serrc := sh.startValidatePrefix(ctx, pfx, false)
dst, derrc := sh.startValidatePrefix(ctx, pfx, true)
srcErr := &chanError{
C: serrc,
Wrap: func(err error) error {
return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
},
}
dstErr := &chanError{
C: derrc,
Wrap: func(err error) error {
return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
},
}
missingc := make(chan blob.SizedRef, 8)
go blobserver.ListMissingDestinationBlobs(missingc, func(blob.Ref) {}, src, dst)
var missing []blob.SizedRef
for sb := range missingc {
missing = append(missing, sb)
}
if err := srcErr.Get(); err != nil {
return err
}
if err := dstErr.Get(); err != nil {
return err
}
for _, sb := range missing {
if enqErr := sh.enqueue(sb); enqErr != nil {
if err == nil {
err = enqErr
}
} else {
sh.mu.Lock()
sh.vmissing += 1
sh.mu.Unlock()
}
}
return err
}
func (s *storage) anyZipPacks() (v bool) {
ctx := context.New()
defer ctx.Cancel()
dest := make(chan blob.SizedRef, 1)
if err := s.large.EnumerateBlobs(ctx, dest, "", 1); err != nil {
// Not a great interface in general, but only needed
// by the start-up check for now, where it doesn't
// really matter.
return false
}
_, ok := <-dest
return ok
}
func TestEnumerateEmpty(t *testing.T) {
ds := NewStorage(t)
defer cleanUp(ds)
limit := 5000
ch := make(chan blob.SizedRef)
errCh := make(chan error)
go func() {
errCh <- ds.EnumerateBlobs(context.New(), ch, "", limit)
}()
_, ok := <-ch
Expect(t, !ok, "no first blob")
ExpectNil(t, <-errCh, "EnumerateBlobs return value")
}
// CreateAccount creates a new importer account for the Host h, and the importer
// implementation named impl. It returns a RunContext setup with that account.
func CreateAccount(h *Host, impl string) (*RunContext, error) {
imp, ok := h.imp[impl]
if !ok {
return nil, fmt.Errorf("host does not have a %v importer", impl)
}
ia, err := imp.newAccount()
if err != nil {
return nil, fmt.Errorf("could not create new account for importer %v: %v", impl, err)
}
return &RunContext{
// TODO: context plumbing
Context: context.New(context.WithHTTPClient(ia.im.host.HTTPClient())),
Host: ia.im.host,
ia: ia,
}, nil
}
// singleBlob assumes that sto contains a single blob and returns it.
// If there are more or fewer than one blob, it's an error.
func singleBlob(sto blobserver.BlobEnumerator) (ret blob.SizedRef, err error) {
ctx := context.New()
defer ctx.Cancel()
n := 0
if err = blobserver.EnumerateAll(ctx, sto, func(sb blob.SizedRef) error {
ret = sb
n++
return nil
}); err != nil {
return blob.SizedRef{}, err
}
if n != 1 {
return blob.SizedRef{}, fmt.Errorf("saw %d blobs; want 1", n)
}
return
}
// It verifies the size and hash of each
// before returning and fails the test if any of the checks fail. It
// also fails the test if StreamBlobs returns a non-nil error.
func streamAll(t *testing.T, s *storage) []*blob.Blob {
var blobs []*blob.Blob
ctx := context.New()
ch := make(chan blobserver.BlobAndToken)
errCh := make(chan error, 1)
go func() { errCh <- s.StreamBlobs(ctx, ch, "") }()
for bt := range ch {
verifySizeAndHash(t, bt.Blob)
blobs = append(blobs, bt.Blob)
}
if err := <-errCh; err != nil {
t.Fatalf("StreamBlobs error = %v", err)
}
return blobs
}
// see if storage proxies through to small for Fetch, Stat, and Enumerate.
func TestSmallFallback(t *testing.T) {
small := new(test.Fetcher)
s := &storage{
small: small,
large: new(test.Fetcher),
meta: sorted.NewMemoryKeyValue(),
log: test.NewLogger(t, "blobpacked: "),
}
s.init()
b1 := &test.Blob{"foo"}
b1.MustUpload(t, small)
wantSB := b1.SizedRef()
// Fetch
rc, _, err := s.Fetch(b1.BlobRef())
if err != nil {
t.Errorf("failed to Get blob: %v", err)
} else {
rc.Close()
}
// Stat.
sb, err := blobserver.StatBlob(s, b1.BlobRef())
if err != nil {
t.Errorf("failed to Stat blob: %v", err)
} else if sb != wantSB {
t.Errorf("Stat = %v; want %v", sb, wantSB)
}
// Enumerate
saw := false
ctx := context.New()
defer ctx.Cancel()
if err := blobserver.EnumerateAll(ctx, s, func(sb blob.SizedRef) error {
if sb != wantSB {
return fmt.Errorf("saw blob %v; want %v", sb, wantSB)
}
saw = true
return nil
}); err != nil {
t.Errorf("EnuerateAll: %v", err)
}
if !saw {
t.Error("didn't see blob in Enumerate")
}
}
func main() {
flag.Parse()
if len(*blobDir)*len(*indexDir) == 0 {
flag.Usage()
return
}
s, err := dir.New(*blobDir)
if err != nil {
log.Fatal(err)
}
src, ok := s.(blobserver.BlobStreamer)
if !ok {
log.Fatalf("%v is not a BlobStreamer", s)
}
db, err := leveldb.NewStorage(*indexDir)
if err != nil {
log.Fatal(err)
}
defer db.Close()
dst, err := index.New(db)
if err != nil {
log.Fatal(err)
}
fe := FetcherEnumerator{
FetcherEnumerator: s,
c1: make(map[string]*blob.Blob),
start: time.Now(),
}
dst.InitBlobSource(&fe)
ch := make(chan blobserver.BlobAndToken)
go fe.PrintStats()
for i := 0; i < *parallel; i++ {
go fe.Index(ch, dst)
}
ctx := context.New()
if err := src.StreamBlobs(ctx, ch, *streamStart); err != nil {
log.Fatal(err)
}
}
// RunOnce scans a.Source and conditionally creates a new zip.
// It returns ErrSourceTooSmall if there aren't enough blobs on Source.
func (a *Archiver) RunOnce() error {
if a.Source == nil {
return errors.New("archiver: nil Source")
}
if a.Store == nil {
return errors.New("archiver: nil Store func")
}
pz := &potentialZip{a: a}
err := blobserver.EnumerateAll(context.New(), a.Source, func(sb blob.SizedRef) error {
if err := pz.addBlob(sb); err != nil {
return err
}
if pz.bigEnough() {
return errStopEnumerate
}
return nil
})
if err == errStopEnumerate {
err = nil
}
if err != nil {
return err
}
if err := pz.condClose(); err != nil {
return err
}
if !pz.bigEnough() {
return ErrSourceTooSmall
}
if err := a.Store(pz.buf.Bytes(), pz.blobs); err != nil {
return err
}
if a.DeleteSourceAfterStore {
blobs := make([]blob.Ref, 0, len(pz.blobs))
for _, sb := range pz.blobs {
blobs = append(blobs, sb.Ref)
}
if err := a.Source.RemoveBlobs(blobs); err != nil {
return err
}
}
return nil
}
func streamBlobs(path, resume string) <-chan blobserver.BlobAndToken {
s, err := dir.New(path)
if err != nil {
log.Fatal(err)
}
bs, ok := s.(blobserver.BlobStreamer)
if !ok {
log.Fatalf("%v is not a BlobStreamer", s)
}
ch := make(chan blobserver.BlobAndToken, 10)
go func() {
ctx := context.New()
if err := bs.StreamBlobs(ctx, ch, resume); err != nil {
log.Fatal(err)
}
}()
return ch
}
func TestGetUserID(t *testing.T) {
ctx := context.New(context.WithHTTPClient(&http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
apiURL + userInfoAPIPath: httputil.FileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
}),
}))
defer ctx.Cancel()
inf, err := getUserInfo(importer.OAuthContext{ctx, &oauth.Client{}, &oauth.Credentials{}})
if err != nil {
t.Fatal(err)
}
want := userInfo{
ID: "2325935334",
ScreenName: "lejatorn",
Name: "Mathieu Lonjaret",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func TestGetUserId(t *testing.T) {
im := &imp{}
ctx := context.New(context.WithHTTPClient(&http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": httputil.FileResponder("testdata/users-me-res.json"),
}),
}))
defer ctx.Cancel()
inf, err := im.getUserInfo(ctx, "footoken")
if err != nil {
t.Fatal(err)
}
want := user{
Id: "13674",
FirstName: "Brad",
LastName: "Fitzpatrick",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func TestStreamBlobs(t *testing.T) {
small := new(test.Fetcher)
s := &storage{
small: small,
large: new(test.Fetcher),
meta: sorted.NewMemoryKeyValue(),
log: test.NewLogger(t, "blobpacked: "),
}
s.init()
all := map[blob.Ref]bool{}
const nBlobs = 10
for i := 0; i < nBlobs; i++ {
b := &test.Blob{strconv.Itoa(i)}
b.MustUpload(t, small)
all[b.BlobRef()] = true
}
ctx := context.New()
defer ctx.Cancel()
token := "" // beginning
got := map[blob.Ref]bool{}
dest := make(chan blobserver.BlobAndToken, 16)
done := make(chan bool)
go func() {
defer close(done)
for bt := range dest {
got[bt.Blob.Ref()] = true
}
}()
err := s.StreamBlobs(ctx, dest, token)
if err != nil {
t.Fatalf("StreamBlobs = %v", err)
}
<-done
if !reflect.DeepEqual(got, all) {
t.Errorf("Got blobs %v; want %v", got, all)
}
storagetest.TestStreamer(t, s, storagetest.WantN(nBlobs))
}
func TestCollector(t *testing.T) {
for _, tt := range collectTests {
if tt.name == "" {
panic("no name in test")
}
w := newWorldSet(tt.world)
c := &Collector{
World: testWorld{},
Marker: testMarker(map[Item]bool{}),
Roots: testEnum(tt.roots),
Sweeper: testEnum(tt.world),
ItemEnumerator: testItemEnum(tt.graph),
Deleter: w,
}
if err := c.Collect(context.New()); err != nil {
t.Errorf("%s: Collect = %v", tt.name, err)
}
got := w.items()
if !reflect.DeepEqual(tt.wantWorld, got) {
t.Errorf("%s: world = %q; want %q", tt.name, got, tt.wantWorld)
}
}
}
func TestGetUserId(t *testing.T) {
userID := "11047045264"
responder := httputil.FileResponder("testdata/users-me-res.xml")
ctx := context.New(context.WithHTTPClient(&http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
"https://picasaweb.google.com/data/feed/api/user/default/contacts?kind=user": responder,
"https://picasaweb.google.com/data/feed/api/user/" + userID + "/contacts?kind=user": responder,
}),
}))
defer ctx.Cancel()
inf, err := picago.GetUser(ctx.HTTPClient(), "default")
if err != nil {
t.Fatal(err)
}
want := picago.User{
ID: userID,
URI: "https://picasaweb.google.com/" + userID,
Name: "Tamás Gulácsi",
Thumbnail: "https://lh4.googleusercontent.com/-qqove344/AAAAAAAAAAI/AAAAAAABcbg/TXl3f2K9dzI/s64-c/11047045264.jpg",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func TestForeachZipBlob(t *testing.T) {
const fileSize = 2 << 20
const fileName = "foo.dat"
fileContents := randBytes(fileSize)
ctx := context.New()
defer ctx.Cancel()
pt := testPack(t,
func(sto blobserver.Storage) error {
_, err := schema.WriteFileFromReader(sto, fileName, bytes.NewReader(fileContents))
return err
},
wantNumLargeBlobs(1),
wantNumSmallBlobs(0),
)
zipBlob, err := singleBlob(pt.large)
if err != nil {
t.Fatal(err)
}
zipBytes := slurpBlob(t, pt.large, zipBlob.Ref)
zipSize := len(zipBytes)
all := map[blob.Ref]blob.SizedRef{}
if err := blobserver.EnumerateAll(ctx, pt.logical, func(sb blob.SizedRef) error {
all[sb.Ref] = sb
return nil
}); err != nil {
t.Fatal(err)
}
foreachSaw := 0
blobSizeSum := 0
if err := pt.sto.foreachZipBlob(zipBlob.Ref, func(bap BlobAndPos) error {
foreachSaw++
blobSizeSum += int(bap.Size)
want, ok := all[bap.Ref]
if !ok {
t.Errorf("unwanted blob ref returned from foreachZipBlob: %v", bap.Ref)
return nil
}
delete(all, bap.Ref)
if want.Size != bap.Size {
t.Errorf("for %v, foreachZipBlob size = %d; want %d", bap.Ref, bap.Size, want.Size)
return nil
}
// Verify the offset.
h := bap.Ref.Hash()
h.Write(zipBytes[bap.Offset : bap.Offset+int64(bap.Size)])
if !bap.Ref.HashMatches(h) {
return fmt.Errorf("foreachZipBlob returned blob %v at offset %d that failed validation", bap.Ref, bap.Offset)
}
return nil
}); err != nil {
t.Fatal(err)
}
t.Logf("foreachZipBlob enumerated %d blobs", foreachSaw)
if len(all) > 0 {
t.Errorf("foreachZipBlob forgot to enumerate %d blobs: %v", len(all), all)
}
// Calculate per-blobref zip overhead (zip file headers/TOC/manifest file, etc)
zipOverhead := zipSize - blobSizeSum
t.Logf("zip fixed overhead = %d bytes, for %d blobs (%d bytes each)", zipOverhead, foreachSaw, zipOverhead/foreachSaw)
}
// TestStreamer tests that the BlobStreamer bs implements all of the
// promised interface behavior and ultimately yields the provided
// blobs.
//
// If bs also implements BlobEnumerator, the two are compared for
// consistency.
func TestStreamer(t *testing.T, bs blobserver.BlobStreamer, opts ...StreamerTestOpt) {
var sawEnum map[blob.SizedRef]bool
if enumer, ok := bs.(blobserver.BlobEnumerator); ok {
sawEnum = make(map[blob.SizedRef]bool)
// First do an enumerate over all blobs as a baseline. The Streamer should
// yield the same blobs, even if it's in a different order.
enumCtx := context.New()
defer enumCtx.Cancel()
if err := blobserver.EnumerateAll(enumCtx, enumer, func(sb blob.SizedRef) error {
sawEnum[sb] = true
return nil
}); err != nil {
t.Fatalf("Enumerate: %v", err)
}
}
// See if, without cancelation, it yields the right
// result and without errors.
ch := make(chan blobserver.BlobAndToken)
errCh := make(chan error, 1)
go func() {
ctx := context.New()
defer ctx.Cancel()
errCh <- bs.StreamBlobs(ctx, ch, "")
}()
var gotRefs []blob.SizedRef
sawStreamed := map[blob.Ref]int{}
for b := range ch {
sawStreamed[b.Ref()]++
sbr := b.SizedRef()
if sawEnum != nil {
if _, ok := sawEnum[sbr]; ok {
delete(sawEnum, sbr)
} else {
t.Errorf("Streamer yielded blob not returned by Enumerate: %v", sbr)
}
}
gotRefs = append(gotRefs, sbr)
}
if err := <-errCh; err != nil {
t.Errorf("initial uninterrupted StreamBlobs error: %v", err)
}
for br, n := range sawStreamed {
if n > 1 {
t.Errorf("Streamed returned duplicate %v, %d times", br, n)
}
}
nMissing := 0
for sbr := range sawEnum {
t.Errorf("Enumerate found %v but Streamer didn't return it", sbr)
nMissing++
if nMissing == 10 && len(sawEnum) > 10 {
t.Errorf("... etc ...")
break
}
}
for _, opt := range opts {
if err := opt.verify(gotRefs); err != nil {
t.Errorf("error after first uninterrupted StreamBlobs pass: %v", err)
}
}
if t.Failed() {
return
}
// Next, the "complex pass": test a cancelation at each point,
// to test that resume works properly.
//
// Basic strategy:
// -- receive 1 blob, note the blobref, cancel.
// -- start again with that blobref, receive 2, cancel. first should be same,
// second should be new. note its blobref.
// Each iteration should yield 1 new unique blob and all but
// the first and last will return 2 blobs.
wantRefs := append([]blob.SizedRef(nil), gotRefs...) // copy
sawStreamed = map[blob.Ref]int{}
gotRefs = gotRefs[:0]
contToken := ""
for i := 0; i < len(wantRefs); i++ {
ctx := context.New()
ch := make(chan blobserver.BlobAndToken)
errc := make(chan error, 1)
go func() {
errc <- bs.StreamBlobs(ctx, ch, contToken)
}()
nrecv := 0
nextToken := ""
for bt := range ch {
nrecv++
sbr := bt.Blob.SizedRef()
isNew := len(gotRefs) == 0 || sbr != gotRefs[len(gotRefs)-1]
if isNew {
if sawStreamed[sbr.Ref] > 0 {
t.Fatalf("In complex pass, returned duplicate blob %v\n\nSo far, before interrupting:\n%v\n\nWant:\n%v", sbr, gotRefs, wantRefs)
}
sawStreamed[sbr.Ref]++
gotRefs = append(gotRefs, sbr)
nextToken = bt.Token
ctx.Cancel()
break
} else if i == 0 {
t.Fatalf("first iteration should receive a new value")
} else if nrecv == 2 {
t.Fatalf("at cut point %d of testStream, Streamer received 2 values, both not unique. Looping?", i)
}
}
err := <-errc
if err != nil && err != context.ErrCanceled {
t.Fatalf("StreamBlobs on iteration %d (token %q) returned error: %v", i, contToken, err)
}
if err == nil {
break
}
contToken = nextToken
}
if !reflect.DeepEqual(gotRefs, wantRefs) {
t.Errorf("Mismatch on complex pass (got %d, want %d):\n got %q\nwant %q\n", len(gotRefs), len(wantRefs), gotRefs, wantRefs)
wantMap := map[blob.SizedRef]bool{}
for _, sbr := range wantRefs {
wantMap[sbr] = true
}
for _, sbr := range gotRefs {
if _, ok := wantMap[sbr]; ok {
delete(wantMap, sbr)
} else {
t.Errorf("got has unwanted: %v", sbr)
}
}
missing := wantMap // found stuff has been deleted
for sbr := range missing {
t.Errorf("got is missing: %v", sbr)
}
t.FailNow()
}
}
func CheckEnumerate(sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) error {
var after string
var n = 1000
for _, opt := range opts {
switch v := opt.(type) {
case string:
after = v
case int:
n = v
default:
panic("bad option of type " + fmt.Sprintf("%T", v))
}
}
want := append([]blob.SizedRef(nil), wantUnsorted...)
sort.Sort(blob.SizedByRef(want))
sbc := make(chan blob.SizedRef, 10)
var got []blob.SizedRef
var grp syncutil.Group
sawEnd := make(chan bool, 1)
grp.Go(func() error {
ctx := context.New()
defer ctx.Cancel()
if err := sto.EnumerateBlobs(ctx, sbc, after, n); err != nil {
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n, err)
}
return nil
})
grp.Go(func() error {
var lastRef blob.Ref
for sb := range sbc {
if !sb.Valid() {
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
}
got = append(got, sb)
if lastRef.Valid() && sb.Ref.Less(lastRef) {
return fmt.Errorf("blobs appearing out of order")
}
lastRef = sb.Ref
}
sawEnd <- true
return nil
})
grp.Go(func() error {
select {
case <-sawEnd:
return nil
case <-time.After(10 * time.Second):
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
}
})
if err := grp.Err(); err != nil {
return fmt.Errorf("Enumerate error: %v", err)
}
if len(got) == 0 && len(want) == 0 {
return nil
}
var gotSet = map[blob.SizedRef]bool{}
for _, sb := range got {
if gotSet[sb] {
return fmt.Errorf("duplicate blob %v returned in enumerate", sb)
}
gotSet[sb] = true
}
if !reflect.DeepEqual(got, want) {
return fmt.Errorf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
len(got), len(want), got, want)
}
return nil
}
func TestRemoveBlobs(t *testing.T) {
ctx := context.New()
defer ctx.Cancel()
// The basic small cases are handled via storagetest in TestStorage,
// so this only tests removing packed blobs.
small := new(test.Fetcher)
large := new(test.Fetcher)
sto := &storage{
small: small,
large: large,
meta: sorted.NewMemoryKeyValue(),
log: test.NewLogger(t, "blobpacked: "),
}
sto.init()
const fileSize = 1 << 20
fileContents := randBytes(fileSize)
if _, err := schema.WriteFileFromReader(sto, "foo.dat", bytes.NewReader(fileContents)); err != nil {
t.Fatal(err)
}
if small.NumBlobs() != 0 || large.NumBlobs() == 0 {
t.Fatalf("small, large counts == %d, %d; want 0, non-zero", small.NumBlobs(), large.NumBlobs())
}
var all []blob.SizedRef
if err := blobserver.EnumerateAll(ctx, sto, func(sb blob.SizedRef) error {
all = append(all, sb)
return nil
}); err != nil {
t.Fatal(err)
}
// Find the zip
zipBlob, err := singleBlob(sto.large)
if err != nil {
t.Fatalf("failed to find packed zip: %v", err)
}
// The zip file is in use, so verify we can't delete it.
if err := sto.deleteZipPack(zipBlob.Ref); err == nil {
t.Fatalf("zip pack blob deleted but it should not have been allowed")
}
// Delete everything
for len(all) > 0 {
del := all[0].Ref
all = all[1:]
if err := sto.RemoveBlobs([]blob.Ref{del}); err != nil {
t.Fatalf("RemoveBlobs: %v", err)
}
if err := storagetest.CheckEnumerate(sto, all); err != nil {
t.Fatalf("After deleting %v, %v", del, err)
}
}
dRows := func() (n int) {
if err := sorted.ForeachInRange(sto.meta, "d:", "", func(key, value string) error {
if strings.HasPrefix(key, "d:") {
n++
}
return nil
}); err != nil {
t.Fatalf("meta iteration error: %v", err)
}
return
}
if n := dRows(); n == 0 {
t.Fatalf("expected a 'd:' row after deletes")
}
// TODO: test the background pack-deleter loop? figure out its design first.
if err := sto.deleteZipPack(zipBlob.Ref); err != nil {
t.Errorf("error deleting zip %v: %v", zipBlob.Ref, err)
}
if n := dRows(); n != 0 {
t.Errorf("expected the 'd:' row to be deleted")
}
}
func testStorage(t *testing.T, bucketDir string) {
if *bucket == "" && *configFile == "" {
t.Skip("Skipping test without --bucket or --config flag")
}
var refreshToken string
if *configFile != "" {
data, err := ioutil.ReadFile(*configFile)
if err != nil {
t.Fatalf("Error reading config file %v: %v", *configFile, err)
}
var conf Config
if err := json.Unmarshal(data, &conf); err != nil {
t.Fatalf("Error decoding config file %v: %v", *configFile, err)
}
*clientID = conf.Auth.ClientID
*clientSecret = conf.Auth.ClientSecret
refreshToken = conf.Auth.RefreshToken
*bucket = conf.Bucket
}
if *bucket == "" {
t.Fatal("bucket not provided in config file or as a flag.")
}
if *clientID == "" || *clientSecret == "" {
t.Fatal("client ID and client secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
}
if *configFile == "" {
config := &oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: *clientID,
ClientSecret: *clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
token, err := oauth2.ReuseTokenSource(nil,
&oauthutil.TokenSource{
Config: config,
CacheFile: *tokenCache,
AuthCode: func() string {
if *authCode == "" {
t.Skipf("Re-run using --auth_code= with the value obtained from %s",
config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return ""
}
return *authCode
},
}).Token()
if err != nil {
t.Fatalf("could not acquire token: %v", err)
}
refreshToken = token.RefreshToken
}
bucketWithDir := path.Join(*bucket, bucketDir)
storagetest.TestOpt(t, storagetest.Opts{
New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
sto, err := newFromConfig(nil, jsonconfig.Obj{
"bucket": bucketWithDir,
"auth": map[string]interface{}{
"client_id": *clientID,
"client_secret": *clientSecret,
"refresh_token": refreshToken,
},
})
if err != nil {
t.Fatal(err)
}
if !testing.Short() {
log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
}
// Bail if bucket is not empty
objs, err := sto.(*Storage).client.EnumerateObjects(*bucket, "", 1)
if err != nil {
t.Fatalf("Error checking if bucket is empty: %v", err)
}
if len(objs) != 0 {
t.Fatalf("Refusing to run test: bucket %v is not empty", *bucket)
}
if bucketWithDir != *bucket {
// Adding "a", and "c" objects in the bucket to make sure objects out of the
// "directory" are not touched and have no influence.
for _, key := range []string{"a", "c"} {
err := sto.(*Storage).client.PutObject(
&googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key},
strings.NewReader(key))
if err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}
}
}
clearBucket := func(beforeTests bool) func() {
return func() {
var all []blob.Ref
blobserver.EnumerateAll(context.New(), sto, func(sb blob.SizedRef) error {
t.Logf("Deleting: %v", sb.Ref)
all = append(all, sb.Ref)
return nil
})
if err := sto.RemoveBlobs(all); err != nil {
t.Fatalf("Error removing blobs during cleanup: %v", err)
}
if beforeTests {
return
}
if bucketWithDir != *bucket {
// checking that "a" and "c" at the root were left untouched.
for _, key := range []string{"a", "c"} {
if _, _, err := sto.(*Storage).client.GetObject(&googlestorage.Object{Bucket: sto.(*Storage).bucket,
Key: key}); err != nil {
t.Fatalf("could not find object %s after tests: %v", key, err)
}
if err := sto.(*Storage).client.DeleteObject(&googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key}); err != nil {
t.Fatalf("could not remove object %s after tests: %v", key, err)
}
}
}
}
}
clearBucket(true)()
return sto, clearBucket(false)
},
})
}
func newGeocodeContext() *context.Context {
url := "https://maps.googleapis.com/maps/api/geocode/json?address=Uitdam&sensor=false"
transport := httputil.NewFakeTransport(map[string]func() *http.Response{url: httputil.StaticResponder(uitdamGoogle)})
return context.New(context.WithHTTPClient(&http.Client{Transport: transport}))
}