// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
// See the similar function in ../storage/invoke.go. The main difference is the
// reason for retrying.
func runWithRetry(ctx context.Context, call func() error) error {
backoff := gax.Backoff{
Initial: 2 * time.Second,
Max: 32 * time.Second,
Multiplier: 2,
}
return internal.Retry(ctx, backoff, func() (stop bool, err error) {
err = call()
if err == nil {
return true, nil
}
e, ok := err.(*googleapi.Error)
if !ok {
return true, err
}
var reason string
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
// Retry using the criteria in
// https://cloud.google.com/bigquery/troubleshooting-errors
if reason == "backendError" && (e.Code == 500 || e.Code == 503) {
return false, nil
}
return true, err
})
}
// waitFor calls f repeatedly with exponential backoff, blocking until it returns true.
// It returns false after a while (if it times out).
func waitFor(f func() bool) bool {
// TODO(shadams): Find a better way to deflake these tests.
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
err := cinternal.Retry(ctx,
gax.Backoff{Initial: time.Second, Multiplier: 2},
func() (bool, error) { return f(), nil })
return err == nil
}
// Wait blocks until the job or th context is done. It returns the final status
// of the job.
// If an error occurs while retrieving the status, Wait returns that error. But
// Wait returns nil if the status was retrieved successfully, even if
// status.Err() != nil. So callers must check both errors. See the example.
func (j *Job) Wait(ctx context.Context) (*JobStatus, error) {
var js *JobStatus
err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
js, err = j.Status(ctx)
if err != nil {
return true, err
}
if js.Done() {
return true, nil
}
return false, nil
})
if err != nil {
return nil, err
}
return js, nil
}
// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
func runWithRetry(ctx context.Context, call func() error) error {
return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
err = call()
if err == nil {
return true, nil
}
e, ok := err.(*googleapi.Error)
if !ok {
return true, err
}
// Retry on 429 and 5xx, according to
// https://cloud.google.com/storage/docs/exponential-backoff.
if e.Code == 429 || (e.Code >= 500 && e.Code < 600) {
return false, nil
}
return true, err
})
}
func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) {
ctx := context.Background()
// Collect the list of items we expect: ObjectAttrs in lexical order by name.
names := make([]string, len(objects))
copy(names, objects)
sort.Strings(names)
var attrs []*ObjectAttrs
for _, name := range names {
attr, err := bkt.Object(name).Attrs(ctx)
if err != nil {
t.Errorf("Object(%q).Attrs: %v", name, err)
return
}
attrs = append(attrs, attr)
}
// The following iterator test fails occasionally, probably because the
// underlying Objects.List operation is eventually consistent. So we retry
// it.
tctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
var msg string
var ok bool
err := internal.Retry(tctx, gax.Backoff{}, func() (stop bool, err error) {
msg, ok = itesting.TestIterator(attrs,
func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) },
func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
if ok {
return true, nil
} else {
t.Logf("TestIterator failed, trying again: %s", msg)
return false, nil
}
})
if !ok {
t.Errorf("ObjectIterator.Next: %s (err=%v)", msg, err)
}
// TODO(jba): test query.Delimiter != ""
}