// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
// See the similar function in ../storage/invoke.go. The main difference is the
// reason for retrying.
func runWithRetry(ctx context.Context, call func() error) error {
	backoff := gax.Backoff{
		Initial:    2 * time.Second,
		Max:        32 * time.Second,
		Multiplier: 2,
	}
	return internal.Retry(ctx, backoff, func() (stop bool, err error) {
		err = call()
		if err == nil {
			return true, nil
		}
		e, ok := err.(*googleapi.Error)
		if !ok {
			return true, err
		}
		var reason string
		if len(e.Errors) > 0 {
			reason = e.Errors[0].Reason
		}
		// Retry using the criteria in
		// https://cloud.google.com/bigquery/troubleshooting-errors
		if reason == "backendError" && (e.Code == 500 || e.Code == 503) {
			return false, nil
		}
		return true, err
	})
}
示例#2
0
// waitFor calls f repeatedly with exponential backoff, blocking until it returns true.
// It returns false after a while (if it times out).
func waitFor(f func() bool) bool {
	// TODO(shadams): Find a better way to deflake these tests.
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()
	err := cinternal.Retry(ctx,
		gax.Backoff{Initial: time.Second, Multiplier: 2},
		func() (bool, error) { return f(), nil })
	return err == nil
}
示例#3
0
// Wait blocks until the job or th context is done. It returns the final status
// of the job.
// If an error occurs while retrieving the status, Wait returns that error. But
// Wait returns nil if the status was retrieved successfully, even if
// status.Err() != nil. So callers must check both errors. See the example.
func (j *Job) Wait(ctx context.Context) (*JobStatus, error) {
	var js *JobStatus
	err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
		js, err = j.Status(ctx)
		if err != nil {
			return true, err
		}
		if js.Done() {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return nil, err
	}
	return js, nil
}
示例#4
0
文件: invoke.go 项目: naunga/vault
// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
func runWithRetry(ctx context.Context, call func() error) error {
	return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
		err = call()
		if err == nil {
			return true, nil
		}
		e, ok := err.(*googleapi.Error)
		if !ok {
			return true, err
		}
		// Retry on 429 and 5xx, according to
		// https://cloud.google.com/storage/docs/exponential-backoff.
		if e.Code == 429 || (e.Code >= 500 && e.Code < 600) {
			return false, nil
		}
		return true, err
	})
}
示例#5
0
func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) {
	ctx := context.Background()
	// Collect the list of items we expect: ObjectAttrs in lexical order by name.
	names := make([]string, len(objects))
	copy(names, objects)
	sort.Strings(names)
	var attrs []*ObjectAttrs
	for _, name := range names {
		attr, err := bkt.Object(name).Attrs(ctx)
		if err != nil {
			t.Errorf("Object(%q).Attrs: %v", name, err)
			return
		}
		attrs = append(attrs, attr)
	}
	// The following iterator test fails occasionally, probably because the
	// underlying Objects.List operation is eventually consistent. So we retry
	// it.
	tctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()
	var msg string
	var ok bool
	err := internal.Retry(tctx, gax.Backoff{}, func() (stop bool, err error) {
		msg, ok = itesting.TestIterator(attrs,
			func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) },
			func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
		if ok {
			return true, nil
		} else {
			t.Logf("TestIterator failed, trying again: %s", msg)
			return false, nil
		}
	})
	if !ok {
		t.Errorf("ObjectIterator.Next: %s (err=%v)", msg, err)
	}
	// TODO(jba): test query.Delimiter != ""
}