func runSqlWithRetry(t *testing.T, db *gosql.DB, sql string) { for { _, err := db.Exec(sql) if err == nil { break } else if !testutils.IsSQLRetryError(err) { t.Fatal(err) } t.Logf("retry %s on err: %s", sql, err) } }
// queryRowScan performs first a QueryRow and follows that up with a Scan using // a preexisting or new connection. func (dc *dynamicClient) queryRowScan(query string, queryArgs, destArgs []interface{}) error { for dc.isRunning() { client, err := dc.getClient() if err != nil { return err } if err := client.QueryRow( query, queryArgs..., ).Scan(destArgs...); err == nil || !testutils.IsSQLRetryError(err) { return err } } return errTestFinished }
// exec calls exec on a client using a preexisting or new connection. func (dc *dynamicClient) exec(query string, args ...interface{}) (gosql.Result, error) { for dc.isRunning() { client, err := dc.getClient() if err != nil { return nil, err } if result, err := client.Exec( query, args..., ); err == nil || !testutils.IsSQLRetryError(err) { return result, err } } return nil, errTestFinished }
// Continuously transfers money until done(). func transferMoneyLoop(idx int, state *testState, numAccounts, maxTransfer int) { client := &state.clients[idx] for !state.done() { if err := transferMoney(client, numAccounts, maxTransfer); err != nil { // Ignore some errors. if !testutils.IsSQLRetryError(err) { // Report the err and terminate. state.errChan <- err break } } else { // Only advance the counts on a successful update. atomic.AddUint64(&client.count, 1) } } log.Infof(context.Background(), "client %d shutting down", idx) state.errChan <- nil }
// Verify accounts. func verifyAccounts(t *testing.T, client *testClient) { var sum int util.SucceedsSoon(t, func() error { // Hold the read lock on the client to prevent it being restarted by // chaos monkey. client.RLock() defer client.RUnlock() err := client.db.QueryRow("SELECT SUM(balance) FROM bank.accounts").Scan(&sum) if err != nil && !testutils.IsSQLRetryError(err) { t.Fatal(err) } return err }) if sum != 0 { t.Fatalf("The bank is not in good order. Total value: %d", sum) } }