func runSqlWithRetry(t *testing.T, db *gosql.DB, sql string) {
for {
_, err := db.Exec(sql)
if err == nil {
break
} else if !testutils.IsSQLRetryError(err) {
t.Fatal(err)
}
t.Logf("retry %s on err: %s", sql, err)
}
}
// queryRowScan performs first a QueryRow and follows that up with a Scan using
// a preexisting or new connection.
func (dc *dynamicClient) queryRowScan(query string, queryArgs, destArgs []interface{}) error {
for dc.isRunning() {
client, err := dc.getClient()
if err != nil {
return err
}
if err := client.QueryRow(
query, queryArgs...,
).Scan(destArgs...); err == nil || !testutils.IsSQLRetryError(err) {
return err
}
}
return errTestFinished
}
// exec calls exec on a client using a preexisting or new connection.
func (dc *dynamicClient) exec(query string, args ...interface{}) (gosql.Result, error) {
for dc.isRunning() {
client, err := dc.getClient()
if err != nil {
return nil, err
}
if result, err := client.Exec(
query, args...,
); err == nil || !testutils.IsSQLRetryError(err) {
return result, err
}
}
return nil, errTestFinished
}
// Continuously transfers money until done().
func transferMoneyLoop(idx int, state *testState, numAccounts, maxTransfer int) {
client := &state.clients[idx]
for !state.done() {
if err := transferMoney(client, numAccounts, maxTransfer); err != nil {
// Ignore some errors.
if !testutils.IsSQLRetryError(err) {
// Report the err and terminate.
state.errChan <- err
break
}
} else {
// Only advance the counts on a successful update.
atomic.AddUint64(&client.count, 1)
}
}
log.Infof(context.Background(), "client %d shutting down", idx)
state.errChan <- nil
}
// Verify accounts.
func verifyAccounts(t *testing.T, client *testClient) {
var sum int
util.SucceedsSoon(t, func() error {
// Hold the read lock on the client to prevent it being restarted by
// chaos monkey.
client.RLock()
defer client.RUnlock()
err := client.db.QueryRow("SELECT SUM(balance) FROM bank.accounts").Scan(&sum)
if err != nil && !testutils.IsSQLRetryError(err) {
t.Fatal(err)
}
return err
})
if sum != 0 {
t.Fatalf("The bank is not in good order. Total value: %d", sum)
}
}