// LoadTLSConfig creates a TLSConfig by loading our keys and certs from the
// specified directory. The directory must contain the following files:
// - ca.crt -- the certificate of the cluster CA
// - node.crt -- the certificate of this node; should be signed by the CA
// - node.key -- the private key of this node
func LoadTLSConfig(certDir string) (*TLSConfig, error) {
cert, err := tls.LoadX509KeyPair(
path.Join(certDir, "node.crt"),
path.Join(certDir, "node.key"),
)
if err != nil {
log.Info(err)
return nil, err
}
certPool := x509.NewCertPool()
pemData, err := ioutil.ReadFile(path.Join(certDir, "ca.crt"))
if err != nil {
log.Info(err)
return nil, err
}
if ok := certPool.AppendCertsFromPEM(pemData); !ok {
err = errors.New("failed to parse PEM data to pool")
log.Info(err)
return nil, err
}
return &TLSConfig{
config: &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
RootCAs: certPool,
ClientCAs: certPool,
// TODO(jqmp): Set CipherSuites?
// TODO(jqmp): Set MinVersion?
},
}, nil
}
// GetJSON retrieves the URL specified by https://Addr(<port>)<path>
// and unmarshals the result as JSON.
func (c *Container) GetJSON(port, path string, v interface{}) error {
client := &http.Client{
Timeout: 200 * time.Millisecond,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}}
resp, err := client.Get(fmt.Sprintf("https://%s%s", c.Addr(port), path))
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
if err := json.Unmarshal(b, v); err != nil {
if log.V(1) {
log.Info(err)
}
}
return nil
}
// getJSON retrieves the URL specified by the parameters and
// and unmarshals the result into the supplied interface.
func getJSON(tls bool, hostport, path string, v interface{}) error {
protocol := "https"
if !tls {
protocol = "http"
}
resp, err := HTTPClient.Get(fmt.Sprintf("%s://%s%s", protocol, hostport, path))
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
if err := json.Unmarshal(b, v); err != nil {
if log.V(1) {
log.Info(err)
}
}
return nil
}
func (at *allocatorTest) Run(t *testing.T) {
at.f = farmer(t, at.Prefix)
if at.CockroachDiskSizeGB != 0 {
at.f.AddVars["cockroach_disk_size"] = strconv.Itoa(at.CockroachDiskSizeGB)
}
log.Infof(context.Background(), "creating cluster with %d node(s)", at.StartNodes)
if err := at.f.Resize(at.StartNodes); err != nil {
t.Fatal(err)
}
checkGossip(t, at.f, longWaitTime, hasPeers(at.StartNodes))
at.f.Assert(t)
log.Info(context.Background(), "initial cluster is up")
// We must stop the cluster because a) `nodectl` pokes at the data directory
// and, more importantly, b) we don't want the cluster above and the cluster
// below to ever talk to each other (see #7224).
log.Info(context.Background(), "stopping cluster")
for i := 0; i < at.f.NumNodes(); i++ {
if err := at.f.Kill(i); err != nil {
t.Fatalf("error stopping node %d: %s", i, err)
}
}
log.Info(context.Background(), "downloading archived stores from Google Cloud Storage in parallel")
errors := make(chan error, at.f.NumNodes())
for i := 0; i < at.f.NumNodes(); i++ {
go func(nodeNum int) {
errors <- at.f.Exec(nodeNum, "./nodectl download "+at.StoreURL)
}(i)
}
for i := 0; i < at.f.NumNodes(); i++ {
if err := <-errors; err != nil {
t.Fatalf("error downloading store %d: %s", i, err)
}
}
log.Info(context.Background(), "restarting cluster with archived store(s)")
for i := 0; i < at.f.NumNodes(); i++ {
if err := at.f.Restart(i); err != nil {
t.Fatalf("error restarting node %d: %s", i, err)
}
}
at.f.Assert(t)
log.Infof(context.Background(), "resizing cluster to %d nodes", at.EndNodes)
if err := at.f.Resize(at.EndNodes); err != nil {
t.Fatal(err)
}
checkGossip(t, at.f, longWaitTime, hasPeers(at.EndNodes))
at.f.Assert(t)
log.Info(context.Background(), "waiting for rebalance to finish")
if err := at.WaitForRebalance(t); err != nil {
t.Fatal(err)
}
at.f.Assert(t)
}
// RefreshLeases starts a goroutine that refreshes the lease manager
// leases for tables received in the latest system configuration via gossip.
func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) {
s.RunWorker(func() {
descKeyPrefix := keys.MakeTablePrefix(uint32(sqlbase.DescriptorTable.ID))
gossipUpdateC := gossip.RegisterSystemConfigChannel()
for {
select {
case <-gossipUpdateC:
cfg, _ := gossip.GetSystemConfig()
if m.testingKnobs.GossipUpdateEvent != nil {
m.testingKnobs.GossipUpdateEvent(cfg)
}
// Read all tables and their versions
if log.V(2) {
log.Info("received a new config; will refresh leases")
}
// Loop through the configuration to find all the tables.
for _, kv := range cfg.Values {
if !bytes.HasPrefix(kv.Key, descKeyPrefix) {
continue
}
// Attempt to unmarshal config into a table/database descriptor.
var descriptor sqlbase.Descriptor
if err := kv.Value.GetProto(&descriptor); err != nil {
log.Warningf("%s: unable to unmarshal descriptor %v", kv.Key, kv.Value)
continue
}
switch union := descriptor.Union.(type) {
case *sqlbase.Descriptor_Table:
table := union.Table
if err := table.Validate(); err != nil {
log.Errorf("%s: received invalid table descriptor: %v", kv.Key, table)
continue
}
if log.V(2) {
log.Infof("%s: refreshing lease table: %d (%s), version: %d",
kv.Key, table.ID, table.Name, table.Version)
}
// Try to refresh the table lease to one >= this version.
if t := m.findTableState(table.ID, false /* create */, nil); t != nil {
if err := t.purgeOldLeases(
db, table.Deleted(), table.Version, m.LeaseStore); err != nil {
log.Warningf("error purging leases for table %d(%s): %s",
table.ID, table.Name, err)
}
}
case *sqlbase.Descriptor_Database:
// Ignore.
}
}
if m.testingKnobs.TestingLeasesRefreshedEvent != nil {
m.testingKnobs.TestingLeasesRefreshedEvent(cfg)
}
case <-s.ShouldStop():
return
}
}
})
}
// TestClientGossip verifies a client can gossip a delta to the server.
func TestClientGossip(t *testing.T) {
local, remote, lserver, rserver := startGossip(t)
local.AddInfo("local-key", "local value", time.Second)
remote.AddInfo("remote-key", "remote value", time.Second)
disconnected := make(chan *client, 1)
client := newClient(remote.is.NodeAddr)
go client.start(local, disconnected)
if err := util.IsTrueWithin(func() bool {
_, lerr := remote.GetInfo("local-key")
_, rerr := local.GetInfo("remote-key")
return lerr == nil && rerr == nil
}, 500*time.Millisecond); err != nil {
t.Errorf("gossip exchange failed or taking too long")
}
remote.stop()
local.stop()
lserver.Close()
rserver.Close()
log.Info("done serving")
if client != <-disconnected {
t.Errorf("expected client disconnect after remote close")
}
}
// TestClientGossip verifies a client can gossip a delta to the server.
func TestClientGossip(t *testing.T) {
defer leaktest.AfterTest(t)
local, remote, stopper := startGossip(t)
if err := local.AddInfo("local-key", "local value", time.Second); err != nil {
t.Fatal(err)
}
if err := remote.AddInfo("remote-key", "remote value", time.Second); err != nil {
t.Fatal(err)
}
disconnected := make(chan *client, 1)
client := newClient(remote.is.NodeAddr)
// Use an insecure context. We're talking to unix socket which are not in the certs.
lclock := hlc.NewClock(hlc.UnixNano)
rpcContext := rpc.NewContext(insecureTestBaseContext, lclock, stopper)
client.start(local, disconnected, rpcContext, stopper)
if err := util.IsTrueWithin(func() bool {
_, lerr := remote.GetInfo("local-key")
_, rerr := local.GetInfo("remote-key")
return lerr == nil && rerr == nil
}, 500*time.Millisecond); err != nil {
t.Errorf("gossip exchange failed or taking too long")
}
stopper.Stop()
log.Info("done serving")
if client != <-disconnected {
t.Errorf("expected client disconnect after remote close")
}
}
func verifyBank(db *sql.DB) {
var sum int64
if *aggregate {
if err := db.QueryRow("SELECT SUM(balance) FROM accounts").Scan(&sum); err != nil {
log.Fatal(err)
}
} else {
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
rows, err := tx.Query("SELECT balance FROM accounts")
if err != nil {
log.Fatal(err)
}
for rows.Next() {
var balance int64
if err = rows.Scan(&balance); err != nil {
log.Fatal(err)
}
sum += balance
}
if err = tx.Commit(); err != nil {
log.Fatal(err)
}
}
if sum == 0 {
log.Info("The bank is in good order.")
} else {
log.Fatalf("The bank is not in good order. Total value: %d", sum)
}
}
// runStart starts the cockroach node using -stores as the list of
// storage devices ("stores") on this machine and -gossip as the list
// of "well-known" hosts used to join this node to the cockroach
// cluster via the gossip network.
func runStart(cmd *commander.Command, args []string) {
log.Info("Starting cockroach cluster")
s, err := newServer()
if err != nil {
log.Errorf("Failed to start Cockroach server: %v", err)
return
}
// Init engines from -stores.
engines, err := initEngines(*stores)
if err != nil {
log.Errorf("Failed to initialize engines from -stores=%q: %v", *stores, err)
return
}
if len(engines) == 0 {
log.Errorf("No valid engines specified after initializing from -stores=%q", *stores)
return
}
err = s.start(engines, false)
defer s.stop()
if err != nil {
log.Errorf("Cockroach server exited with error: %v", err)
return
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
// Block until one of the signals above is received.
<-c
}
// runStart starts the cockroach node using --stores as the list of
// storage devices ("stores") on this machine and --gossip as the list
// of "well-known" hosts used to join this node to the cockroach
// cluster via the gossip network.
func runStart(cmd *cobra.Command, args []string) {
info := util.GetBuildInfo()
log.Infof("build Vers: %s", info.Vers)
log.Infof("build Tag: %s", info.Tag)
log.Infof("build Time: %s", info.Time)
log.Infof("build Deps: %s", info.Deps)
// Default user for servers.
Context.User = security.NodeUser
// First initialize the Context as it is used in other places.
err := Context.Init("start")
if err != nil {
log.Errorf("failed to initialize context: %s", err)
return
}
log.Info("starting cockroach cluster")
stopper := util.NewStopper()
stopper.AddWorker()
s, err := server.NewServer(Context, stopper)
if err != nil {
log.Errorf("failed to start Cockroach server: %s", err)
return
}
err = s.Start(false)
if err != nil {
log.Errorf("cockroach server exited with error: %s", err)
return
}
signalCh := make(chan os.Signal, 1)
signal.Notify(signalCh, os.Interrupt, os.Kill)
// TODO(spencer): move this behind a build tag.
signal.Notify(signalCh, syscall.SIGTERM)
// Block until one of the signals above is received or the stopper
// is stopped externally (for example, via the quit endpoint).
select {
case <-stopper.ShouldStop():
stopper.SetStopped()
case <-signalCh:
log.Infof("initiating graceful shutdown of server")
stopper.SetStopped()
go func() {
s.Stop()
}()
}
select {
case <-signalCh:
log.Warningf("second signal received, initiating hard shutdown")
case <-time.After(time.Minute):
log.Warningf("time limit reached, initiating hard shutdown")
return
case <-stopper.IsStopped():
log.Infof("server drained and shutdown completed")
}
log.Flush()
}
// runStart starts the cockroach node using -stores as the list of
// storage devices ("stores") on this machine and -gossip as the list
// of "well-known" hosts used to join this node to the cockroach
// cluster via the gossip network.
func runStart(cmd *commander.Command, args []string) {
info := util.GetBuildInfo()
log.Infof("Build Vers: %s", info.Vers)
log.Infof("Build Tag: %s", info.Tag)
log.Infof("Build Time: %s", info.Time)
log.Infof("Build Deps: %s", info.Deps)
log.Info("Starting cockroach cluster")
s, err := server.NewServer(Context)
if err != nil {
log.Errorf("Failed to start Cockroach server: %v", err)
return
}
err = Context.Init()
if err != nil {
log.Errorf("Failed to initialize context: %v", err)
return
}
err = s.Start(false)
defer s.Stop()
if err != nil {
log.Errorf("Cockroach server exited with error: %v", err)
return
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
// Block until one of the signals above is received.
<-c
}
// NewClient returns a client RPC stub for the specified address
// (usually a TCP host:port, but for testing may be a unix domain
// socket). The process-wide client RPC cache is consulted first; if
// the requested client is not present, it's created and the cache is
// updated. Specify opts to fine tune client connection behavior or
// nil to use defaults (i.e. indefinite retries with exponential
// backoff).
//
// The Client.Ready channel is closed after the client has connected
// and completed one successful heartbeat. The Closed channel is
// closed if the client fails to connect or if the client's Close()
// method is invoked.
func NewClient(addr net.Addr, opts *util.RetryOptions) *Client {
clientMu.Lock()
if c, ok := clients[addr.String()]; ok {
clientMu.Unlock()
return c
}
c := &Client{
addr: addr,
Ready: make(chan struct{}),
Closed: make(chan struct{}),
}
clients[c.Addr().String()] = c
clientMu.Unlock()
// Attempt to dial connection.
retryOpts := clientRetryOptions
if opts != nil {
retryOpts = *opts
}
retryOpts.Tag = fmt.Sprintf("client %s connection", addr)
go func() {
err := util.RetryWithBackoff(retryOpts, func() (bool, error) {
// TODO(spencer): use crypto.tls.
conn, err := net.Dial(addr.Network(), addr.String())
if err != nil {
log.Info(err)
return false, nil
}
c.mu.Lock()
c.Client = rpc.NewClient(conn)
c.lAddr = conn.LocalAddr()
c.mu.Unlock()
// Ensure at least one heartbeat succeeds before exiting the
// retry loop.
if err = c.heartbeat(); err != nil {
c.Close()
return false, err
}
// Signal client is ready by closing Ready channel.
log.Infof("client %s connected", addr)
close(c.Ready)
// Launch periodic heartbeat.
go c.startHeartbeat()
return true, nil
})
if err != nil {
log.Errorf("client %s failed to connect", addr)
c.Close()
}
}()
return c
}
// deleteAllRows runs the kv operations necessary to delete all sql rows in the
// table passed at construction. This may require a scan.
func (td *tableDeleter) deleteAllRows(ctx context.Context) error {
if td.rd.helper.tableDesc.IsInterleaved() {
if log.V(2) {
log.Info(ctx, "delete forced to scan: table is interleaved")
}
return td.deleteAllRowsScan(ctx)
}
return td.deleteAllRowsFast(ctx)
}
func (c *sqlConn) Close() {
if c.conn != nil {
err := c.conn.Close()
if err != nil && err != driver.ErrBadConn {
log.Info(err)
}
c.conn = nil
}
}
// getJSON retrieves the URL specified by the parameters and
// and unmarshals the result into the supplied interface.
func getJSON(url, rel string, v interface{}) error {
resp, err := cluster.HTTPClient().Get(url + rel)
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
defer func() { _ = resp.Body.Close() }()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
return json.Unmarshal(b, v)
}
// deleteIndex runs the kv operations necessary to delete all kv entries in the
// given index. This may require a scan.
func (td *tableDeleter) deleteIndex(ctx context.Context, idx *sqlbase.IndexDescriptor) error {
if len(idx.Interleave.Ancestors) > 0 || len(idx.InterleavedBy) > 0 {
if log.V(2) {
log.Info(ctx, "delete forced to scan: table is interleaved")
}
return td.deleteIndexScan(ctx, idx)
}
return td.deleteIndexFast(ctx, idx)
}
// addHistory persists a line of input to the readline history
// file.
func addHistory(line string) {
// readline.AddHistory will push command into memory and try to
// persist to disk (if readline.SetHistoryPath was called). err can
// be not nil only if it got a IO error while trying to persist.
if err := readline.AddHistory(line); err != nil {
log.Warningf("cannot save command-line history: %s", err)
log.Info("command-line history will not be saved in this session")
readline.SetHistoryPath("")
}
}
func (testRecorder) RecordSpan(sp standardtracer.RawSpan) {
if log.V(2) {
var buf bytes.Buffer
fmt.Fprintf(&buf, "[%s]", sp.Operation)
for _, log := range sp.Logs {
fmt.Fprint(&buf, "\n * ", log.Timestamp.Format(traceTimeFormat), " ", log.Event)
}
log.Info(buf.String())
}
}
// isNetworkConnected returns true if the network is fully connected
// with no partitions (i.e. every node knows every other node's
// network address).
func (n *Network) isNetworkConnected() bool {
for _, leftNode := range n.Nodes {
for _, rightNode := range n.Nodes {
if _, err := leftNode.Gossip.GetInfo(rightNode.Server.Addr().String()); err != nil {
log.Info(err)
return false
}
}
}
return true
}
// RefreshLeases starts a goroutine that refreshes the lease manager
// leases for tables received in the latest system configuration via gossip.
func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) {
s.RunWorker(func() {
descKeyPrefix := keys.MakeTablePrefix(uint32(DescriptorTable.ID))
gossip.RegisterSystemConfigCallback(m.updateSystemConfig)
for {
select {
case <-m.newConfig:
// Read all tables and their versions
cfg := m.getSystemConfig()
if log.V(2) {
log.Info("received a new config %v", cfg)
}
// Loop through the configuration to find all the tables.
for _, kv := range cfg.Values {
if kv.Value.Tag != roachpb.ValueType_BYTES {
continue
}
if !bytes.HasPrefix(kv.Key, descKeyPrefix) {
continue
}
// Attempt to unmarshal config into a table/database descriptor.
var descriptor Descriptor
if err := kv.Value.GetProto(&descriptor); err != nil {
log.Warningf("unable to unmarshal descriptor %v", kv.Value)
continue
}
switch union := descriptor.Union.(type) {
case *Descriptor_Table:
table := union.Table
if err := table.Validate(); err != nil {
log.Errorf("received invalid table descriptor: %v", table)
continue
}
if log.V(2) {
log.Infof("refreshing lease table: %d, version: %d", table.ID, table.Version)
}
// Try to refresh the table lease to one >= this version.
if err := m.refreshLease(db, table.ID, table.Version); err != nil {
log.Warning(err)
}
case *Descriptor_Database:
// Ignore.
}
}
case <-s.ShouldStop():
return
}
}
})
}
// fastPathAvailable returns true if the fastDelete optimization can be used.
func (td *tableDeleter) fastPathAvailable(ctx context.Context) bool {
if len(td.rd.helper.indexes) != 0 {
if log.V(2) {
log.Infof(ctx, "delete forced to scan: values required to update %d secondary indexes", len(td.rd.helper.indexes))
}
return false
}
if td.rd.helper.tableDesc.IsInterleaved() {
if log.V(2) {
log.Info(ctx, "delete forced to scan: table is interleaved")
}
return false
}
if len(td.rd.helper.tableDesc.PrimaryIndex.ReferencedBy) > 0 {
if log.V(2) {
log.Info(ctx, "delete forced to scan: table is referenced by foreign keys")
}
return false
}
return true
}
func verifyBank(db *sql.DB) {
var sum int
if err := db.QueryRow("SELECT SUM(balance) FROM accounts").Scan(&sum); err != nil {
log.Fatal(err)
}
if sum == *numAccounts*1000 {
log.Info("The bank is in good order.")
} else {
log.Errorf("The bank is not in good order. Total value: %d", sum)
os.Exit(1)
}
}
// FindOrCreateLoadBalancer looks for the cockroach load balancer
// and creates it if it does not exist.
// Returns the external DNS name of the load balancer.
func FindOrCreateLoadBalancer(region string, cockroachPort int64, zone string,
securityGroupID string) (string, error) {
log.Infof("looking for load balancer")
dnsName, err := FindCockroachELB(region)
if err != nil {
return "", util.Errorf("failed to lookup existing load balancer: %v", err)
}
if dnsName != "" {
log.Info("found load balancer")
return dnsName, nil
}
log.Infof("no existing load balancer, creating one")
dnsName, err = CreateCockroachELB(region, cockroachPort, zone, securityGroupID)
if err != nil {
return "", util.Errorf("failed to create load balancer: %v", err)
}
log.Info("created load balancer")
return dnsName, nil
}
// addHistory persists a line of input to the readline history
// file.
func addHistory(ins *readline.Instance, line string) {
// ins.SaveHistory will push command into memory and try to
// persist to disk (if ins's config.HistoryFile is set). err can
// be not nil only if it got a IO error while trying to persist.
if err := ins.SaveHistory(line); err != nil {
log.Warningf("cannot save command-line history: %s", err)
log.Info("command-line history will not be saved in this session")
cfg := ins.Config.Clone()
cfg.HistoryFile = ""
ins.SetConfig(cfg)
}
}
// GetJSON retrieves the URL specified by https://Addr(<port>)<path>
// and unmarshals the result as JSON.
func (c *Container) GetJSON(port, path string, v interface{}) error {
resp, err := HTTPClient.Get(fmt.Sprintf("https://%s%s", c.Addr(port), path))
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
if log.V(1) {
log.Info(err)
}
return err
}
if err := json.Unmarshal(b, v); err != nil {
if log.V(1) {
log.Info(err)
}
}
return nil
}
// AfterFirstNode runs any steps needed after the first node was created.
// This tweaks the security group to allow cockroach ports and creates
// the load balancer.
func (a *Amazon) AfterFirstNode() error {
securityGroupID, err := FindSecurityGroup(a.region)
if err != nil {
return err
}
log.Info("adding security group rule")
if err := AddCockroachSecurityGroupIngress(a.region, a.context.Port, securityGroupID); err != nil {
return util.Errorf("failed to add security group rule: %v", err)
}
_, err = FindOrCreateLoadBalancer(a.region, a.context.Port, a.zone, securityGroupID)
return err
}
func runStatus(cmd *cobra.Command, args []string) {
// Check dependencies first.
if err := docker.CheckDockerMachine(); err != nil {
log.Errorf("docker-machine is not properly installed: %v", err)
return
}
log.Info("docker-machine binary found")
if err := docker.CheckDocker(); err != nil {
log.Errorf("docker is not properly installed: %v", err)
return
}
log.Info("docker binary found")
// Initialize driver: this refreshes oauth.
driver, err := NewDriver(Context)
if err != nil {
log.Errorf("could not create driver: %v", err)
return
}
// Print docker-machine status.
fmt.Println("######## docker-machine ########")
c := exec.Command("docker-machine", "ls")
c.Stdin = os.Stdin
c.Stdout = os.Stdout
c.Stderr = os.Stderr
err = c.Run()
if err != nil {
log.Error(err)
}
// Get driver-specific status.
fmt.Printf("\n######### %s ########\n", driver.DockerMachineDriver())
driver.PrintStatus()
}
// WaitForRebalance waits until there's been no recent range adds, removes, and
// splits. We wait until the cluster is balanced or until `StableInterval`
// elapses, whichever comes first. Then, it prints stats about the rebalancing
// process. If the replica count appears unbalanced, an error is returned.
//
// This method is crude but necessary. If we were to wait until range counts
// were just about even, we'd miss potential post-rebalance thrashing.
func (at *allocatorTest) WaitForRebalance(t *testing.T) error {
const statsInterval = 20 * time.Second
db, err := gosql.Open("postgres", at.f.PGUrl(0))
if err != nil {
return err
}
defer func() {
_ = db.Close()
}()
var statsTimer timeutil.Timer
var assertTimer timeutil.Timer
defer statsTimer.Stop()
defer assertTimer.Stop()
statsTimer.Reset(statsInterval)
assertTimer.Reset(0)
for {
select {
case <-statsTimer.C:
statsTimer.Read = true
stats, err := at.allocatorStats(db)
if err != nil {
return err
}
log.Info(context.Background(), stats)
if StableInterval <= stats.ElapsedSinceLastEvent {
host := at.f.Nodes()[0]
log.Infof(context.Background(), "replica count = %f, max = %f", stats.ReplicaCountStdDev, *flagATMaxStdDev)
if stats.ReplicaCountStdDev > *flagATMaxStdDev {
_ = at.printRebalanceStats(db, host)
return errors.Errorf(
"%s elapsed without changes, but replica count standard "+
"deviation is %.2f (>%.2f)", stats.ElapsedSinceLastEvent,
stats.ReplicaCountStdDev, *flagATMaxStdDev)
}
return at.printRebalanceStats(db, host)
}
statsTimer.Reset(statsInterval)
case <-assertTimer.C:
assertTimer.Read = true
at.f.Assert(t)
assertTimer.Reset(time.Minute)
case <-stopper:
return errors.New("interrupted")
}
}
}
// fastPathAvailable returns true if the fastDelete optimization can be used.
func (td *tableDeleter) fastPathAvailable() bool {
if len(td.rd.helper.indexes) != 0 {
if log.V(2) {
log.Infof("delete forced to scan: values required to update %d secondary indexes", len(td.rd.helper.indexes))
}
return false
}
if td.rd.helper.tableDesc.IsInterleaved() {
if log.V(2) {
log.Info("delete forced to scan: table is interleaved")
}
return false
}
return true
}
// LogStatus logs the current status of gossip such as the incoming and
// outgoing connections.
func (g *Gossip) LogStatus() {
g.mu.Lock()
n := len(g.nodeDescs)
status := "ok"
if g.mu.is.getInfo(KeySentinel) == nil {
status = "stalled"
}
g.mu.Unlock()
msg := fmt.Sprintf("gossip status (%s, %d node%s)\n%s%s",
status, n, util.Pluralize(int64(n)),
g.clientStatus(), g.server.status())
log.Info(g.ctx, msg)
}