// TestGossipRestart verifies that the gossip network can be
// re-bootstrapped after a time when all nodes were down
// simultaneously.
func TestGossipRestart(t *testing.T) {
// This already replicates the first range (in the local setup).
// The replication of the first range is important: as long as the
// first range only exists on one node, that node can trivially
// acquire the leader lease. Once the range is replicated, however,
// nodes must be able to discover each other over gossip before the
// lease can be acquired.
c := StartCluster(t)
defer c.AssertAndStop(t)
num := c.NumNodes()
log.Infof("waiting for initial gossip connections")
checkGossip(t, c, 20*time.Second, hasPeers(num))
checkGossip(t, c, time.Second, hasClusterID)
checkGossip(t, c, time.Second, hasSentinel)
log.Infof("killing all nodes")
for i := 0; i < num; i++ {
if err := c.Kill(i); err != nil {
t.Fatal(err)
}
}
log.Infof("restarting all nodes")
for i := 0; i < num; i++ {
if err := c.Restart(i); err != nil {
t.Fatal(err)
}
}
log.Infof("waiting for gossip to be connected")
checkGossip(t, c, 20*time.Second, hasPeers(num))
checkGossip(t, c, time.Second, hasClusterID)
checkGossip(t, c, time.Second, hasSentinel)
for i := 0; i < num; i++ {
db, dbStopper := c.MakeClient(t, i)
if i == 0 {
if err := db.Del("count"); err != nil {
t.Fatal(err)
}
}
var kv client.KeyValue
if err := db.Txn(func(txn *client.Txn) error {
var err error
kv, err = txn.Inc("count", 1)
return err
}); err != nil {
t.Fatal(err)
} else if v := kv.ValueInt(); v != int64(i+1) {
t.Fatalf("unexpected value %d for write #%d (expected %d)", v, i, i+1)
}
dbStopper.Stop()
}
}
func (ia *idAllocator) start() {
ia.stopper.RunWorker(func() {
defer close(ia.ids)
for {
var newValue int64
for newValue <= int64(ia.minID) {
var (
err error
res client.KeyValue
)
for r := retry.Start(idAllocationRetryOpts); r.Next(); {
if ia.stopper.StartTask() {
idKey := ia.idKey.Load().(proto.Key)
res, err = ia.db.Inc(idKey, int64(ia.blockSize))
ia.stopper.FinishTask()
if err == nil {
newValue = res.ValueInt()
break
}
log.Warningf("unable to allocate %d ids from %s: %s", ia.blockSize, idKey, err)
} else {
return
}
}
if err != nil {
panic(fmt.Sprintf("unexpectedly exited id allocation retry loop: %s", err))
}
}
end := newValue + 1
start := end - int64(ia.blockSize)
if start < int64(ia.minID) {
start = int64(ia.minID)
}
// Add all new ids to the channel for consumption.
for i := start; i < end; i++ {
select {
case ia.ids <- uint32(i):
case <-ia.stopper.ShouldStop():
return
}
}
}
})
}
func unmarshalValue(col structured.ColumnDescriptor, kv client.KeyValue) parser.Datum {
if kv.Exists() {
switch col.Type.Kind {
case structured.ColumnType_BIT, structured.ColumnType_INT:
return parser.DInt(kv.ValueInt())
case structured.ColumnType_FLOAT:
return parser.DFloat(math.Float64frombits(uint64(kv.ValueInt())))
case structured.ColumnType_CHAR, structured.ColumnType_TEXT,
structured.ColumnType_BLOB:
return parser.DString(kv.ValueBytes())
}
}
return parser.DNull
}
func (n *scanNode) unmarshalValue(kv client.KeyValue) (parser.Datum, bool) {
kind, ok := n.colKind[n.colID]
if !ok {
n.err = fmt.Errorf("column-id \"%d\" does not exist", n.colID)
return nil, false
}
if kv.Exists() {
switch kind {
case ColumnType_INT:
return parser.DInt(kv.ValueInt()), true
case ColumnType_BOOL:
return parser.DBool(kv.ValueInt() != 0), true
case ColumnType_FLOAT:
return parser.DFloat(math.Float64frombits(uint64(kv.ValueInt()))), true
case ColumnType_STRING, ColumnType_BYTES:
return parser.DString(kv.ValueBytes()), true
case ColumnType_DATE:
var t time.Time
if err := t.UnmarshalBinary(kv.ValueBytes()); err != nil {
return nil, false
}
return parser.DDate{Time: t}, true
case ColumnType_TIMESTAMP:
var t time.Time
if err := t.UnmarshalBinary(kv.ValueBytes()); err != nil {
return nil, false
}
return parser.DTimestamp{Time: t}, true
case ColumnType_INTERVAL:
return parser.DInterval{Duration: time.Duration(kv.ValueInt())}, true
}
}
return parser.DNull, true
}
func unmarshalValue(col structured.ColumnDescriptor, kv client.KeyValue) driver.Value {
if !kv.Exists() {
return nil
}
switch col.Type.Kind {
case structured.ColumnType_BIT, structured.ColumnType_INT:
return kv.ValueInt()
case structured.ColumnType_FLOAT:
return math.Float64frombits(uint64(kv.ValueInt()))
case structured.ColumnType_CHAR, structured.ColumnType_BINARY,
structured.ColumnType_TEXT, structured.ColumnType_BLOB:
// TODO(pmattis): The conversion to string isn't strictly necessary, but
// makes log messages more readable right now.
return string(kv.ValueBytes())
}
return kv.ValueBytes()
}
func unmarshalValue(col structured.ColumnDescriptor, kv client.KeyValue) sqlwire.Datum {
var d sqlwire.Datum
if !kv.Exists() {
return d
}
switch col.Type.Kind {
case structured.ColumnType_BIT, structured.ColumnType_INT:
tmp := kv.ValueInt()
d.IntVal = &tmp
case structured.ColumnType_FLOAT:
tmp := math.Float64frombits(uint64(kv.ValueInt()))
d.FloatVal = &tmp
case structured.ColumnType_CHAR, structured.ColumnType_TEXT,
structured.ColumnType_BLOB:
// TODO(pmattis): The conversion to string isn't strictly necessary, but
// makes log messages more readable right now.
tmp := string(kv.ValueBytes())
d.StringVal = &tmp
}
return d
}
func (n *scanNode) unmarshalValue(kv client.KeyValue) (parser.Datum, bool) {
kind, ok := n.colKind[n.colID]
if !ok {
n.err = fmt.Errorf("column-id \"%d\" does not exist", n.colID)
return nil, false
}
if kv.Exists() {
switch kind {
case ColumnType_INT:
return parser.DInt(kv.ValueInt()), true
case ColumnType_BOOL:
return parser.DBool(kv.ValueInt() != 0), true
case ColumnType_FLOAT:
return parser.DFloat(math.Float64frombits(uint64(kv.ValueInt()))), true
case ColumnType_STRING, ColumnType_BYTES:
return parser.DString(kv.ValueBytes()), true
}
}
return parser.DNull, true
}
func testGossipRestartInner(t *testing.T, c cluster.Cluster, cfg cluster.TestConfig) {
// This already replicates the first range (in the local setup).
// The replication of the first range is important: as long as the
// first range only exists on one node, that node can trivially
// acquire the leader lease. Once the range is replicated, however,
// nodes must be able to discover each other over gossip before the
// lease can be acquired.
num := c.NumNodes()
deadline := timeutil.Now().Add(cfg.Duration)
waitTime := longWaitTime
if cfg.Duration < waitTime {
waitTime = shortWaitTime
}
for timeutil.Now().Before(deadline) {
log.Infof("waiting for initial gossip connections")
checkGossip(t, c, waitTime, hasPeers(num))
checkGossip(t, c, waitTime, hasClusterID)
checkGossip(t, c, waitTime, hasSentinel)
log.Infof("killing all nodes")
for i := 0; i < num; i++ {
if err := c.Kill(i); err != nil {
t.Fatal(err)
}
}
log.Infof("restarting all nodes")
for i := 0; i < num; i++ {
if err := c.Restart(i); err != nil {
t.Fatal(err)
}
}
log.Infof("waiting for gossip to be connected")
checkGossip(t, c, waitTime, hasPeers(num))
checkGossip(t, c, waitTime, hasClusterID)
checkGossip(t, c, waitTime, hasSentinel)
for i := 0; i < num; i++ {
db, dbStopper := c.NewClient(t, i)
if i == 0 {
if err := db.Del("count"); err != nil {
t.Fatal(err)
}
}
var kv client.KeyValue
if err := db.Txn(func(txn *client.Txn) error {
var err error
kv, err = txn.Inc("count", 1)
return err
}); err != nil {
t.Fatal(err)
} else if v := kv.ValueInt(); v != int64(i+1) {
t.Fatalf("unexpected value %d for write #%d (expected %d)", v, i, i+1)
}
dbStopper.Stop()
}
}
}