func printRaftLogEntry(kv engine.MVCCKeyValue) (bool, error) {
var meta engine.MVCCMetadata
if err := meta.Unmarshal(kv.Value); err != nil {
return false, err
}
value := roachpb.Value{
RawBytes: meta.RawBytes,
}
var ent raftpb.Entry
if err := value.GetProto(&ent); err != nil {
return false, err
}
if len(ent.Data) > 0 {
_, cmdData := storage.DecodeRaftCommand(ent.Data)
var cmd roachpb.RaftCommand
if err := cmd.Unmarshal(cmdData); err != nil {
return false, err
}
ent.Data = nil
fmt.Printf("%s\n", &ent)
fmt.Printf("%s\n", &cmd)
} else {
fmt.Printf("%s: EMPTY\n", &ent)
}
return false, nil
}
func maybeUnmarshalInline(v []byte, dest proto.Message) error {
var meta enginepb.MVCCMetadata
if err := meta.Unmarshal(v); err != nil {
return err
}
value := roachpb.Value{
RawBytes: meta.RawBytes,
}
return value.GetProto(dest)
}
// storeGossipUpdate is the gossip callback used to keep the StorePool up to date.
func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) {
var storeDesc roachpb.StoreDescriptor
if err := content.GetProto(&storeDesc); err != nil {
log.Error(context.TODO(), err)
return
}
sp.mu.Lock()
defer sp.mu.Unlock()
// Does this storeDetail exist yet?
detail := sp.getStoreDetailLocked(storeDesc.StoreID)
detail.markAlive(sp.clock.Now(), &storeDesc)
sp.mu.queue.enqueue(detail)
}
func tryMeta(kv engine.MVCCKeyValue) (string, error) {
if !bytes.HasPrefix(kv.Key.Key, keys.Meta1Prefix) && !bytes.HasPrefix(kv.Key.Key, keys.Meta2Prefix) {
return "", errors.New("not a meta key")
}
value := roachpb.Value{
Timestamp: kv.Key.Timestamp,
RawBytes: kv.Value,
}
var desc roachpb.RangeDescriptor
if err := value.GetProto(&desc); err != nil {
return "", err
}
return descStr(desc), nil
}
// deadReplicasGossipUpdate is the gossip callback used to keep the StorePool up to date.
func (sp *StorePool) deadReplicasGossipUpdate(_ string, content roachpb.Value) {
var replicas roachpb.StoreDeadReplicas
if err := content.GetProto(&replicas); err != nil {
log.Error(context.TODO(), err)
return
}
sp.mu.Lock()
defer sp.mu.Unlock()
detail := sp.getStoreDetailLocked(replicas.StoreID)
deadReplicas := make(map[roachpb.RangeID][]roachpb.ReplicaDescriptor)
for _, r := range replicas.Replicas {
deadReplicas[r.RangeID] = append(deadReplicas[r.RangeID], r.Replica)
}
detail.deadReplicas = deadReplicas
}
// updateNodeAddress is a gossip callback which fires with each
// update to the node address. This allows us to compute the
// total size of the gossip network (for determining max peers
// each gossip node is allowed to have), as well as to create
// new resolvers for each encountered host and to write the
// set of gossip node addresses to persistent storage when it
// changes.
func (g *Gossip) updateNodeAddress(_ string, content roachpb.Value) {
var desc roachpb.NodeDescriptor
if err := content.GetProto(&desc); err != nil {
log.Error(err)
return
}
g.mu.Lock()
defer g.mu.Unlock()
// Recompute max peers based on size of network and set the max
// sizes for incoming and outgoing node sets.
defer func() {
maxPeers := g.maxPeers(len(g.nodeDescs))
g.incoming.setMaxSize(maxPeers)
g.outgoing.setMaxSize(maxPeers)
}()
// Skip if the node has already been seen or it's our own address.
if _, ok := g.nodeDescs[desc.NodeID]; ok || desc.Address == g.is.NodeAddr {
return
}
g.nodeDescs[desc.NodeID] = &desc
// Add this new node to our list of resolvers so we can keep
// connecting to gossip if the original resolvers go offline.
r, err := resolver.NewResolverFromUnresolvedAddr(desc.Address)
if err != nil {
log.Warningf("bad address from gossip node %s: %s", desc, err)
return
}
if !g.haveResolver(r) {
g.resolvers = append(g.resolvers, r)
}
// Add new address to bootstrap info and persist if possible.
if !g.haveBootstrapAddress(desc.Address) {
g.bootstrapInfo.Addresses = append(g.bootstrapInfo.Addresses, desc.Address)
if g.storage != nil {
// TODO(spencer): need to clean up ancient gossip nodes, which
// will otherwise stick around in the bootstrap info forever.
if err := g.storage.WriteBootstrapInfo(&g.bootstrapInfo); err != nil {
log.Error(err)
}
}
}
}
func tryRangeDescriptor(kv engine.MVCCKeyValue) (string, error) {
_, suffix, _, err := keys.DecodeRangeKey(kv.Key.Key)
if err != nil {
return "", err
}
if !bytes.Equal(suffix, keys.LocalRangeDescriptorSuffix) {
return "", fmt.Errorf("wrong suffix: %s", suffix)
}
value := roachpb.Value{
RawBytes: kv.Value,
}
var desc roachpb.RangeDescriptor
if err := value.GetProto(&desc); err != nil {
return "", err
}
return descStr(desc), nil
}
// updateSystemConfig is the raw gossip info callback.
// Unmarshal the system config, and if successfuly, update out
// copy and run the callbacks.
func (g *Gossip) updateSystemConfig(key string, content roachpb.Value) {
if key != KeySystemConfig {
log.Fatalf("wrong key received on SystemConfig callback: %s", key)
return
}
cfg := &config.SystemConfig{}
if err := content.GetProto(cfg); err != nil {
log.Errorf("could not unmarshal system config on callback: %s", err)
return
}
g.systemConfigMu.Lock()
defer g.systemConfigMu.Unlock()
g.systemConfig = cfg
for _, cb := range g.systemConfigCallbacks {
go cb(cfg)
}
}
func printRangeDescriptor(kv engine.MVCCKeyValue) (bool, error) {
startKey, suffix, _, err := keys.DecodeRangeKey(kv.Key.Key)
if err != nil {
return false, err
}
if !bytes.Equal(suffix, keys.LocalRangeDescriptorSuffix) {
return false, nil
}
value := roachpb.Value{
RawBytes: kv.Value,
}
var desc roachpb.RangeDescriptor
if err := value.GetProto(&desc); err != nil {
return false, err
}
fmt.Printf("Range descriptor with start key %s at time %s\n%s\n", startKey, kv.Key.Timestamp.GoTime(), &desc)
return false, nil
}
// storeGossipUpdate is the gossip callback used to keep the StorePool up to date.
func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) {
var storeDesc roachpb.StoreDescriptor
if err := content.GetProto(&storeDesc); err != nil {
log.Error(err)
return
}
sp.mu.Lock()
defer sp.mu.Unlock()
// Does this storeDetail exist yet?
detail, ok := sp.stores[storeDesc.StoreID]
if !ok {
// Setting index to -1 ensures this gets added to the queue.
detail = &storeDetail{index: -1}
sp.stores[storeDesc.StoreID] = detail
}
detail.markAlive(sp.clock.Now(), storeDesc, true)
sp.queue.enqueue(detail)
}
// updateNodeAddress is a gossip callback which fires with each
// update to the node address. This allows us to compute the
// total size of the gossip network (for determining max peers
// each gossip node is allowed to have), as well as to create
// new resolvers for each encountered host and to write the
// set of gossip node addresses to persistent storage when it
// changes.
func (g *Gossip) updateNodeAddress(_ string, content roachpb.Value) {
var desc roachpb.NodeDescriptor
if err := content.GetProto(&desc); err != nil {
log.Error(err)
return
}
g.mu.Lock()
defer g.mu.Unlock()
// Skip if the node has already been seen.
if _, ok := g.nodeDescs[desc.NodeID]; ok {
return
}
g.nodeDescs[desc.NodeID] = &desc
// Recompute max peers based on size of network and set the max
// sizes for incoming and outgoing node sets.
maxPeers := g.maxPeers(len(g.nodeDescs))
g.incoming.setMaxSize(maxPeers)
g.outgoing.setMaxSize(maxPeers)
// Skip if it's our own address.
if desc.Address == g.is.NodeAddr {
return
}
// Add this new node address (if it's not already there) to our list
// of resolvers so we can keep connecting to gossip if the original
// resolvers go offline.
g.maybeAddResolver(desc.Address)
// Add new address (if it's not already there) to bootstrap info and
// persist if possible.
if g.maybeAddBootstrapAddress(desc.Address) && g.storage != nil {
// TODO(spencer): need to clean up ancient gossip nodes, which
// will otherwise stick around in the bootstrap info forever.
if err := g.storage.WriteBootstrapInfo(&g.bootstrapInfo); err != nil {
log.Error(err)
}
}
}
// updateSystemConfig is the raw gossip info callback.
// Unmarshal the system config, and if successfully, update out
// copy and run the callbacks.
func (g *Gossip) updateSystemConfig(key string, content roachpb.Value) {
if key != KeySystemConfig {
log.Fatalf("wrong key received on SystemConfig callback: %s", key)
return
}
cfg := config.SystemConfig{}
if err := content.GetProto(&cfg); err != nil {
log.Errorf("could not unmarshal system config on callback: %s", err)
return
}
g.systemConfigMu.Lock()
defer g.systemConfigMu.Unlock()
g.systemConfig = cfg
g.systemConfigSet = true
for _, c := range g.systemConfigChannels {
select {
case c <- struct{}{}:
default:
}
}
}
func TestSystemDBGossip(t *testing.T) {
defer leaktest.AfterTest(t)
s := StartTestServer(t)
defer s.Stop()
resultChan := make(chan roachpb.Value)
var count int32
db := s.db
key := sql.MakeDescMetadataKey(keys.MaxReservedDescID)
valAt := func(i int) *sql.DatabaseDescriptor {
return &sql.DatabaseDescriptor{Name: "foo", ID: sql.ID(i)}
}
// Register a callback for gossip updates.
s.Gossip().RegisterCallback(gossip.KeySystemConfig, func(_ string, content roachpb.Value) {
newCount := atomic.AddInt32(&count, 1)
if newCount != 2 {
// RegisterCallback calls us right away with the contents,
// so ignore the very first call.
// We also only want the first value of all our writes.
return
}
resultChan <- content
})
// The span only gets gossiped when it first shows up, or when
// the EndTransaction trigger is set.
// Try a plain KV write first.
if err := db.Put(key, valAt(0)); err != nil {
t.Fatal(err)
}
// Now do it as part of a transaction, but without the trigger set.
if err := db.Txn(func(txn *client.Txn) error {
return txn.Put(key, valAt(1))
}); err != nil {
t.Fatal(err)
}
// This time mark the transaction as having a SystemDB trigger.
if err := db.Txn(func(txn *client.Txn) error {
txn.SetSystemDBTrigger()
return txn.Put(key, valAt(2))
}); err != nil {
t.Fatal(err)
}
// Wait for the callback.
var systemConfig config.SystemConfig
select {
case content := <-resultChan:
if err := content.GetProto(&systemConfig); err != nil {
t.Fatal(err)
}
case <-time.After(500 * time.Millisecond):
t.Fatal("did not receive gossip callback")
}
// Now check the gossip callback.
var val *roachpb.Value
for _, kv := range systemConfig.Values {
if bytes.Equal(key, kv.Key) {
val = &kv.Value
break
}
}
if val == nil {
t.Fatal("key not found in gossiped info")
}
// Make sure the returned value is valAt(2).
var got sql.DatabaseDescriptor
if err := val.GetProto(&got); err != nil {
t.Fatal(err)
}
if got.ID != 2 {
t.Fatalf("mismatch: expected %+v, got %+v", valAt(2), got)
}
}
func tryRangeIDKey(kv engine.MVCCKeyValue) (string, error) {
if kv.Key.Timestamp != hlc.ZeroTimestamp {
return "", fmt.Errorf("range ID keys shouldn't have timestamps: %s", kv.Key)
}
_, _, suffix, _, err := keys.DecodeRangeIDKey(kv.Key.Key)
if err != nil {
return "", err
}
// All range ID keys are stored inline on the metadata.
var meta enginepb.MVCCMetadata
if err := meta.Unmarshal(kv.Value); err != nil {
return "", err
}
value := roachpb.Value{RawBytes: meta.RawBytes}
// Values encoded as protobufs set msg and continue outside the
// switch. Other types are handled inside the switch and return.
var msg proto.Message
switch {
case bytes.Equal(suffix, keys.LocalLeaseAppliedIndexSuffix):
fallthrough
case bytes.Equal(suffix, keys.LocalRaftAppliedIndexSuffix):
i, err := value.GetInt()
if err != nil {
return "", err
}
return strconv.FormatInt(i, 10), nil
case bytes.Equal(suffix, keys.LocalRangeFrozenStatusSuffix):
b, err := value.GetBool()
if err != nil {
return "", err
}
return strconv.FormatBool(b), nil
case bytes.Equal(suffix, keys.LocalAbortCacheSuffix):
msg = &roachpb.AbortCacheEntry{}
case bytes.Equal(suffix, keys.LocalRangeLastGCSuffix):
msg = &hlc.Timestamp{}
case bytes.Equal(suffix, keys.LocalRaftTombstoneSuffix):
msg = &roachpb.RaftTombstone{}
case bytes.Equal(suffix, keys.LocalRaftTruncatedStateSuffix):
msg = &roachpb.RaftTruncatedState{}
case bytes.Equal(suffix, keys.LocalRangeLeaseSuffix):
msg = &roachpb.Lease{}
case bytes.Equal(suffix, keys.LocalRangeStatsSuffix):
msg = &enginepb.MVCCStats{}
case bytes.Equal(suffix, keys.LocalRaftHardStateSuffix):
msg = &raftpb.HardState{}
case bytes.Equal(suffix, keys.LocalRaftLastIndexSuffix):
i, err := value.GetInt()
if err != nil {
return "", err
}
return strconv.FormatInt(i, 10), nil
case bytes.Equal(suffix, keys.LocalRangeLastVerificationTimestampSuffix):
msg = &hlc.Timestamp{}
case bytes.Equal(suffix, keys.LocalRangeLastReplicaGCTimestampSuffix):
msg = &hlc.Timestamp{}
default:
return "", fmt.Errorf("unknown raft id key %s", suffix)
}
if err := value.GetProto(msg); err != nil {
return "", err
}
return msg.String(), nil
}
func TestSystemConfigGossip(t *testing.T) {
defer leaktest.AfterTest(t)()
s := StartTestServer(t)
defer s.Stop()
db := s.db
key := sql.MakeDescMetadataKey(keys.MaxReservedDescID)
valAt := func(i int) *sql.DatabaseDescriptor {
return &sql.DatabaseDescriptor{Name: "foo", ID: sql.ID(i)}
}
// Register a callback for gossip updates.
resultChan := s.Gossip().RegisterSystemConfigChannel()
// The span gets gossiped when it first shows up.
select {
case <-resultChan:
case <-time.After(500 * time.Millisecond):
t.Fatal("did not receive gossip message")
}
// Try a plain KV write first.
if err := db.Put(key, valAt(0)); err != nil {
t.Fatal(err)
}
// Now do it as part of a transaction, but without the trigger set.
if pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
return txn.Put(key, valAt(1))
}); pErr != nil {
t.Fatal(pErr)
}
// Gossip channel should be dormant.
var systemConfig config.SystemConfig
select {
case <-resultChan:
systemConfig = *s.gossip.GetSystemConfig()
t.Fatalf("unexpected message received on gossip channel: %v", systemConfig)
case <-time.After(50 * time.Millisecond):
}
// This time mark the transaction as having a Gossip trigger.
if pErr := db.Txn(func(txn *client.Txn) *roachpb.Error {
txn.SetSystemConfigTrigger()
return txn.Put(key, valAt(2))
}); pErr != nil {
t.Fatal(pErr)
}
// New system config received.
select {
case <-resultChan:
systemConfig = *s.gossip.GetSystemConfig()
case <-time.After(500 * time.Millisecond):
t.Fatal("did not receive gossip message")
}
// Now check the new config.
var val *roachpb.Value
for _, kv := range systemConfig.Values {
if bytes.Equal(key, kv.Key) {
val = &kv.Value
break
}
}
if val == nil {
t.Fatal("key not found in gossiped info")
}
// Make sure the returned value is valAt(2).
got := new(sql.DatabaseDescriptor)
if err := val.GetProto(got); err != nil {
t.Fatal(err)
}
if expected := valAt(2); !reflect.DeepEqual(got, expected) {
t.Fatalf("mismatch: expected %+v, got %+v", *expected, *got)
}
}
func TestSystemConfigGossip(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
key := sqlbase.MakeDescMetadataKey(keys.MaxReservedDescID)
valAt := func(i int) *sqlbase.DatabaseDescriptor {
return &sqlbase.DatabaseDescriptor{Name: "foo", ID: sqlbase.ID(i)}
}
// Register a callback for gossip updates.
resultChan := ts.Gossip().RegisterSystemConfigChannel()
// The span gets gossiped when it first shows up.
select {
case <-resultChan:
case <-time.After(500 * time.Millisecond):
t.Fatal("did not receive gossip message")
}
// Try a plain KV write first.
if err := kvDB.Put(key, valAt(0)); err != nil {
t.Fatal(err)
}
// Now do it as part of a transaction, but without the trigger set.
if err := kvDB.Txn(context.TODO(), func(txn *client.Txn) error {
return txn.Put(key, valAt(1))
}); err != nil {
t.Fatal(err)
}
// Gossip channel should be dormant.
// TODO(tschottdorf): This test is likely flaky. Why can't some other
// process trigger gossip? It seems that a new range lease being
// acquired will gossip a new system config since the hash changed and fail
// the test (seen in practice during some buggy WIP).
var systemConfig config.SystemConfig
select {
case <-resultChan:
systemConfig, _ = ts.gossip.GetSystemConfig()
t.Fatalf("unexpected message received on gossip channel: %v", systemConfig)
case <-time.After(50 * time.Millisecond):
}
// This time mark the transaction as having a Gossip trigger.
if err := kvDB.Txn(context.TODO(), func(txn *client.Txn) error {
txn.SetSystemConfigTrigger()
return txn.Put(key, valAt(2))
}); err != nil {
t.Fatal(err)
}
// New system config received.
select {
case <-resultChan:
systemConfig, _ = ts.gossip.GetSystemConfig()
case <-time.After(500 * time.Millisecond):
t.Fatal("did not receive gossip message")
}
// Now check the new config.
var val *roachpb.Value
for _, kv := range systemConfig.Values {
if bytes.Equal(key, kv.Key) {
val = &kv.Value
break
}
}
if val == nil {
t.Fatal("key not found in gossiped info")
}
// Make sure the returned value is valAt(2).
got := new(sqlbase.DatabaseDescriptor)
if err := val.GetProto(got); err != nil {
t.Fatal(err)
}
if expected := valAt(2); !reflect.DeepEqual(got, expected) {
t.Fatalf("mismatch: expected %+v, got %+v", *expected, *got)
}
}
func getProtoValue(data []byte, msg proto.Message) error {
value := roachpb.Value{
RawBytes: data,
}
return value.GetProto(msg)
}
