func maybeUnmarshalInline(v []byte, dest proto.Message) error { var meta enginepb.MVCCMetadata if err := meta.Unmarshal(v); err != nil { return err } value := roachpb.Value{ RawBytes: meta.RawBytes, } return value.GetProto(dest) }
func tryMeta(kv engine.MVCCKeyValue) (string, error) { if !bytes.HasPrefix(kv.Key.Key, keys.Meta1Prefix) && !bytes.HasPrefix(kv.Key.Key, keys.Meta2Prefix) { return "", errors.New("not a meta key") } value := roachpb.Value{ Timestamp: kv.Key.Timestamp, RawBytes: kv.Value, } var desc roachpb.RangeDescriptor if err := value.GetProto(&desc); err != nil { return "", err } return descStr(desc), nil }
// storeGossipUpdate is the gossip callback used to keep the StorePool up to date. func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) { var storeDesc roachpb.StoreDescriptor if err := content.GetProto(&storeDesc); err != nil { ctx := sp.AnnotateCtx(context.TODO()) log.Error(ctx, err) return } sp.mu.Lock() defer sp.mu.Unlock() // Does this storeDetail exist yet? detail := sp.getStoreDetailLocked(storeDesc.StoreID) detail.markAlive(sp.clock.Now(), &storeDesc) sp.mu.queue.enqueue(detail) }
// storeGossipUpdate is the gossip callback used to keep the StorePool up to date. func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) { var storeDesc roachpb.StoreDescriptor if err := content.GetProto(&storeDesc); err != nil { ctx := sp.AnnotateCtx(context.TODO()) log.Error(ctx, err) return } sp.mu.Lock() defer sp.mu.Unlock() detail := sp.getStoreDetailLocked(storeDesc.StoreID) detail.desc = &storeDesc detail.lastUpdatedTime = sp.clock.PhysicalTime() sp.mu.nodeLocalities[storeDesc.Node.NodeID] = storeDesc.Node.Locality }
// deadReplicasGossipUpdate is the gossip callback used to keep the StorePool up to date. func (sp *StorePool) deadReplicasGossipUpdate(_ string, content roachpb.Value) { var replicas roachpb.StoreDeadReplicas if err := content.GetProto(&replicas); err != nil { ctx := sp.AnnotateCtx(context.TODO()) log.Error(ctx, err) return } sp.mu.Lock() defer sp.mu.Unlock() detail := sp.getStoreDetailLocked(replicas.StoreID) deadReplicas := make(map[roachpb.RangeID][]roachpb.ReplicaDescriptor) for _, r := range replicas.Replicas { deadReplicas[r.RangeID] = append(deadReplicas[r.RangeID], r.Replica) } detail.deadReplicas = deadReplicas }
// livenessGossipUpdate is the gossip callback used to keep the // in-memory liveness info up to date. func (nl *NodeLiveness) livenessGossipUpdate(key string, content roachpb.Value) { var liveness Liveness if err := content.GetProto(&liveness); err != nil { log.Error(context.TODO(), err) return } // If there's an existing liveness record, only update the received // timestamp if this is our first receipt of this node's liveness // or if the expiration or epoch was advanced. nl.mu.Lock() defer nl.mu.Unlock() exLiveness, ok := nl.mu.nodes[liveness.NodeID] if !ok || exLiveness.Expiration.Less(liveness.Expiration) || exLiveness.Epoch < liveness.Epoch { nl.mu.nodes[liveness.NodeID] = liveness } }
// updateNodeAddress is a gossip callback which fires with each // update to the node address. This allows us to compute the // total size of the gossip network (for determining max peers // each gossip node is allowed to have), as well as to create // new resolvers for each encountered host and to write the // set of gossip node addresses to persistent storage when it // changes. func (g *Gossip) updateNodeAddress(_ string, content roachpb.Value) { ctx := g.AnnotateCtx(context.TODO()) var desc roachpb.NodeDescriptor if err := content.GetProto(&desc); err != nil { log.Error(ctx, err) return } g.mu.Lock() defer g.mu.Unlock() // Skip if the node has already been seen. if _, ok := g.nodeDescs[desc.NodeID]; ok { return } g.nodeDescs[desc.NodeID] = &desc // Recompute max peers based on size of network and set the max // sizes for incoming and outgoing node sets. maxPeers := g.maxPeers(len(g.nodeDescs)) g.mu.incoming.setMaxSize(maxPeers) g.outgoing.setMaxSize(maxPeers) // Skip if it's our own address. if desc.Address == g.mu.is.NodeAddr { return } // Add this new node address (if it's not already there) to our list // of resolvers so we can keep connecting to gossip if the original // resolvers go offline. g.maybeAddResolver(desc.Address) // Add new address (if it's not already there) to bootstrap info and // persist if possible. if g.storage != nil && g.maybeAddBootstrapAddress(desc.Address) { if err := g.storage.WriteBootstrapInfo(&g.bootstrapInfo); err != nil { log.Error(ctx, err) } } }
// MigrateZoneConfig migrates the legacy ZoneConfig format into the new one. func MigrateZoneConfig(value *roachpb.Value) (ZoneConfig, error) { var zone ZoneConfig if err := value.GetProto(&zone); err != nil { return ZoneConfig{}, err } if len(zone.ReplicaAttrs) > 0 { if zone.NumReplicas > 0 || len(zone.Constraints.Constraints) > 0 { return ZoneConfig{}, errors.New("migration to new ZoneConfig failed due to previous partial upgrade") } zone.NumReplicas = int32(len(zone.ReplicaAttrs)) if zone.NumReplicas > 0 { attrs := zone.ReplicaAttrs[0].Attrs zone.Constraints.Constraints = make([]Constraint, len(attrs)) for i, attr := range attrs { zone.Constraints.Constraints[i].Value = attr } } zone.ReplicaAttrs = nil } return zone, nil }
// updateSystemConfig is the raw gossip info callback. // Unmarshal the system config, and if successfully, update out // copy and run the callbacks. func (g *Gossip) updateSystemConfig(key string, content roachpb.Value) { ctx := g.AnnotateCtx(context.TODO()) if key != KeySystemConfig { log.Fatalf(ctx, "wrong key received on SystemConfig callback: %s", key) } cfg := config.SystemConfig{} if err := content.GetProto(&cfg); err != nil { log.Errorf(ctx, "could not unmarshal system config on callback: %s", err) return } g.systemConfigMu.Lock() defer g.systemConfigMu.Unlock() g.systemConfig = cfg g.systemConfigSet = true for _, c := range g.systemConfigChannels { select { case c <- struct{}{}: default: } } }
func getProtoValue(data []byte, msg proto.Message) error { value := roachpb.Value{ RawBytes: data, } return value.GetProto(msg) }
func tryRangeIDKey(kv engine.MVCCKeyValue) (string, error) { if kv.Key.Timestamp != hlc.ZeroTimestamp { return "", fmt.Errorf("range ID keys shouldn't have timestamps: %s", kv.Key) } _, _, suffix, _, err := keys.DecodeRangeIDKey(kv.Key.Key) if err != nil { return "", err } // All range ID keys are stored inline on the metadata. var meta enginepb.MVCCMetadata if err := meta.Unmarshal(kv.Value); err != nil { return "", err } value := roachpb.Value{RawBytes: meta.RawBytes} // Values encoded as protobufs set msg and continue outside the // switch. Other types are handled inside the switch and return. var msg proto.Message switch { case bytes.Equal(suffix, keys.LocalLeaseAppliedIndexSuffix): fallthrough case bytes.Equal(suffix, keys.LocalRaftAppliedIndexSuffix): i, err := value.GetInt() if err != nil { return "", err } return strconv.FormatInt(i, 10), nil case bytes.Equal(suffix, keys.LocalRangeFrozenStatusSuffix): b, err := value.GetBool() if err != nil { return "", err } return strconv.FormatBool(b), nil case bytes.Equal(suffix, keys.LocalAbortCacheSuffix): msg = &roachpb.AbortCacheEntry{} case bytes.Equal(suffix, keys.LocalRangeLastGCSuffix): msg = &hlc.Timestamp{} case bytes.Equal(suffix, keys.LocalRaftTombstoneSuffix): msg = &roachpb.RaftTombstone{} case bytes.Equal(suffix, keys.LocalRaftTruncatedStateSuffix): msg = &roachpb.RaftTruncatedState{} case bytes.Equal(suffix, keys.LocalRangeLeaseSuffix): msg = &roachpb.Lease{} case bytes.Equal(suffix, keys.LocalRangeStatsSuffix): msg = &enginepb.MVCCStats{} case bytes.Equal(suffix, keys.LocalRaftHardStateSuffix): msg = &raftpb.HardState{} case bytes.Equal(suffix, keys.LocalRaftLastIndexSuffix): i, err := value.GetInt() if err != nil { return "", err } return strconv.FormatInt(i, 10), nil case bytes.Equal(suffix, keys.LocalRangeLastVerificationTimestampSuffixDeprecated): msg = &hlc.Timestamp{} case bytes.Equal(suffix, keys.LocalRangeLastReplicaGCTimestampSuffix): msg = &hlc.Timestamp{} default: return "", fmt.Errorf("unknown raft id key %s", suffix) } if err := value.GetProto(msg); err != nil { return "", err } return msg.String(), nil }
func TestSystemConfigGossip(t *testing.T) { defer leaktest.AfterTest(t)() t.Skip("#12351") s, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) ctx := context.TODO() key := sqlbase.MakeDescMetadataKey(keys.MaxReservedDescID) valAt := func(i int) *sqlbase.DatabaseDescriptor { return &sqlbase.DatabaseDescriptor{Name: "foo", ID: sqlbase.ID(i)} } // Register a callback for gossip updates. resultChan := ts.Gossip().RegisterSystemConfigChannel() // The span gets gossiped when it first shows up. select { case <-resultChan: case <-time.After(500 * time.Millisecond): t.Fatal("did not receive gossip message") } // Try a plain KV write first. if err := kvDB.Put(ctx, key, valAt(0)); err != nil { t.Fatal(err) } // Now do it as part of a transaction, but without the trigger set. if err := kvDB.Txn(ctx, func(txn *client.Txn) error { return txn.Put(key, valAt(1)) }); err != nil { t.Fatal(err) } // Gossip channel should be dormant. // TODO(tschottdorf): This test is likely flaky. Why can't some other // process trigger gossip? It seems that a new range lease being // acquired will gossip a new system config since the hash changed and fail // the test (seen in practice during some buggy WIP). var systemConfig config.SystemConfig select { case <-resultChan: systemConfig, _ = ts.gossip.GetSystemConfig() t.Fatalf("unexpected message received on gossip channel: %v", systemConfig) case <-time.After(50 * time.Millisecond): } // This time mark the transaction as having a Gossip trigger. if err := kvDB.Txn(ctx, func(txn *client.Txn) error { txn.SetSystemConfigTrigger() return txn.Put(key, valAt(2)) }); err != nil { t.Fatal(err) } // New system config received. select { case <-resultChan: systemConfig, _ = ts.gossip.GetSystemConfig() case <-time.After(500 * time.Millisecond): t.Fatal("did not receive gossip message") } // Now check the new config. var val *roachpb.Value for _, kv := range systemConfig.Values { if bytes.Equal(key, kv.Key) { val = &kv.Value break } } if val == nil { t.Fatal("key not found in gossiped info") } // Make sure the returned value is valAt(2). got := new(sqlbase.DatabaseDescriptor) if err := val.GetProto(got); err != nil { t.Fatal(err) } if expected := valAt(2); !reflect.DeepEqual(got, expected) { t.Fatalf("mismatch: expected %+v, got %+v", *expected, *got) } }
// updateNodeAddress is a gossip callback which fires with each // update to the node address. This allows us to compute the // total size of the gossip network (for determining max peers // each gossip node is allowed to have), as well as to create // new resolvers for each encountered host and to write the // set of gossip node addresses to persistent storage when it // changes. func (g *Gossip) updateNodeAddress(key string, content roachpb.Value) { ctx := g.AnnotateCtx(context.TODO()) var desc roachpb.NodeDescriptor if err := content.GetProto(&desc); err != nil { log.Error(ctx, err) return } g.mu.Lock() defer g.mu.Unlock() // If desc is the empty descriptor, that indicates that the node has been // removed from the cluster. If that's the case, remove it from our map of // nodes to prevent other parts of the system from trying to talk to it. // We can't directly compare the node against the empty descriptor because // the proto has a repeated field and thus isn't comparable. if desc.NodeID == 0 && desc.Address.IsEmpty() { nodeID, err := NodeIDFromKey(key) if err != nil { log.Errorf(ctx, "unable to update node address for removed node: %s", err) return } log.Infof(ctx, "removed node %d from gossip", nodeID) delete(g.nodeDescs, nodeID) return } // Skip if the node has already been seen. if _, ok := g.nodeDescs[desc.NodeID]; ok { return } g.nodeDescs[desc.NodeID] = &desc // Recompute max peers based on size of network and set the max // sizes for incoming and outgoing node sets. maxPeers := g.maxPeers(len(g.nodeDescs)) g.mu.incoming.setMaxSize(maxPeers) g.outgoing.setMaxSize(maxPeers) // Skip if it's our own address. if desc.Address == g.mu.is.NodeAddr { return } // Add this new node address (if it's not already there) to our list // of resolvers so we can keep connecting to gossip if the original // resolvers go offline. g.maybeAddResolver(desc.Address) // We ignore empty addresses for the sake of not breaking the many tests // that don't bother specifying addresses. if desc.Address.IsEmpty() { return } // If the new node's address conflicts with another node's address, then it // must be the case that the new node has replaced the previous one. Remove // it from our set of tracked descriptors to ensure we don't attempt to // connect to its previous identity (as came up in issue #10266). oldNodeID, ok := g.bootstrapAddrs[desc.Address] if ok && oldNodeID != unknownNodeID && oldNodeID != desc.NodeID { log.Infof(ctx, "removing node %d which was at same address (%s) as new node %v", oldNodeID, desc.Address, desc) delete(g.nodeDescs, oldNodeID) // Deleting the local copy isn't enough to remove the node from the gossip // network. We also have to clear it out in the infoStore by overwriting // it with an empty descriptor, which can be represented as just an empty // byte array due to how protocol buffers are serialied. // Calling addInfoLocked here is somewhat recursive since // updateNodeAddress is typically called in response to the infoStore // being updated but won't lead to deadlock because it's called // asynchronously. key := MakeNodeIDKey(oldNodeID) var emptyProto []byte if err := g.addInfoLocked(key, emptyProto, ttlNodeDescriptorGossip); err != nil { log.Errorf(ctx, "failed to empty node descriptor for node %d: %s", oldNodeID, err) } } // Add new address (if it's not already there) to bootstrap info and // persist if possible. added := g.maybeAddBootstrapAddress(desc.Address, desc.NodeID) if added && g.storage != nil { if err := g.storage.WriteBootstrapInfo(&g.bootstrapInfo); err != nil { log.Error(ctx, err) } } }
func TestSystemConfigGossip(t *testing.T) { defer leaktest.AfterTest(t)() t.Skip("#12351") s, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{}) defer s.Stopper().Stop() ts := s.(*TestServer) ctx := context.TODO() key := sqlbase.MakeDescMetadataKey(keys.MaxReservedDescID) valAt := func(i int) *sqlbase.DatabaseDescriptor { return &sqlbase.DatabaseDescriptor{Name: "foo", ID: sqlbase.ID(i)} } // Register a callback for gossip updates. resultChan := ts.Gossip().RegisterSystemConfigChannel() // The span gets gossiped when it first shows up. select { case <-resultChan: case <-time.After(500 * time.Millisecond): t.Fatal("did not receive gossip message") } // Write a system key with the transaction marked as having a Gossip trigger. if err := kvDB.Txn(ctx, func(txn *client.Txn) error { txn.SetSystemConfigTrigger() return txn.Put(key, valAt(2)) }); err != nil { t.Fatal(err) } // This has to be wrapped in a SucceedSoon because system migrations on the // testserver's startup can trigger system config updates without the key we // wrote. testutils.SucceedsSoon(t, func() error { // New system config received. var systemConfig config.SystemConfig select { case <-resultChan: systemConfig, _ = ts.gossip.GetSystemConfig() case <-time.After(500 * time.Millisecond): return errors.Errorf("did not receive gossip message") } // Now check the new config. var val *roachpb.Value for _, kv := range systemConfig.Values { if bytes.Equal(key, kv.Key) { val = &kv.Value break } } if val == nil { return errors.Errorf("key not found in gossiped info") } // Make sure the returned value is valAt(2). got := new(sqlbase.DatabaseDescriptor) if err := val.GetProto(got); err != nil { return err } if expected := valAt(2); !reflect.DeepEqual(got, expected) { return errors.Errorf("mismatch: expected %+v, got %+v", *expected, *got) } return nil }) }