func raftLogKeyPrint(key roachpb.Key) string {
var logIndex uint64
var err error
key, logIndex, err = encoding.DecodeUint64Ascending(key)
if err != nil {
return fmt.Sprintf("/err<%v:%q>", err, []byte(key))
}
return fmt.Sprintf("%s%d", strLogIndex, logIndex)
}
// GetFloat decodes a float64 value from the bytes field of the receiver. If
// the bytes field is not 8 bytes in length or the tag is not FLOAT an error
// will be returned.
func (v Value) GetFloat() (float64, error) {
if tag := v.GetTag(); tag != ValueType_FLOAT {
return 0, fmt.Errorf("value type is not %s: %s", ValueType_FLOAT, tag)
}
dataBytes := v.dataBytes()
if len(dataBytes) != 8 {
return 0, fmt.Errorf("float64 value should be exactly 8 bytes: %d", len(dataBytes))
}
_, u, err := encoding.DecodeUint64Ascending(dataBytes)
if err != nil {
return 0, err
}
return math.Float64frombits(u), nil
}
func runDebugCheckStoreCmd(cmd *cobra.Command, args []string) error {
stopper := stop.NewStopper()
defer stopper.Stop()
if len(args) != 1 {
return errors.New("one required argument: dir")
}
db, err := openStore(cmd, args[0], stopper)
if err != nil {
return err
}
// Iterate over the entire range-id-local space.
start := roachpb.Key(keys.LocalRangeIDPrefix)
end := start.PrefixEnd()
replicaInfo := map[roachpb.RangeID]*replicaCheckInfo{}
getReplicaInfo := func(rangeID roachpb.RangeID) *replicaCheckInfo {
if info, ok := replicaInfo[rangeID]; ok {
return info
}
replicaInfo[rangeID] = &replicaCheckInfo{}
return replicaInfo[rangeID]
}
if _, err := engine.MVCCIterate(context.Background(), db, start, end, hlc.MaxTimestamp,
false /* !consistent */, nil, /* txn */
false /* !reverse */, func(kv roachpb.KeyValue) (bool, error) {
rangeID, _, suffix, detail, err := keys.DecodeRangeIDKey(kv.Key)
if err != nil {
return false, err
}
switch {
case bytes.Equal(suffix, keys.LocalRaftTruncatedStateSuffix):
var trunc roachpb.RaftTruncatedState
if err := kv.Value.GetProto(&trunc); err != nil {
return false, err
}
getReplicaInfo(rangeID).truncatedIndex = trunc.Index
case bytes.Equal(suffix, keys.LocalRaftAppliedIndexSuffix):
idx, err := kv.Value.GetInt()
if err != nil {
return false, err
}
getReplicaInfo(rangeID).appliedIndex = uint64(idx)
case bytes.Equal(suffix, keys.LocalRaftLogSuffix):
_, index, err := encoding.DecodeUint64Ascending(detail)
if err != nil {
return false, err
}
ri := getReplicaInfo(rangeID)
if ri.firstIndex == 0 {
ri.firstIndex = index
ri.lastIndex = index
} else {
if index != ri.lastIndex+1 {
fmt.Printf("range %s: log index anomaly: %v followed by %v\n",
rangeID, ri.lastIndex, index)
}
ri.lastIndex = index
}
}
return false, nil
}); err != nil {
return err
}
for rangeID, info := range replicaInfo {
if info.truncatedIndex != info.firstIndex-1 {
fmt.Printf("range %s: truncated index %v should equal first index %v - 1\n",
rangeID, info.truncatedIndex, info.firstIndex)
}
if info.appliedIndex < info.firstIndex || info.appliedIndex > info.lastIndex {
fmt.Printf("range %s: applied index %v should be between first index %v and last index %v\n",
rangeID, info.appliedIndex, info.firstIndex, info.lastIndex)
}
}
return nil
}
// outputDotFile generates a .dot file describing the current state of
// the gossip network. nodes is a map from network address to gossip
// node. edgeSet is empty on the first invocation, but
// its content is set to encompass the entire set of edges in the
// network when this method returns. It should be resupplied with each
// successive invocation, as it is used to determine which edges are
// new and which have been deleted and show those changes visually in
// the output graph. New edges are drawn green; edges which were
// removed over the course of the last simulation step(s) are drawn in
// a lightly-dashed red.
//
// The format of the output looks like this:
//
// digraph G {
// node [shape=record];
// node1 [fontsize=12,label="{Node 1|MH=3}"]
// node1 -> node3 [color=green]
// node1 -> node4
// node1 -> node5 [color=red,style=dotted]
// node2 [fontsize=24,label="{Node 2|MH=2}"]
// node2 -> node5
// node3 [fontsize=18,label="{Node 3|MH=5}"]
// node3 -> node5
// node3 -> node4
// node4 [fontsize=24,label="{Node 4|MH=4}"]
// node4 -> node2
// node5 [fontsize=24,label="{Node 5|MH=1}"]
// node5 -> node2
// node5 -> node3
// }
//
// Returns the name of the output file and a boolean for whether or not
// the network has quiesced (that is, no new edges, and all nodes are
// connected).
func outputDotFile(
dotFN string, cycle int, network *simulation.Network, edgeSet map[string]edge,
) (string, bool) {
f, err := os.Create(dotFN)
if err != nil {
log.Fatalf(context.TODO(), "unable to create temp file: %s", err)
}
defer f.Close()
// Determine maximum number of incoming connections. Create outgoing
// edges, keeping track of which are new since last time (added=true).
outgoingMap := make(edgeMap)
var maxIncoming int
quiescent := true
// The order the graph file is written influences the arrangement
// of nodes in the output image, so it makes sense to eliminate
// randomness here. Unfortunately with graphviz it's fairly hard
// to get a consistent ordering.
for _, simNode := range network.Nodes {
node := simNode.Gossip
incoming := node.Incoming()
for _, iNode := range incoming {
e := edge{dest: node.NodeID.Get()}
key := fmt.Sprintf("%d:%d", iNode, node.NodeID.Get())
if _, ok := edgeSet[key]; !ok {
e.added = true
quiescent = false
}
delete(edgeSet, key)
outgoingMap.addEdge(iNode, e)
}
if len(incoming) > maxIncoming {
maxIncoming = len(incoming)
}
}
// Find all edges which were deleted.
for key, e := range edgeSet {
e.added = false
e.deleted = true
quiescent = false
nodeID, err := strconv.Atoi(strings.Split(key, ":")[0])
if err != nil {
log.Fatal(context.TODO(), err)
}
outgoingMap.addEdge(roachpb.NodeID(nodeID), e)
delete(edgeSet, key)
}
fmt.Fprintln(f, "digraph G {")
fmt.Fprintln(f, "node [shape=record];")
for _, simNode := range network.Nodes {
node := simNode.Gossip
var missing []roachpb.NodeID
var totalAge int64
for _, otherNode := range network.Nodes {
if otherNode == simNode {
continue // skip the node's own info
}
infoKey := otherNode.Addr().String()
// GetInfo returns an error if the info is missing.
if info, err := node.GetInfo(infoKey); err != nil {
missing = append(missing, otherNode.Gossip.NodeID.Get())
quiescent = false
} else {
_, val, err := encoding.DecodeUint64Ascending(info)
if err != nil {
log.Fatalf(context.TODO(), "bad decode of node info cycle: %s", err)
}
totalAge += int64(cycle) - int64(val)
}
}
log.Infof(context.TODO(), "node %d: missing infos for nodes %s", node.NodeID.Get(), missing)
var sentinelAge int64
// GetInfo returns an error if the info is missing.
if info, err := node.GetInfo(gossip.KeySentinel); err != nil {
log.Infof(context.TODO(), "error getting info for sentinel gossip key %q: %s", gossip.KeySentinel, err)
} else {
_, val, err := encoding.DecodeUint64Ascending(info)
if err != nil {
log.Fatalf(context.TODO(), "bad decode of sentinel cycle: %s", err)
}
sentinelAge = int64(cycle) - int64(val)
}
var age, nodeColor string
if len(missing) > 0 {
nodeColor = "color=red,"
age = fmt.Sprintf("missing %d", len(missing))
} else {
age = strconv.FormatFloat(float64(totalAge)/float64(len(network.Nodes)-1-len(missing)), 'f', 4, 64)
}
fontSize := minDotFontSize
if maxIncoming > 0 {
fontSize = minDotFontSize + int(math.Floor(float64(len(node.Incoming())*
(maxDotFontSize-minDotFontSize))/float64(maxIncoming)))
}
fmt.Fprintf(f, "\t%s [%sfontsize=%d,label=\"{%s|AA=%s, MH=%d, SA=%d}\"]\n",
node.NodeID.Get(), nodeColor, fontSize, node.NodeID.Get(), age, node.MaxHops(), sentinelAge)
outgoing := outgoingMap[node.NodeID.Get()]
for _, e := range outgoing {
destSimNode, ok := network.GetNodeFromID(e.dest)
if !ok {
continue
}
dest := destSimNode.Gossip
style := ""
if e.added {
style = " [color=green]"
} else if e.deleted {
style = " [color=red,style=dotted]"
}
fmt.Fprintf(f, "\t%s -> %s%s\n", node.NodeID.Get(), dest.NodeID.Get(), style)
if !e.deleted {
edgeSet[fmt.Sprintf("%d:%d", node.NodeID.Get(), e.dest)] = e
}
}
}
fmt.Fprintln(f, "}")
return f.Name(), quiescent
}
