func TestPrettyPrint(t *testing.T) { tm, _ := time.Parse(time.UnixDate, "Sat Mar 7 11:06:39 UTC 2015") txnID := uuid.NewV4() testCases := []struct { key roachpb.Key exp string }{ // local {StoreIdentKey(), "/Local/Store/storeIdent"}, {StoreGossipKey(), "/Local/Store/gossipBootstrap"}, {SequenceCacheKeyPrefix(roachpb.RangeID(1000001), txnID), fmt.Sprintf(`/Local/RangeID/1000001/r/SequenceCache/%q`, txnID)}, {SequenceCacheKey(roachpb.RangeID(1000001), txnID, uint32(111), uint32(222)), fmt.Sprintf(`/Local/RangeID/1000001/r/SequenceCache/%q/epoch:111/seq:222`, txnID)}, {RaftTombstoneKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTombstone"}, {RaftAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"}, {RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"}, {RangeLeaderLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLeaderLease"}, {RangeStatsKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"}, {RaftHardStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftHardState"}, {RaftLastIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftLastIndex"}, {RaftLogKey(roachpb.RangeID(1000001), uint64(200001)), "/Local/RangeID/1000001/u/RaftLog/logIndex:200001"}, {RangeLastVerificationTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastVerificationTimestamp"}, {MakeRangeKeyPrefix(roachpb.RKey("ok")), `/Local/Range/"ok"`}, {RangeDescriptorKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeDescriptor`}, {RangeTreeNodeKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeTreeNode`}, {TransactionKey(roachpb.Key("111"), txnID), fmt.Sprintf(`/Local/Range/"111"/Transaction/addrKey:/id:%q`, txnID)}, {LocalMax, `/Meta1/""`}, // LocalMax == Meta1Prefix // system {makeKey(Meta2Prefix, roachpb.Key("foo")), `/Meta2/"foo"`}, {makeKey(Meta1Prefix, roachpb.Key("foo")), `/Meta1/"foo"`}, {RangeMetaKey(roachpb.RKey("f")), `/Meta2/"f"`}, {StoreStatusKey(2222), "/System/StatusStore/2222"}, {NodeStatusKey(1111), "/System/StatusNode/1111"}, {SystemMax, "/System/Max"}, // key of key {RangeMetaKey(roachpb.RKey(MakeRangeKeyPrefix(roachpb.RKey("ok")))), `/Meta2/Local/Range/"ok"`}, {RangeMetaKey(roachpb.RKey(makeKey(MakeTablePrefix(42), roachpb.RKey("foo")))), `/Meta2/Table/42/"foo"`}, {RangeMetaKey(roachpb.RKey(makeKey(Meta2Prefix, roachpb.Key("foo")))), `/Meta1/"foo"`}, // table {UserTableDataMin, "/Table/50"}, {MakeTablePrefix(111), "/Table/111"}, {makeKey(MakeTablePrefix(42), roachpb.RKey("foo")), `/Table/42/"foo"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, float64(233.221112)))), "/Table/42/233.221112"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatDescending(nil, float64(-233.221112)))), "/Table/42/233.221112"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, math.Inf(1)))), "/Table/42/+Inf"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, math.NaN()))), "/Table/42/NaN"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222)), roachpb.RKey(encoding.EncodeStringAscending(nil, "handsome man"))), `/Table/42/1222/"handsome man"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222))), `/Table/42/1222`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintDescending(nil, 1222))), `/Table/42/-1223`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255}))), `/Table/42/"\x01\x02\b\xff"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255})), roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesDescending(nil, []byte{1, 2, 8, 255})), roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeNullAscending(nil))), "/Table/42/NULL"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeNotNullAscending(nil))), "/Table/42/#"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeTimeAscending(nil, tm))), "/Table/42/Sat Mar 7 11:06:39 UTC 2015"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeTimeDescending(nil, tm))), "/Table/42/Sat Mar 7 11:06:39 UTC 2015"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeDecimalAscending(nil, inf.NewDec(1234, 2)))), "/Table/42/12.34"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeDecimalDescending(nil, inf.NewDec(1234, 2)))), "/Table/42/-12.34"}, // others {makeKey([]byte("")), "/Min"}, {Meta1KeyMax, "/Meta1/Max"}, {Meta2KeyMax, "/Meta2/Max"}, {makeKey(MakeTablePrefix(42), roachpb.RKey([]byte{0x21, 'a', 0x00, 0x02})), "/Table/42/<util/encoding/encoding.go:9999: unknown escape sequence: 0x0 0x2>"}, } for i, test := range testCases { keyInfo := MassagePrettyPrintedSpanForTest(PrettyPrint(test.key), nil) exp := MassagePrettyPrintedSpanForTest(test.exp, nil) if exp != keyInfo { t.Errorf("%d: expected %s, got %s", i, exp, keyInfo) } if exp != MassagePrettyPrintedSpanForTest(test.key.String(), nil) { t.Errorf("%d: expected %s, got %s", i, exp, test.key.String()) } } }
// EncodeTableKey encodes `val` into `b` and returns the new buffer. func EncodeTableKey(b []byte, val parser.Datum, dir encoding.Direction) ([]byte, error) { if (dir != encoding.Ascending) && (dir != encoding.Descending) { return nil, util.Errorf("invalid direction: %d", dir) } if val == parser.DNull { if dir == encoding.Ascending { return encoding.EncodeNullAscending(b), nil } return encoding.EncodeNullDescending(b), nil } switch t := val.(type) { case *parser.DBool: var x int64 if *t { x = 1 } else { x = 0 } if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, x), nil } return encoding.EncodeVarintDescending(b, x), nil case *parser.DInt: if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, int64(*t)), nil } return encoding.EncodeVarintDescending(b, int64(*t)), nil case *parser.DFloat: if dir == encoding.Ascending { return encoding.EncodeFloatAscending(b, float64(*t)), nil } return encoding.EncodeFloatDescending(b, float64(*t)), nil case *parser.DDecimal: if dir == encoding.Ascending { return encoding.EncodeDecimalAscending(b, &t.Dec), nil } return encoding.EncodeDecimalDescending(b, &t.Dec), nil case *parser.DString: if dir == encoding.Ascending { return encoding.EncodeStringAscending(b, string(*t)), nil } return encoding.EncodeStringDescending(b, string(*t)), nil case *parser.DBytes: if dir == encoding.Ascending { return encoding.EncodeStringAscending(b, string(*t)), nil } return encoding.EncodeStringDescending(b, string(*t)), nil case *parser.DDate: if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, int64(*t)), nil } return encoding.EncodeVarintDescending(b, int64(*t)), nil case *parser.DTimestamp: if dir == encoding.Ascending { return encoding.EncodeTimeAscending(b, t.Time), nil } return encoding.EncodeTimeDescending(b, t.Time), nil case *parser.DTimestampTZ: if dir == encoding.Ascending { return encoding.EncodeTimeAscending(b, t.Time), nil } return encoding.EncodeTimeDescending(b, t.Time), nil case *parser.DInterval: if dir == encoding.Ascending { return encoding.EncodeDurationAscending(b, t.Duration) } return encoding.EncodeDurationDescending(b, t.Duration) case *parser.DTuple: for _, datum := range *t { var err error b, err = EncodeTableKey(b, datum, dir) if err != nil { return nil, err } } return b, nil } return nil, util.Errorf("unable to encode table key: %T", val) }
func TestPrettyPrint(t *testing.T) { tm, _ := time.Parse(time.RFC3339Nano, "2016-03-30T13:40:35.053725008Z") duration := duration.Duration{Months: 1, Days: 1, Nanos: 1 * time.Second.Nanoseconds()} durationAsc, _ := encoding.EncodeDurationAscending(nil, duration) durationDesc, _ := encoding.EncodeDurationDescending(nil, duration) txnID := uuid.NewV4() // The following test cases encode keys with a mixture of ascending and descending direction, // but always decode keys in the ascending direction. This is why some of the decoded values // seem bizarre. testCases := []struct { key roachpb.Key exp string }{ // local {StoreIdentKey(), "/Local/Store/storeIdent"}, {StoreGossipKey(), "/Local/Store/gossipBootstrap"}, {AbortCacheKey(roachpb.RangeID(1000001), txnID), fmt.Sprintf(`/Local/RangeID/1000001/r/AbortCache/%q`, txnID)}, {RaftTombstoneKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTombstone"}, {RaftAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"}, {LeaseAppliedIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LeaseAppliedIndex"}, {RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"}, {RangeLeaderLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLeaderLease"}, {RangeStatsKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"}, {RangeFrozenStatusKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeFrozenStatus"}, {RangeLastGCKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLastGC"}, {RaftHardStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftHardState"}, {RaftLastIndexKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftLastIndex"}, {RaftLogKey(roachpb.RangeID(1000001), uint64(200001)), "/Local/RangeID/1000001/u/RaftLog/logIndex:200001"}, {RangeLastReplicaGCTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastReplicaGCTimestamp"}, {RangeLastVerificationTimestampKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RangeLastVerificationTimestamp"}, {MakeRangeKeyPrefix(roachpb.RKey("ok")), `/Local/Range/"ok"`}, {RangeDescriptorKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeDescriptor`}, {RangeTreeNodeKey(roachpb.RKey("111")), `/Local/Range/"111"/RangeTreeNode`}, {TransactionKey(roachpb.Key("111"), txnID), fmt.Sprintf(`/Local/Range/"111"/Transaction/addrKey:/id:%q`, txnID)}, {LocalMax, `/Meta1/""`}, // LocalMax == Meta1Prefix // system {makeKey(Meta2Prefix, roachpb.Key("foo")), `/Meta2/"foo"`}, {makeKey(Meta1Prefix, roachpb.Key("foo")), `/Meta1/"foo"`}, {RangeMetaKey(roachpb.RKey("f")), `/Meta2/"f"`}, {NodeStatusKey(1111), "/System/StatusNode/1111"}, {SystemMax, "/System/Max"}, // key of key {RangeMetaKey(roachpb.RKey(MakeRangeKeyPrefix(roachpb.RKey("ok")))), `/Meta2/Local/Range/"ok"`}, {RangeMetaKey(roachpb.RKey(makeKey(MakeTablePrefix(42), roachpb.RKey("foo")))), `/Meta2/Table/42/"foo"`}, {RangeMetaKey(roachpb.RKey(makeKey(Meta2Prefix, roachpb.Key("foo")))), `/Meta1/"foo"`}, // table {UserTableDataMin, "/Table/50"}, {MakeTablePrefix(111), "/Table/111"}, {makeKey(MakeTablePrefix(42), roachpb.RKey("foo")), `/Table/42/"foo"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, float64(233.221112)))), "/Table/42/233.221112"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatDescending(nil, float64(-233.221112)))), "/Table/42/233.221112"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, math.Inf(1)))), "/Table/42/+Inf"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeFloatAscending(nil, math.NaN()))), "/Table/42/NaN"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222)), roachpb.RKey(encoding.EncodeStringAscending(nil, "handsome man"))), `/Table/42/1222/"handsome man"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintAscending(nil, 1222))), `/Table/42/1222`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeVarintDescending(nil, 1222))), `/Table/42/-1223`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255}))), `/Table/42/"\x01\x02\b\xff"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesAscending(nil, []byte{1, 2, 8, 255})), roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeBytesDescending(nil, []byte{1, 2, 8, 255})), roachpb.RKey("bar")), `/Table/42/"\x01\x02\b\xff"/"bar"`}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeNullAscending(nil))), "/Table/42/NULL"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeNotNullAscending(nil))), "/Table/42/#"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeTimeAscending(nil, tm))), "/Table/42/2016-03-30T13:40:35.053725008Z"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeTimeDescending(nil, tm))), "/Table/42/1923-10-04T10:19:23.946274991Z"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeDecimalAscending(nil, inf.NewDec(1234, 2)))), "/Table/42/12.34"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(encoding.EncodeDecimalDescending(nil, inf.NewDec(1234, 2)))), "/Table/42/-12.34"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(durationAsc)), "/Table/42/1m1d1s"}, {makeKey(MakeTablePrefix(42), roachpb.RKey(durationDesc)), "/Table/42/-2m-2d743h59m58.999999999s"}, // others {makeKey([]byte("")), "/Min"}, {Meta1KeyMax, "/Meta1/Max"}, {Meta2KeyMax, "/Meta2/Max"}, {makeKey(MakeTablePrefix(42), roachpb.RKey([]byte{0x12, 'a', 0x00, 0x02})), "/Table/42/<unknown escape sequence: 0x0 0x2>"}, } for i, test := range testCases { keyInfo := MassagePrettyPrintedSpanForTest(PrettyPrint(test.key), nil) exp := MassagePrettyPrintedSpanForTest(test.exp, nil) if exp != keyInfo { t.Errorf("%d: expected %s, got %s", i, exp, keyInfo) } if exp != MassagePrettyPrintedSpanForTest(test.key.String(), nil) { t.Errorf("%d: expected %s, got %s", i, exp, test.key.String()) } parsed, err := UglyPrint(keyInfo) if err != nil { if _, ok := err.(*errUglifyUnsupported); !ok { t.Errorf("%d: %s: %s", i, keyInfo, err) } else { t.Logf("%d: skipping parsing of %s; key is unsupported: %v", i, keyInfo, err) } } else if exp, act := test.key, parsed; !bytes.Equal(exp, act) { t.Errorf("%d: expected %q, got %q", i, exp, act) } if t.Failed() { return } } }
// Encodes `val` into `b` and returns the new buffer. func encodeTableKey(b []byte, val parser.Datum, dir encoding.Direction) ([]byte, *roachpb.Error) { if (dir != encoding.Ascending) && (dir != encoding.Descending) { return nil, roachpb.NewErrorf("invalid direction: %d", dir) } if val == parser.DNull { if dir == encoding.Ascending { return encoding.EncodeNullAscending(b), nil } return encoding.EncodeNullDescending(b), nil } switch t := val.(type) { case parser.DBool: var x int64 if t { x = 1 } else { x = 0 } if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, x), nil } return encoding.EncodeVarintDescending(b, x), nil case parser.DInt: if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, int64(t)), nil } return encoding.EncodeVarintDescending(b, int64(t)), nil case parser.DFloat: if dir == encoding.Ascending { return encoding.EncodeFloatAscending(b, float64(t)), nil } return encoding.EncodeFloatDescending(b, float64(t)), nil case parser.DDecimal: if dir == encoding.Ascending { return encoding.EncodeDecimalAscending(b, t.Decimal), nil } return encoding.EncodeDecimalDescending(b, t.Decimal), nil case parser.DString: if dir == encoding.Ascending { return encoding.EncodeStringAscending(b, string(t)), nil } return encoding.EncodeStringDescending(b, string(t)), nil case parser.DBytes: if dir == encoding.Ascending { return encoding.EncodeStringAscending(b, string(t)), nil } return encoding.EncodeStringDescending(b, string(t)), nil case parser.DDate: if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, int64(t)), nil } return encoding.EncodeVarintDescending(b, int64(t)), nil case parser.DTimestamp: if dir == encoding.Ascending { return encoding.EncodeTimeAscending(b, t.Time), nil } return encoding.EncodeTimeDescending(b, t.Time), nil case parser.DInterval: if dir == encoding.Ascending { return encoding.EncodeVarintAscending(b, int64(t.Duration)), nil } return encoding.EncodeVarintDescending(b, int64(t.Duration)), nil } return nil, roachpb.NewUErrorf("unable to encode table key: %T", val) }
// TestStoreRangeSplitRaceUninitializedRHS reproduces #7600 (before it was // fixed). While splits are happening, we simulate incoming messages for the // right-hand side to trigger a race between the creation of the proper replica // and the uninitialized replica reacting to messages. func TestStoreRangeSplitRaceUninitializedRHS(t *testing.T) { defer leaktest.AfterTest(t)() mtc := &multiTestContext{} storeCtx := storage.TestStoreContext() // An aggressive tick interval lets groups communicate more and thus // triggers test failures much more reliably. We can't go too aggressive // or race tests never make any progress. storeCtx.RaftTickInterval = 50 * time.Millisecond storeCtx.RaftElectionTimeoutTicks = 2 currentTrigger := make(chan *roachpb.SplitTrigger) seen := make(map[storagebase.CmdIDKey]struct{}) storeCtx.TestingKnobs.TestingCommandFilter = func(args storagebase.FilterArgs) *roachpb.Error { et, ok := args.Req.(*roachpb.EndTransactionRequest) if !ok || et.InternalCommitTrigger == nil { return nil } trigger := protoutil.Clone(et.InternalCommitTrigger.GetSplitTrigger()).(*roachpb.SplitTrigger) if trigger != nil && len(trigger.NewDesc.Replicas) == 2 && args.Hdr.Txn.Epoch == 0 && args.Sid == mtc.stores[0].StoreID() { if _, ok := seen[args.CmdID]; ok { return nil } // Without replay protection, a single reproposal locks up the // test. seen[args.CmdID] = struct{}{} currentTrigger <- trigger return roachpb.NewError(roachpb.NewReadWithinUncertaintyIntervalError(args.Hdr.Timestamp, args.Hdr.Timestamp)) } return nil } mtc.storeContext = &storeCtx mtc.Start(t, 2) defer mtc.Stop() leftRange := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil) // We'll fake messages from term 1, ..., .magicIters-1. The exact number // doesn't matter for anything but for its likelihood of triggering the // race. const magicIters = 5 // Replicate the left range onto the second node. We don't wait since we // don't actually care what the second node does. All we want is that the // first node isn't surprised by messages from that node. mtc.replicateRange(leftRange.RangeID, 1) for i := 0; i < 10; i++ { var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() // Split the data range. The split keys are chosen so that they move // towards "a" (so that the range being split is always the first // range). splitKey := roachpb.Key(encoding.EncodeVarintDescending([]byte("a"), int64(i))) splitArgs := adminSplitArgs(keys.SystemMax, splitKey) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &splitArgs); pErr != nil { t.Fatal(pErr) } }() go func() { defer wg.Done() trigger := <-currentTrigger // our own copy // Make sure the first node is first for convenience. replicas := trigger.NewDesc.Replicas if replicas[0].NodeID > replicas[1].NodeID { tmp := replicas[1] replicas[1] = replicas[0] replicas[0] = tmp } // Send a few vote requests which look like they're from the other // node's right hand side of the split. This triggers a race which // is discussed in #7600 (briefly, the creation of the right hand // side in the split trigger was racing with the uninitialized // version for the same group, resulting in clobbered HardState). for term := uint64(1); term < magicIters; term++ { if err := mtc.stores[0].HandleRaftMessage(&storage.RaftMessageRequest{ RangeID: trigger.NewDesc.RangeID, ToReplica: replicas[0], FromReplica: replicas[1], Message: raftpb.Message{ Type: raftpb.MsgVote, To: uint64(replicas[0].ReplicaID), From: uint64(replicas[1].ReplicaID), Term: term, }, }); err != nil { t.Error(err) } } }() wg.Wait() } }