// raft replicas are numbered 1..n and reside in array indices 0..n-1
// A copy of this function exists in raftlog_test.go
func setupRaftLogCluster(t *testing.T, nReplicas, nStandbys int) (ret []replication.LogReplicator, dbs []kv.DB, clks []*clock.Mock, nw *nettestutil.Network, teardown func()) {
m := nReplicas
n := nReplicas + nStandbys
replicaIDs := make([]uint64, 0, n)
for i := uint64(0); i < uint64(n); i++ {
replicaIDs = append(replicaIDs, 1+i)
}
addrs := make([]string, 0, n)
nw = nettestutil.New(n)
lookupDialerFrom := func(src int) func(uint64) raftproto.RaftClient {
return func(dstPlus1 uint64) raftproto.RaftClient {
cc, err := grpc.Dial(addrs[dstPlus1-1], grpc.WithInsecure(), grpc.WithDialer(
func(addr string, timeout time.Duration) (net.Conn, error) {
nc, err := net.DialTimeout("tcp", addr, timeout)
return nw.Wrap(nc, src, int(dstPlus1-1)), err
}))
if err != nil {
panic(err) // async dial should not err
}
return raftproto.NewRaftClient(cc)
}
}
teardown = func() {}
for i := 0; i < n; i++ {
clk := clock.NewMock()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
s := grpc.NewServer()
db, dbDown := setupDB(t)
dbs = append(dbs, db)
l := raftlog.New(
uint64(i+1), replicaIDs[:m],
db, nil,
clk, tick,
s, lookupDialerFrom(i),
)
go s.Serve(ln)
ret = append(ret, l)
clks = append(clks, clk)
addrs = append(addrs, ln.Addr().String())
teardown = chain(func() { s.Stop(); ln.Close(); l.Stop() }, dbDown, teardown)
}
for _, l := range ret {
go func(l replication.LogReplicator) {
for _ = range l.LeaderHintSet() {
}
}(l)
}
return ret, dbs, clks, nw, teardown
}
func RunWithConfig(cfg *proto.ReplicaConfig) {
// TODO: since we only want to support precisely this ratification policy,
// this should be moved into server.go
ratificationPolicy := &proto.AuthorizationPolicy{
PublicKeys: make(map[uint64]*proto.PublicKey),
PolicyType: &proto.AuthorizationPolicy_Quorum{Quorum: &proto.QuorumExpr{
Threshold: uint32(majority(len(cfg.KeyserverConfig.InitialReplicas)))},
},
}
replicaIDs := []uint64{}
for _, replica := range cfg.KeyserverConfig.InitialReplicas {
replicaIDs = append(replicaIDs, replica.ID)
replicaExpr := &proto.QuorumExpr{
Threshold: 1,
}
for _, pk := range replica.PublicKeys {
pkid := proto.KeyID(pk)
ratificationPolicy.PublicKeys[pkid] = pk
replicaExpr.Candidates = append(replicaExpr.Candidates, pkid)
}
ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions = append(ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions, replicaExpr)
}
leveldb, err := leveldb.OpenFile(cfg.LevelDBPath, nil)
if err != nil {
log.Fatalf("Couldn't open DB in directory %s: %s", cfg.LevelDBPath, err)
}
db := leveldbkv.Wrap(leveldb)
clk := clock.New()
raftListener, err := net.Listen("tcp", cfg.RaftAddr)
if err != nil {
log.Fatalf("Couldn't bind to Raft node address %s: %s", cfg.RaftAddr, err)
}
defer raftListener.Close()
raftTLS, err := cfg.RaftTLS.Config(getKey)
if err != nil {
log.Fatalf("Bad Raft TLS configuration: %s", err)
}
raftCreds := credentials.NewTLS(raftTLS)
raftServer := grpc.NewServer(grpc.Creds(raftCreds))
go raftServer.Serve(raftListener)
defer raftServer.Stop()
dialRaftPeer := func(id uint64) raftproto.RaftClient {
// TODO use current, not initial, config
for _, replica := range cfg.KeyserverConfig.InitialReplicas {
if replica.ID == id {
conn, err := grpc.Dial(replica.RaftAddr, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{RootCAs: raftTLS.RootCAs})))
if err != nil {
log.Panicf("Raft GRPC dial failed: %s", err)
}
return raftproto.NewRaftClient(conn)
}
}
log.Panicf("No raft peer %x in configuration", id)
return nil
}
raft := raftlog.New(
cfg.ReplicaID, replicaIDs, db, []byte{tableReplicationLogPrefix},
clk, cfg.RaftHeartbeat.Duration(), raftServer, dialRaftPeer,
)
defer raft.Stop()
server, err := Open(cfg, db, raft, ratificationPolicy, clk, getKey, net.LookupTXT)
if err != nil {
log.Fatalf("Failed to initialize keyserver: %s", err)
}
server.Start()
defer server.Stop()
ch := make(chan os.Signal, 1)
signal.Notify(ch, os.Interrupt)
<-ch
}