Example #1
0
// raft replicas are numbered 1..n  and reside in array indices 0..n-1
// A copy of this function exists in raftlog_test.go
func setupRaftLogCluster(t *testing.T, nReplicas, nStandbys int) (ret []replication.LogReplicator, dbs []kv.DB, clks []*clock.Mock, nw *nettestutil.Network, teardown func()) {
	m := nReplicas
	n := nReplicas + nStandbys
	replicaIDs := make([]uint64, 0, n)
	for i := uint64(0); i < uint64(n); i++ {
		replicaIDs = append(replicaIDs, 1+i)
	}

	addrs := make([]string, 0, n)
	nw = nettestutil.New(n)
	lookupDialerFrom := func(src int) func(uint64) raftproto.RaftClient {
		return func(dstPlus1 uint64) raftproto.RaftClient {
			cc, err := grpc.Dial(addrs[dstPlus1-1], grpc.WithInsecure(), grpc.WithDialer(
				func(addr string, timeout time.Duration) (net.Conn, error) {
					nc, err := net.DialTimeout("tcp", addr, timeout)
					return nw.Wrap(nc, src, int(dstPlus1-1)), err
				}))
			if err != nil {
				panic(err) // async dial should not err
			}
			return raftproto.NewRaftClient(cc)
		}
	}
	teardown = func() {}

	for i := 0; i < n; i++ {
		clk := clock.NewMock()
		ln, err := net.Listen("tcp", "127.0.0.1:0")
		if err != nil {
			t.Fatal(err)
		}
		s := grpc.NewServer()
		db, dbDown := setupDB(t)
		dbs = append(dbs, db)
		l := raftlog.New(
			uint64(i+1), replicaIDs[:m],
			db, nil,
			clk, tick,
			s, lookupDialerFrom(i),
		)
		go s.Serve(ln)

		ret = append(ret, l)
		clks = append(clks, clk)
		addrs = append(addrs, ln.Addr().String())
		teardown = chain(func() { s.Stop(); ln.Close(); l.Stop() }, dbDown, teardown)
	}

	for _, l := range ret {
		go func(l replication.LogReplicator) {
			for _ = range l.LeaderHintSet() {
			}
		}(l)
	}
	return ret, dbs, clks, nw, teardown
}
Example #2
0
File: main.go Project: yahoo/coname
func RunWithConfig(cfg *proto.ReplicaConfig) {
	// TODO: since we only want to support precisely this ratification policy,
	// this should be moved into server.go
	ratificationPolicy := &proto.AuthorizationPolicy{
		PublicKeys: make(map[uint64]*proto.PublicKey),
		PolicyType: &proto.AuthorizationPolicy_Quorum{Quorum: &proto.QuorumExpr{
			Threshold: uint32(majority(len(cfg.KeyserverConfig.InitialReplicas)))},
		},
	}
	replicaIDs := []uint64{}
	for _, replica := range cfg.KeyserverConfig.InitialReplicas {
		replicaIDs = append(replicaIDs, replica.ID)
		replicaExpr := &proto.QuorumExpr{
			Threshold: 1,
		}
		for _, pk := range replica.PublicKeys {
			pkid := proto.KeyID(pk)
			ratificationPolicy.PublicKeys[pkid] = pk
			replicaExpr.Candidates = append(replicaExpr.Candidates, pkid)
		}
		ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions = append(ratificationPolicy.PolicyType.(*proto.AuthorizationPolicy_Quorum).Quorum.Subexpressions, replicaExpr)
	}

	leveldb, err := leveldb.OpenFile(cfg.LevelDBPath, nil)
	if err != nil {
		log.Fatalf("Couldn't open DB in directory %s: %s", cfg.LevelDBPath, err)
	}
	db := leveldbkv.Wrap(leveldb)

	clk := clock.New()

	raftListener, err := net.Listen("tcp", cfg.RaftAddr)
	if err != nil {
		log.Fatalf("Couldn't bind to Raft node address %s: %s", cfg.RaftAddr, err)
	}
	defer raftListener.Close()
	raftTLS, err := cfg.RaftTLS.Config(getKey)
	if err != nil {
		log.Fatalf("Bad Raft TLS configuration: %s", err)
	}
	raftCreds := credentials.NewTLS(raftTLS)
	raftServer := grpc.NewServer(grpc.Creds(raftCreds))
	go raftServer.Serve(raftListener)
	defer raftServer.Stop()

	dialRaftPeer := func(id uint64) raftproto.RaftClient {
		// TODO use current, not initial, config
		for _, replica := range cfg.KeyserverConfig.InitialReplicas {
			if replica.ID == id {
				conn, err := grpc.Dial(replica.RaftAddr, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{RootCAs: raftTLS.RootCAs})))
				if err != nil {
					log.Panicf("Raft GRPC dial failed: %s", err)
				}
				return raftproto.NewRaftClient(conn)
			}
		}
		log.Panicf("No raft peer %x in configuration", id)
		return nil
	}

	raft := raftlog.New(
		cfg.ReplicaID, replicaIDs, db, []byte{tableReplicationLogPrefix},
		clk, cfg.RaftHeartbeat.Duration(), raftServer, dialRaftPeer,
	)
	defer raft.Stop()

	server, err := Open(cfg, db, raft, ratificationPolicy, clk, getKey, net.LookupTXT)
	if err != nil {
		log.Fatalf("Failed to initialize keyserver: %s", err)
	}
	server.Start()
	defer server.Stop()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	<-ch
}