func newEtcdStore(
proposalc chan<- []byte,
snapshotc <-chan raftpb.Snapshot,
entryc <-chan raftpb.Entry,
confentryc chan<- raftpb.Entry,
logger mesh.Logger,
) *etcdStore {
// It would be much better if we could have a proper in-memory backend. Alas:
// backend.Backend is tightly coupled to bolt.DB, and both are tightly coupled
// to os.Open &c. So we'd need to fork both Bolt and backend. A task for
// another day.
f, err := ioutil.TempFile(os.TempDir(), "mesh_etcd_backend_")
if err != nil {
panic(err)
}
dbPath := f.Name()
f.Close()
logger.Printf("etcd store: using %s", dbPath)
b := backend.NewDefaultBackend(dbPath)
lessor := lease.NewLessor(b)
index := &consistentIndex{0}
kv := mvcc.New(b, lessor, index)
s := &etcdStore{
proposalc: proposalc,
snapshotc: snapshotc,
entryc: entryc,
confentryc: confentryc,
actionc: make(chan func()),
quitc: make(chan struct{}),
terminatedc: make(chan struct{}),
logger: logger,
dbPath: dbPath,
kv: kv,
lessor: lessor,
index: index,
idgen: makeIDGen(),
pending: map[uint64]responseChans{},
}
go s.loop()
return s
}
func serverManager(
router *mesh.Router,
peer *meshconn.Peer,
minPeerCount int,
terminatec <-chan struct{},
terminatedc chan<- error,
logger mesh.Logger,
out chan<- Server,
) {
// Identify mesh peers to either create or join a cluster.
// This algorithm is presently completely insufficient.
// It suffers from timing failures, and doesn't understand channels.
// TODO(pb): use gossip to agree on better starting conditions
var (
self = meshconn.MeshAddr{PeerName: router.Ourself.Peer.Name, PeerUID: router.Ourself.UID}
others = []net.Addr{}
)
for {
others = others[:0]
for _, desc := range router.Peers.Descriptions() {
others = append(others, meshconn.MeshAddr{PeerName: desc.Name, PeerUID: desc.UID})
}
if len(others) == minPeerCount {
logger.Printf("detected %d peers; creating", len(others))
break
} else if len(others) > minPeerCount {
logger.Printf("detected %d peers; joining", len(others))
others = others[:0] // empty others slice means join
break
}
logger.Printf("detected %d peers; waiting...", len(others))
time.Sleep(time.Second)
}
var (
incomingc = make(chan raftpb.Message) // from meshconn to ctrl
outgoingc = make(chan raftpb.Message) // from ctrl to meshconn
unreachablec = make(chan uint64, 10000) // from meshconn to ctrl
confchangec = make(chan raftpb.ConfChange) // from meshconn to ctrl
snapshotc = make(chan raftpb.Snapshot) // from ctrl to state machine
entryc = make(chan raftpb.Entry) // from ctrl to state
confentryc = make(chan raftpb.Entry) // from state to configurator
proposalc = make(chan []byte) // from state machine to ctrl
removedc = make(chan struct{}) // from ctrl to us
shrunkc = make(chan struct{}) // from membership to us
)
// Create the thing that watches the cluster membership via the router. It
// signals conf changes, and closes shrunkc when the cluster is too small.
var (
addc = make(chan uint64)
remc = make(chan uint64)
)
m := newMembership(router, membershipSet(router), minPeerCount, addc, remc, shrunkc, logger)
defer m.stop()
// Create the thing that converts mesh membership changes to Raft ConfChange
// proposals.
c := newConfigurator(addc, remc, confchangec, confentryc, logger)
defer c.stop()
// Create a packet transport, wrapping the meshconn.Peer.
transport := newPacketTransport(peer, translateVia(router), incomingc, outgoingc, unreachablec, logger)
defer transport.stop()
// Create the API server. store.stop must go on the defer stack before
// ctrl.stop so that the ctrl stops first. Otherwise, ctrl can deadlock
// processing the last tick.
store := newEtcdStore(proposalc, snapshotc, entryc, confentryc, logger)
defer store.stop()
// Create the controller, which drives the Raft node internally.
ctrl := newCtrl(self, others, minPeerCount, incomingc, outgoingc, unreachablec, confchangec, snapshotc, entryc, proposalc, removedc, logger)
defer ctrl.stop()
// Return the store to the client.
out <- store
errc := make(chan error)
go func() {
<-terminatec
errc <- fmt.Errorf("metcd server terminated by user request")
}()
go func() {
<-removedc
errc <- fmt.Errorf("the Raft peer was removed from the cluster")
}()
go func() {
<-shrunkc
errc <- fmt.Errorf("the Raft cluster got too small")
}()
terminatedc <- <-errc
}