func mustWait(s *store.Store, n int64) <-chan store.Event {
c, err := s.Wait(store.Any, n)
if err != nil {
panic(err)
}
return c
}
func Clean(c chan string, st *store.Store, p consensus.Proposer) {
for addr := range c {
_, g := st.Snap()
name := getName(addr, g)
if name != "" {
go func() {
clearSlot(p, g, name)
removeInfo(p, g, name)
}()
}
}
}
func walk(path string, st *store.Store, ch chan store.Event) {
for path != "/" && strings.HasSuffix(path, "/") {
// TODO generalize and factor this into pkg store.
path = path[0 : len(path)-1]
}
v, rev := st.Get(path)
if rev != store.Dir {
ch <- store.Event{0, path, v[0], rev, "", nil, nil}
return
}
if path == "/" {
path = ""
}
for _, ent := range v {
walk(path+"/"+ent, st, ch)
}
}
func sendLearn(out chan<- Packet, p *packet, st *store.Store) {
if p.Msg.Cmd != nil && *p.Msg.Cmd == Msg_INVITE {
ch, err := st.Wait(store.Any, *p.Seqn)
if err == store.ErrTooLate {
log.Println(err)
} else {
e := <-ch
m := Msg{
Seqn: &e.Seqn,
Cmd: learn,
Value: []byte(e.Mut),
}
out <- Packet{p.Addr, m}
}
}
}
func activate(st *store.Store, self string, c *doozer.Conn) int64 {
rev, _ := st.Snap()
for _, base := range store.Getdir(st, calDir) {
p := calDir + "/" + base
v, rev := st.Get(p)
if rev != store.Dir && v[0] == "" {
seqn, err := c.Set(p, rev, []byte(self))
if err != nil {
log.Println(err)
continue
}
return seqn
}
}
for {
ch, err := st.Wait(calGlob, rev+1)
if err != nil {
panic(err)
}
ev, ok := <-ch
if !ok {
panic(io.EOF)
}
rev = ev.Rev
// TODO ev.IsEmpty()
if ev.IsSet() && ev.Body == "" {
seqn, err := c.Set(ev.Path, ev.Rev, []byte(self))
if err != nil {
log.Println(err)
continue
}
return seqn
} else if ev.IsSet() && ev.Body == self {
return ev.Seqn
}
}
return 0
}
func Clean(st *store.Store, keep int64, ticker <-chan time.Time) {
for _ = range ticker {
last := (<-st.Seqns) - keep
st.Clean(last)
}
}
// Main has doc
func Main(
clusterName, self, buri, rwsk, rosk string,
cl *doozer.Conn,
udpConn *net.UDPConn,
listener, webListener net.Listener,
pulseInterval, fillDelay, kickTimeout, hi int64,
st *store.Store,
) {
listenAddr := listener.Addr().String()
canWrite := make(chan bool, 1)
in := make(chan consensus.Packet, 1024)
pr := &proposer{
seqns: make(chan int64, alpha),
props: make(chan *consensus.Prop),
st: st,
}
calSrv := func(start int64) {
go gc.Pulse(self, st.Seqns, pr, pulseInterval)
go gc.Clean(st, hi, time.Tick(1e9))
var m consensus.Manager
m.Self = self
m.DefRev = start
m.Alpha = alpha
m.In = in
m.Out = in
m.Ops = st.Ops
m.PSeqn = pr.seqns
m.Props = pr.props
m.TFill = fillDelay
m.Store = st
m.Ticker = time.Tick(10e6)
go m.Run()
}
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
if cl == nil { // we are the only node in a new cluster
set(st, "/ctl/name", clusterName, store.Missing)
set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing)
set(st, "/ctl/node/"+self+"/hostname", hostname, store.Missing)
set(st, "/ctl/node/"+self+"/version", Version, store.Missing)
set(st, "/ctl/cal/0", self, store.Missing)
if buri == "" {
set(st, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing)
}
calSrv(<-st.Seqns)
// Skip ahead alpha steps so that the registrar can provide a
// meaningful cluster.
for i := 0; i < alpha; i++ {
st.Ops <- store.Op{1 + <-st.Seqns, store.Nop}
}
canWrite <- true
go setReady(pr, self)
} else {
setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber)
setC(cl, "/ctl/node/"+self+"/hostname", hostname, store.Clobber)
setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber)
rev, err := cl.Rev()
if err != nil {
panic(err)
}
stop := make(chan bool, 1)
go follow(st, cl, rev+1, stop)
err = doozer.Walk(cl, rev, "/", func(path string, f *doozer.FileInfo, e error) (err error) {
if f.IsDir {
return
}
if e != nil {
return e
}
// store.Clobber is okay here because the event
// has already passed through another store
body, _, err := cl.Get(path, &f.Rev)
if err != nil {
return
}
mut := store.MustEncodeSet(path, string(body), store.Clobber)
st.Ops <- store.Op{f.Rev, mut}
return
})
if err != nil {
panic(err)
}
st.Flush()
ch, err := st.Wait(store.Any, rev+1)
if err == nil {
<-ch
}
go func() {
n := activate(st, self, cl)
calSrv(n)
advanceUntil(cl, st.Seqns, n+alpha)
stop <- true
canWrite <- true
go setReady(pr, self)
if buri != "" {
b, err := doozer.DialUri(buri, "")
if err != nil {
panic(err)
}
setC(
b,
"/ctl/ns/"+clusterName+"/"+self,
listenAddr,
store.Missing,
)
}
}()
}
shun := make(chan string, 3) // sufficient for a cluster of 7
go member.Clean(shun, st, pr)
go server.ListenAndServe(listener, canWrite, st, pr, rwsk, rosk, self)
if rwsk == "" && rosk == "" && webListener != nil {
web.Store = st
web.ClusterName = clusterName
go web.Serve(webListener)
}
select {}
}