func Main(clusterName, self, buri, rwsk, rosk string, cl *doozer.Conn, udpConn net.PacketConn, listener, webListener net.Listener, pulseInterval, fillDelay, kickTimeout int64, hi int64) { listenAddr := listener.Addr().String() canWrite := make(chan bool, 1) in := make(chan consensus.Packet, 50) out := make(chan consensus.Packet, 50) st := store.New() pr := &proposer{ seqns: make(chan int64, alpha), props: make(chan *consensus.Prop), st: st, } calSrv := func(start int64) { go gc.Pulse(self, st.Seqns, pr, pulseInterval) go gc.Clean(st, hi, time.Tick(1e9)) var m consensus.Manager m.Self = self m.DefRev = start m.Alpha = alpha m.In = in m.Out = out m.Ops = st.Ops m.PSeqn = pr.seqns m.Props = pr.props m.TFill = fillDelay m.Store = st m.Ticker = time.Tick(10e6) go m.Run() } if cl == nil { // we are the only node in a new cluster set(st, "/ctl/name", clusterName, store.Missing) set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing) set(st, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing) set(st, "/ctl/node/"+self+"/version", Version, store.Missing) set(st, "/ctl/cal/0", self, store.Missing) calSrv(<-st.Seqns) // Skip ahead alpha steps so that the registrar can provide a // meaningful cluster. for i := 0; i < alpha; i++ { st.Ops <- store.Op{1 + <-st.Seqns, store.Nop} } canWrite <- true } else { setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber) setC(cl, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Clobber) setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber) rev, err := cl.Rev() if err != nil { panic(err) } stop := make(chan bool, 1) go follow(st, cl, rev+1, stop) errs := make(chan os.Error) go func() { e, ok := <-errs if ok { panic(e) } }() doozer.Walk(cl, rev, "/", cloner{st.Ops, cl}, errs) close(errs) st.Flush() ch, err := st.Wait(store.Any, rev+1) if err == nil { <-ch } go func() { n := activate(st, self, cl) calSrv(n) advanceUntil(cl, st.Seqns, n+alpha) stop <- true canWrite <- true if buri != "" { b, err := doozer.DialUri(buri, "") if err != nil { panic(err) } setC( b, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing, ) } }() } shun := make(chan string, 3) // sufficient for a cluster of 7 go member.Clean(shun, st, pr) go server.ListenAndServe(listener, canWrite, st, pr, rwsk, rosk) if rwsk == "" && rosk == "" && webListener != nil { web.Store = st web.ClusterName = clusterName go web.Serve(webListener) } go func() { for p := range out { addr, err := net.ResolveUDPAddr("udp", p.Addr) if err != nil { log.Println(err) continue } n, err := udpConn.WriteTo(p.Data, addr) if err != nil { log.Println(err) continue } if n != len(p.Data) { log.Println("packet len too long:", len(p.Data)) continue } } }() lv := liveness{ timeout: kickTimeout, ival: kickTimeout / 2, times: make(map[string]int64), self: self, shun: shun, } for { t := time.Nanoseconds() buf := make([]byte, maxUDPLen) n, addr, err := udpConn.ReadFrom(buf) if err == os.EINVAL { return } if err != nil { log.Println(err) continue } buf = buf[:n] // Update liveness time stamp for this addr lv.times[addr.String()] = t lv.check(t) in <- consensus.Packet{addr.String(), buf} } }
func Main(clusterName, self, baddr string, cl *doozer.Client, udpConn net.PacketConn, listener, webListener net.Listener, pulseInterval, fillDelay, kickTimeout int64) { listenAddr := listener.Addr().String() var activateSeqn int64 useSelf := make(chan bool, 1) st := store.New() pr := &proposer{ seqns: make(chan int64, alpha), props: make(chan *consensus.Prop), st: st, } calSrv := func() { go gc.Pulse(self, st.Seqns, pr, pulseInterval) go gc.Clean(st, 360000, time.Tick(1e9)) } if cl == nil { // we are the only node in a new cluster set(st, "/ctl/name", clusterName, store.Missing) set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing) set(st, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing) set(st, "/ctl/node/"+self+"/version", Version, store.Missing) set(st, "/ctl/cal/0", self, store.Missing) calSrv() close(useSelf) } else { setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber) setC(cl, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Clobber) setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber) rev, err := cl.Rev() if err != nil { panic(err) } walk, err := cl.Walk("/**", &rev, nil, nil) if err != nil { panic(err) } watch, err := cl.Watch("/**", rev+1) if err != nil { panic(err) } go follow(st.Ops, watch.C) follow(st.Ops, walk.C) st.Flush() ch, err := st.Wait(rev + 1) if err == nil { <-ch } go func() { activateSeqn = activate(st, self, cl) calSrv() advanceUntil(cl, st.Seqns, activateSeqn+alpha) err := watch.Cancel() if err != nil { panic(err) } close(useSelf) if baddr != "" { b := doozer.New("<boot>", baddr) setC( b, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing, ) } }() } start := <-st.Seqns cmw := st.Watch(store.Any) in := make(chan consensus.Packet, 50) out := make(chan consensus.Packet, 50) consensus.NewManager(self, start, alpha, in, out, st.Ops, pr.seqns, pr.props, cmw, fillDelay, st) if cl == nil { // Skip ahead alpha steps so that the registrar can provide a // meaningful cluster. for i := start + 1; i < start+alpha+1; i++ { st.Ops <- store.Op{i, store.Nop} } } shun := make(chan string, 3) // sufficient for a cluster of 7 go member.Clean(shun, st, pr) sv := &server.Server{listenAddr, st, pr, self, alpha} go sv.Serve(listener, useSelf) if webListener != nil { web.Store = st web.ClusterName = clusterName go web.Serve(webListener) } go func() { for p := range out { addr, err := net.ResolveUDPAddr(p.Addr) if err != nil { log.Println(err) continue } n, err := udpConn.WriteTo(p.Data, addr) if err != nil { log.Println(err) continue } if n != len(p.Data) { log.Println("packet len too long:", len(p.Data)) continue } } }() lv := liveness{ timeout: kickTimeout, ival: kickTimeout / 2, times: make(map[string]int64), self: self, shun: shun, } for { t := time.Nanoseconds() buf := make([]byte, maxUDPLen) n, addr, err := udpConn.ReadFrom(buf) if err == os.EINVAL { return } if err != nil { log.Println(err) continue } buf = buf[:n] // Update liveness time stamp for this addr lv.times[addr.String()] = t lv.check(t) in <- consensus.Packet{addr.String(), buf} } }
func Main(clusterName, attachAddr string, udpConn net.PacketConn, listener, webListener net.Listener) { logger := util.NewLogger("main") var err os.Error listenAddr := listener.Addr().String() outs := make(paxos.ChanPutCloserTo) cal := make(chan int) var cl *client.Client self := util.RandId() st := store.New() if attachAddr == "" { // we are the only node in a new cluster set(st, "/doozer/info/"+self+"/public-addr", listenAddr, store.Missing) set(st, "/doozer/info/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing) set(st, "/doozer/members/"+self, listenAddr, store.Missing) set(st, "/doozer/slot/"+"1", self, store.Missing) set(st, "/doozer/leader", self, store.Missing) set(st, "/ping", "pong", store.Missing) close(cal) cl, err = client.Dial(listenAddr) if err != nil { panic(err) } } else { cl, err = client.Dial(attachAddr) if err != nil { panic(err) } path := "/doozer/info/" + self + "/public-addr" _, err = cl.Set(path, listenAddr, store.Clobber) if err != nil { panic(err) } path = "/doozer/info/" + self + "/hostname" _, err = cl.Set(path, os.Getenv("HOSTNAME"), store.Clobber) if err != nil { panic(err) } joinSeqn, snap, err := cl.Join(self, listenAddr) if err != nil { panic(err) } done := make(chan int) st.Ops <- store.Op{1, snap} go advanceUntil(cl, done) go func() { st.Sync(joinSeqn + alpha) close(done) activate(st, self, cl, cal) }() // TODO sink needs a way to pick up missing values if there are any // gaps in its sequence } mg := paxos.NewManager(self, alpha, st, outs) if attachAddr == "" { // Skip ahead alpha steps so that the registrar can provide a // meaningful cluster. n := <-st.Seqns for i := n + 1; i < n+alpha; i++ { st.Ops <- store.Op{i, store.Nop} } } go func() { <-cal go lock.Clean(st, mg) go session.Clean(st, mg) go member.Clean(st, mg) go gc.Pulse(self, st.Seqns, cl, pulseInterval) go gc.Clean(st) }() sv := &server.Server{udpConn, listenAddr, st, mg, self} go func() { cas := store.Missing for _ = range time.Tick(checkinInterval) { _, cas, err = cl.Checkin(self, cas) if err != nil { logger.Println(err) } } }() go func() { err := sv.Serve(listener, cal) if err != nil { panic(err) } }() if webListener != nil { web.Store = st web.ClusterName = clusterName go web.Serve(webListener) } sv.ServeUdp(outs) }
func Main(clusterName, self, buri, secret string, cl *doozer.Conn, udpConn net.PacketConn, listener, webListener net.Listener, pulseInterval, fillDelay, kickTimeout int64) { listenAddr := listener.Addr().String() canWrite := make(chan bool, 1) in := make(chan consensus.Packet, 50) out := make(chan consensus.Packet, 50) st := store.New() pr := &proposer{ seqns: make(chan int64, alpha), props: make(chan *consensus.Prop), st: st, } calSrv := func(start int64) { go gc.Pulse(self, st.Seqns, pr, pulseInterval) go gc.Clean(st, 360000, time.Tick(1e9)) consensus.NewManager(self, start, alpha, in, out, st.Ops, pr.seqns, pr.props, fillDelay, st) } if cl == nil { // we are the only node in a new cluster set(st, "/ctl/name", clusterName, store.Missing) set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing) set(st, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing) set(st, "/ctl/node/"+self+"/version", Version, store.Missing) set(st, "/ctl/cal/0", self, store.Missing) calSrv(<-st.Seqns) // Skip ahead alpha steps so that the registrar can provide a // meaningful cluster. for i := 0; i < alpha; i++ { st.Ops <- store.Op{1 + <-st.Seqns, store.Nop} } canWrite <- true } else { setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber) setC(cl, "/ctl/node/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Clobber) setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber) rev, err := cl.Rev() if err != nil { panic(err) } stop := make(chan bool, 1) go follow(st, cl, rev+1, stop) info, err := cl.Walk("/**", rev, 0, -1) if err != nil { panic(err) } for _, ev := range info { // store.Clobber is okay here because the event // has already passed through another store mut := store.MustEncodeSet(ev.Path, string(ev.Body), store.Clobber) st.Ops <- store.Op{ev.Rev, mut} } st.Flush() ch, err := st.Wait(store.Any, rev+1) if err == nil { <-ch } go func() { n := activate(st, self, cl) calSrv(n) advanceUntil(cl, st.Seqns, n+alpha) stop <- true canWrite <- true if buri != "" { b, err := doozer.DialUri(buri) if err != nil { panic(err) } setC( b, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing, ) } }() } shun := make(chan string, 3) // sufficient for a cluster of 7 go member.Clean(shun, st, pr) go server.ListenAndServe(listener, canWrite, st, pr, secret) if secret == "" && webListener != nil { web.Store = st web.ClusterName = clusterName go web.Serve(webListener) } go func() { for p := range out { addr, err := net.ResolveUDPAddr(p.Addr) if err != nil { log.Println(err) continue } n, err := udpConn.WriteTo(p.Data, addr) if err != nil { log.Println(err) continue } if n != len(p.Data) { log.Println("packet len too long:", len(p.Data)) continue } } }() lv := liveness{ timeout: kickTimeout, ival: kickTimeout / 2, times: make(map[string]int64), self: self, shun: shun, } for { t := time.Nanoseconds() buf := make([]byte, maxUDPLen) n, addr, err := udpConn.ReadFrom(buf) if err == os.EINVAL { return } if err != nil { log.Println(err) continue } buf = buf[:n] // Update liveness time stamp for this addr lv.times[addr.String()] = t lv.check(t) in <- consensus.Packet{addr.String(), buf} } }