func TestManagerEvent(t *testing.T) { const alpha = 2 runs := make(map[int64]*run) st := store.New() defer close(st.Ops) st.Ops <- store.Op{ Seqn: 1, Mut: store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0), } st.Ops <- store.Op{ Seqn: 2, Mut: store.MustEncodeSet(cal+"/1", "a", 0), } ch, err := st.Wait(store.Any, 2) if err != nil { panic(err) } x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5") pseqn := make(chan int64, 1) m := &Manager{ Alpha: alpha, Self: "a", PSeqn: pseqn, Ops: st.Ops, Out: make(chan Packet), run: runs, } m.event(<-ch) exp := &run{ self: "a", seqn: 2 + alpha, cals: []string{"a"}, addr: []*net.UDPAddr{x}, ops: st.Ops, out: m.Out, bound: initialWaitBound, } exp.c = coordinator{ crnd: 1, size: 1, quor: exp.quorum(), } exp.l = learner{ round: 1, size: 1, quorum: int64(exp.quorum()), votes: map[string]int64{}, voted: []bool{false}, } assert.Equal(t, 1, len(runs)) assert.Equal(t, exp, runs[exp.seqn]) assert.Equal(t, exp.seqn, <-pseqn) assert.Equal(t, exp.seqn+1, m.next) }
func TestManagerFilterPropSeqn(t *testing.T) { ps := make(chan int64, 100) st := store.New() defer close(st.Ops) m := &Manager{ DefRev: 2, Alpha: 1, Self: "b", PSeqn: ps, Store: st, } go m.Run() st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/cal/0", "a", 0)} st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", "b", 0)} st.Ops <- store.Op{3, store.Nop} st.Ops <- store.Op{4, store.Nop} assert.Equal(t, int64(3), <-ps) assert.Equal(t, int64(5), <-ps) st.Ops <- store.Op{5, store.Nop} st.Ops <- store.Op{6, store.Nop} assert.Equal(t, int64(7), <-ps) }
func TestGetCalsPartial(t *testing.T) { st := store.New() defer close(st.Ops) st.Ops <- store.Op{Seqn: 1, Mut: store.MustEncodeSet(cal+"/1", "a", 0)} st.Ops <- store.Op{Seqn: 2, Mut: store.MustEncodeSet(cal+"/2", "", 0)} st.Ops <- store.Op{Seqn: 3, Mut: store.MustEncodeSet(cal+"/3", "", 0)} <-st.Seqns assert.Equal(t, []string{"a"}, getCals(st)) }
func TestGetAddrs(t *testing.T) { st := store.New() defer close(st.Ops) st.Ops <- store.Op{1, store.MustEncodeSet(node+"/1/addr", "1.2.3.4:5", 0)} st.Ops <- store.Op{2, store.MustEncodeSet(node+"/2/addr", "2.3.4.5:6", 0)} st.Ops <- store.Op{3, store.MustEncodeSet(node+"/3/addr", "3.4.5.6:7", 0)} <-st.Seqns x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5") y, _ := net.ResolveUDPAddr("udp", "2.3.4.5:6") z, _ := net.ResolveUDPAddr("udp", "3.4.5.6:7") addrs := getAddrs(st, []string{"1", "2", "3"}) assert.Equal(t, []*net.UDPAddr{x, y, z}, addrs) }
func TestManagerPumpDropsOldPackets(t *testing.T) { st := store.New() defer close(st.Ops) x := &net.UDPAddr{net.IP{1, 2, 3, 4}, 5} st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)} st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)} var m Manager m.run = make(map[int64]*run) m.event(<-mustWait(st, 2)) m.pump() recvPacket(&m.packet, Packet{x, mustMarshal(&msg{Seqn: proto.Int64(1)})}) m.pump() assert.Equal(t, 0, m.Stats.WaitPackets) }
func TestMemberSimple(t *testing.T) { st := store.New() defer close(st.Ops) fp := &test.FakeProposer{Store: st} c := make(chan string) go Clean(c, fp.Store, fp) fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/x", "a", store.Missing))) fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/y", "b", store.Missing))) fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/addr", "1.2.3.4", store.Missing))) fp.Propose([]byte(store.MustEncodeSet("/ctl/cal/0", "a", store.Missing))) calCh, err := fp.Wait(store.MustCompileGlob("/ctl/cal/0"), 1+<-fp.Seqns) if err != nil { panic(err) } nodeCh, err := fp.Wait(store.MustCompileGlob("/ctl/node/a/?"), 1+<-fp.Seqns) if err != nil { panic(err) } // indicate that this peer is inactive go func() { c <- "1.2.3.4" }() ev := <-calCh assert.T(t, ev.IsSet()) assert.Equal(t, "", ev.Body) cs := []int{} ev = <-nodeCh assert.T(t, ev.IsDel()) cs = append(cs, int(ev.Path[len(ev.Path)-1])) nodeCh, err = fp.Wait(store.MustCompileGlob("/ctl/node/a/?"), ev.Seqn+1) if err != nil { panic(err) } ev = <-nodeCh assert.T(t, ev.IsDel()) cs = append(cs, int(ev.Path[len(ev.Path)-1])) sort.Ints(cs) assert.Equal(t, []int{'x', 'y'}, cs) }
func TestDelRun(t *testing.T) { const alpha = 2 runs := make(map[int64]*run) st := store.New() defer close(st.Ops) st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "x", 0)} st.Ops <- store.Op{2, store.MustEncodeSet(cal+"/1", "a", 0)} st.Ops <- store.Op{3, store.Nop} st.Ops <- store.Op{4, store.Nop} c2, err := st.Wait(store.Any, 2) if err != nil { panic(err) } c3, err := st.Wait(store.Any, 3) if err != nil { panic(err) } c4, err := st.Wait(store.Any, 4) if err != nil { panic(err) } pseqn := make(chan int64, 100) m := &Manager{ Alpha: alpha, Self: "a", PSeqn: pseqn, Ops: st.Ops, Out: make(chan Packet), run: runs, } m.event(<-c2) assert.Equal(t, 1, len(m.run)) m.event(<-c3) assert.Equal(t, 2, len(m.run)) m.event(<-c4) assert.Equal(t, 2, len(m.run)) }
func TestManagerTickQueue(t *testing.T) { st := store.New() defer close(st.Ops) st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)} st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)} var m Manager m.run = make(map[int64]*run) m.Alpha = 1 m.Store = st m.Out = make(chan Packet, 100) m.event(<-mustWait(st, 2)) // get it to tick for seqn 3 recvPacket(&m.packet, Packet{Data: mustMarshal(&msg{Seqn: proto.Int64(3), Cmd: propose})}) m.pump() assert.Equal(t, 1, m.tick.Len()) m.doTick(time.Now().UnixNano() + initialWaitBound*2) assert.Equal(t, int64(1), m.Stats.TotalTicks) }
func TestManagerPacketProcessing(t *testing.T) { st := store.New() defer close(st.Ops) in := make(chan Packet) out := make(chan Packet, 100) var m Manager m.run = make(map[int64]*run) m.Alpha = 1 m.Store = st m.In = in m.Out = out m.Ops = st.Ops st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)} st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)} m.event(<-mustWait(st, 2)) recvPacket(&m.packet, Packet{ Data: mustMarshal(&msg{Seqn: proto.Int64(2), Cmd: learn, Value: []byte("foo")}), Addr: &net.UDPAddr{net.IP{127, 0, 0, 1}, 9999}, }) m.pump() assert.Equal(t, 0, m.packet.Len()) }
func Main(clusterName, self, buri, rwsk, rosk string, cl *doozer.Conn, udpConn *net.UDPConn, listener, webListener net.Listener, pulseInterval, fillDelay, kickTimeout int64, hi int64) { listenAddr := listener.Addr().String() canWrite := make(chan bool, 1) in := make(chan consensus.Packet, 50) out := make(chan consensus.Packet, 50) st := store.New() pr := &proposer{ seqns: make(chan int64, alpha), props: make(chan *consensus.Prop), st: st, } calSrv := func(start int64) { go gc.Pulse(self, st.Seqns, pr, pulseInterval) go gc.Clean(st, hi, time.Tick(1e9)) var m consensus.Manager m.Self = self m.DefRev = start m.Alpha = alpha m.In = in m.Out = out m.Ops = st.Ops m.PSeqn = pr.seqns m.Props = pr.props m.TFill = fillDelay m.Store = st m.Ticker = time.Tick(10e6) go m.Run() } hostname, err := os.Hostname() if err != nil { hostname = "unknown" } if cl == nil { // we are the only node in a new cluster set(st, "/ctl/name", clusterName, store.Missing) set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing) set(st, "/ctl/node/"+self+"/hostname", hostname, store.Missing) set(st, "/ctl/node/"+self+"/version", Version, store.Missing) set(st, "/ctl/cal/0", self, store.Missing) if buri == "" { set(st, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing) } calSrv(<-st.Seqns) // Skip ahead alpha steps so that the registrar can provide a // meaningful cluster. for i := 0; i < alpha; i++ { st.Ops <- store.Op{1 + <-st.Seqns, store.Nop} } canWrite <- true go setReady(pr, self) } else { setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber) setC(cl, "/ctl/node/"+self+"/hostname", hostname, store.Clobber) setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber) rev, err := cl.Rev() if err != nil { panic(err) } stop := make(chan bool, 1) go follow(st, cl, rev+1, stop) errs := make(chan error) go func() { e, ok := <-errs if ok { panic(e) } }() doozer.Walk(cl, rev, "/", cloner{st.Ops, cl, rev}, errs) close(errs) st.Flush() ch, err := st.Wait(store.Any, rev+1) if err == nil { <-ch } go func() { n := activate(st, self, cl) calSrv(n) advanceUntil(cl, st.Seqns, n+alpha) stop <- true canWrite <- true go setReady(pr, self) if buri != "" { b, err := doozer.DialUri(buri, "") if err != nil { panic(err) } setC( b, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing, ) } }() } shun := make(chan string, 3) // sufficient for a cluster of 7 go member.Clean(shun, st, pr) go server.ListenAndServe(listener, canWrite, st, pr, rwsk, rosk) if rwsk == "" && rosk == "" && webListener != nil { web.Store = st web.ClusterName = clusterName go web.Serve(webListener) } go func() { for p := range out { n, err := udpConn.WriteTo(p.Data, p.Addr) if err != nil { log.Println(err) continue } if n != len(p.Data) { log.Println("packet len too long:", len(p.Data)) continue } } }() selfAddr, ok := udpConn.LocalAddr().(*net.UDPAddr) if !ok { panic("no UDP addr") } lv := liveness{ timeout: kickTimeout, ival: kickTimeout / 2, self: selfAddr, shun: shun, } for { t := time.Now().UnixNano() buf := make([]byte, maxUDPLen) n, addr, err := udpConn.ReadFromUDP(buf) if err == syscall.EINVAL { return } if err != nil { log.Println(err) continue } buf = buf[:n] lv.mark(addr, t) lv.check(t) in <- consensus.Packet{addr, buf} } }