func TestManagerEvent(t *testing.T) {
const alpha = 2
runs := make(map[int64]*run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0),
}
st.Ops <- store.Op{
Seqn: 2,
Mut: store.MustEncodeSet(cal+"/1", "a", 0),
}
ch, err := st.Wait(store.Any, 2)
if err != nil {
panic(err)
}
x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5")
pseqn := make(chan int64, 1)
m := &Manager{
Alpha: alpha,
Self: "a",
PSeqn: pseqn,
Ops: st.Ops,
Out: make(chan Packet),
run: runs,
}
m.event(<-ch)
exp := &run{
self: "a",
seqn: 2 + alpha,
cals: []string{"a"},
addr: []*net.UDPAddr{x},
ops: st.Ops,
out: m.Out,
bound: initialWaitBound,
}
exp.c = coordinator{
crnd: 1,
size: 1,
quor: exp.quorum(),
}
exp.l = learner{
round: 1,
size: 1,
quorum: int64(exp.quorum()),
votes: map[string]int64{},
voted: []bool{false},
}
assert.Equal(t, 1, len(runs))
assert.Equal(t, exp, runs[exp.seqn])
assert.Equal(t, exp.seqn, <-pseqn)
assert.Equal(t, exp.seqn+1, m.next)
}
func TestManagerFilterPropSeqn(t *testing.T) {
ps := make(chan int64, 100)
st := store.New()
defer close(st.Ops)
m := &Manager{
DefRev: 2,
Alpha: 1,
Self: "b",
PSeqn: ps,
Store: st,
}
go m.Run()
st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", "b", 0)}
st.Ops <- store.Op{3, store.Nop}
st.Ops <- store.Op{4, store.Nop}
assert.Equal(t, int64(3), <-ps)
assert.Equal(t, int64(5), <-ps)
st.Ops <- store.Op{5, store.Nop}
st.Ops <- store.Op{6, store.Nop}
assert.Equal(t, int64(7), <-ps)
}
func TestConsensusOne(t *testing.T) {
self := "test"
const alpha = 1
st := store.New()
st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/node/"+self+"/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", self, 0)}
<-st.Seqns
in := make(chan Packet)
out := make(chan Packet)
seqns := make(chan int64, alpha)
props := make(chan *Prop)
m := &Manager{
Self: self,
DefRev: 2,
Alpha: alpha,
In: in,
Out: out,
Ops: st.Ops,
PSeqn: seqns,
Props: props,
TFill: 10e9,
Store: st,
Ticker: time.Tick(10e6),
}
go m.Run()
go func() {
for o := range out {
in <- o
}
}()
n := <-seqns
w, err := st.Wait(store.Any, n)
if err != nil {
panic(err)
}
props <- &Prop{n, []byte("foo")}
e := <-w
exp := store.Event{
Seqn: 3,
Path: "/ctl/err",
Body: "bad mutation",
Rev: 3,
Mut: "foo",
Err: errors.New("bad mutation"),
}
e.Getter = nil
assert.Equal(t, exp, e)
}
func TestGetCalsPartial(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{Seqn: 1, Mut: store.MustEncodeSet(cal+"/1", "a", 0)}
st.Ops <- store.Op{Seqn: 2, Mut: store.MustEncodeSet(cal+"/2", "", 0)}
st.Ops <- store.Op{Seqn: 3, Mut: store.MustEncodeSet(cal+"/3", "", 0)}
<-st.Seqns
assert.Equal(t, []string{"a"}, getCals(st))
}
func TestManagerPumpDropsOldPackets(t *testing.T) {
st := store.New()
defer close(st.Ops)
x := &net.UDPAddr{net.IP{1, 2, 3, 4}, 5}
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
var m Manager
m.run = make(map[int64]*run)
m.event(<-mustWait(st, 2))
m.pump()
recvPacket(&m.packet, Packet{x, mustMarshal(&msg{Seqn: proto.Int64(1)})})
m.pump()
assert.Equal(t, 0, m.Stats.WaitPackets)
}
func TestGetAddrs(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/1/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet(node+"/2/addr", "2.3.4.5:6", 0)}
st.Ops <- store.Op{3, store.MustEncodeSet(node+"/3/addr", "3.4.5.6:7", 0)}
<-st.Seqns
x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5")
y, _ := net.ResolveUDPAddr("udp", "2.3.4.5:6")
z, _ := net.ResolveUDPAddr("udp", "3.4.5.6:7")
addrs := getAddrs(st, []string{"1", "2", "3"})
assert.Equal(t, []*net.UDPAddr{x, y, z}, addrs)
}
func TestRestore(t *testing.T) {
f, err := ioutil.TempFile("", "j")
if err != nil {
t.Fatal(err)
}
j := f.Name()
f.Close()
defer os.Remove(j)
journal, err := persistence.NewJournal(j)
if err != nil {
t.Fatal(err)
}
for k, v := range testData {
m := store.MustEncodeSet("/ken/"+strconv.Itoa(k), v, int64(k))
journal.WriteMutation(m)
}
journal.Close()
c := NewCluster(t, fmt.Sprintf("-j=%s", j), "-r")
defer c.Close()
for k, v := range testData {
v1, _, err := c.conn.Get("/ken/"+strconv.Itoa(k), nil)
if err != nil {
t.Fatal(err)
}
v2 := string(v1)
if v != v2 {
t.Fatalf("restored data is not what is expected: %s != %s", v, v2)
}
}
}
func TestMemberSimple(t *testing.T) {
st := store.New()
defer close(st.Ops)
fp := &test.FakeProposer{Store: st}
c := make(chan string)
go Clean(c, fp.Store, fp)
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/x", "a", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/y", "b", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/addr", "1.2.3.4", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/cal/0", "a", store.Missing)))
calCh, err := fp.Wait(store.MustCompileGlob("/ctl/cal/0"), 1+<-fp.Seqns)
if err != nil {
panic(err)
}
nodeCh, err := fp.Wait(store.MustCompileGlob("/ctl/node/a/?"), 1+<-fp.Seqns)
if err != nil {
panic(err)
}
// indicate that this peer is inactive
go func() { c <- "1.2.3.4" }()
ev := <-calCh
assert.T(t, ev.IsSet())
assert.Equal(t, "", ev.Body)
cs := []int{}
ev = <-nodeCh
assert.T(t, ev.IsDel())
cs = append(cs, int(ev.Path[len(ev.Path)-1]))
nodeCh, err = fp.Wait(store.MustCompileGlob("/ctl/node/a/?"), ev.Seqn+1)
if err != nil {
panic(err)
}
ev = <-nodeCh
assert.T(t, ev.IsDel())
cs = append(cs, int(ev.Path[len(ev.Path)-1]))
sort.Ints(cs)
assert.Equal(t, []int{'x', 'y'}, cs)
}
// encode takes the binary representation of an operation
// and returns the human readable form.
func encode(ev doozer.Event) (mut string) {
switch {
case ev.IsSet():
mut = st.MustEncodeSet(ev.Path, string(ev.Body), ev.Rev)
case ev.IsDel():
mut = st.MustEncodeDel(ev.Path, ev.Rev)
}
return
}
func (c cloner) VisitFile(path string, f *doozer.FileInfo) {
// store.Clobber is okay here because the event
// has already passed through another store
body, _, err := c.cl.Get(path, &c.storeRev)
if err != nil {
panic(err)
}
mut := store.MustEncodeSet(path, string(body), store.Clobber)
c.ch <- store.Op{f.Rev, mut}
}
func TestDelRun(t *testing.T) {
const alpha = 2
runs := make(map[int64]*run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "x", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet(cal+"/1", "a", 0)}
st.Ops <- store.Op{3, store.Nop}
st.Ops <- store.Op{4, store.Nop}
c2, err := st.Wait(store.Any, 2)
if err != nil {
panic(err)
}
c3, err := st.Wait(store.Any, 3)
if err != nil {
panic(err)
}
c4, err := st.Wait(store.Any, 4)
if err != nil {
panic(err)
}
pseqn := make(chan int64, 100)
m := &Manager{
Alpha: alpha,
Self: "a",
PSeqn: pseqn,
Ops: st.Ops,
Out: make(chan Packet),
run: runs,
}
m.event(<-c2)
assert.Equal(t, 1, len(m.run))
m.event(<-c3)
assert.Equal(t, 2, len(m.run))
m.event(<-c4)
assert.Equal(t, 2, len(m.run))
}
func TestManagerTickQueue(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
var m Manager
m.run = make(map[int64]*run)
m.Alpha = 1
m.Store = st
m.Out = make(chan Packet, 100)
m.event(<-mustWait(st, 2))
// get it to tick for seqn 3
recvPacket(&m.packet, Packet{Data: mustMarshal(&msg{Seqn: proto.Int64(3), Cmd: propose})})
m.pump()
assert.Equal(t, 1, m.tick.Len())
m.doTick(time.Now().UnixNano() + initialWaitBound*2)
assert.Equal(t, int64(1), m.Stats.TotalTicks)
}
func TestManagerPacketProcessing(t *testing.T) {
st := store.New()
defer close(st.Ops)
in := make(chan Packet)
out := make(chan Packet, 100)
var m Manager
m.run = make(map[int64]*run)
m.Alpha = 1
m.Store = st
m.In = in
m.Out = out
m.Ops = st.Ops
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
m.event(<-mustWait(st, 2))
recvPacket(&m.packet, Packet{
Data: mustMarshal(&msg{Seqn: proto.Int64(2), Cmd: learn, Value: []byte("foo")}),
Addr: &net.UDPAddr{net.IP{127, 0, 0, 1}, 9999},
})
m.pump()
assert.Equal(t, 0, m.packet.Len())
}
func follow(st *store.Store, cl *doozer.Conn, rev int64, stop chan bool) {
for {
ev, err := cl.Wait("/**", rev)
if err != nil {
panic(err)
}
// store.Clobber is okay here because the event
// has already passed through another store
mut := store.MustEncodeSet(ev.Path, string(ev.Body), store.Clobber)
st.Ops <- store.Op{ev.Rev, mut}
rev = ev.Rev + 1
select {
case <-stop:
return
default:
}
}
}
func Main(clusterName, self, buri, rwsk, rosk string, cl *doozer.Conn, udpConn *net.UDPConn, listener, webListener net.Listener, pulseInterval, fillDelay, kickTimeout int64, hi int64) {
listenAddr := listener.Addr().String()
canWrite := make(chan bool, 1)
in := make(chan consensus.Packet, 50)
out := make(chan consensus.Packet, 50)
st := store.New()
pr := &proposer{
seqns: make(chan int64, alpha),
props: make(chan *consensus.Prop),
st: st,
}
calSrv := func(start int64) {
go gc.Pulse(self, st.Seqns, pr, pulseInterval)
go gc.Clean(st, hi, time.Tick(1e9))
var m consensus.Manager
m.Self = self
m.DefRev = start
m.Alpha = alpha
m.In = in
m.Out = out
m.Ops = st.Ops
m.PSeqn = pr.seqns
m.Props = pr.props
m.TFill = fillDelay
m.Store = st
m.Ticker = time.Tick(10e6)
go m.Run()
}
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
if cl == nil { // we are the only node in a new cluster
set(st, "/ctl/name", clusterName, store.Missing)
set(st, "/ctl/node/"+self+"/addr", listenAddr, store.Missing)
set(st, "/ctl/node/"+self+"/hostname", hostname, store.Missing)
set(st, "/ctl/node/"+self+"/version", Version, store.Missing)
set(st, "/ctl/cal/0", self, store.Missing)
if buri == "" {
set(st, "/ctl/ns/"+clusterName+"/"+self, listenAddr, store.Missing)
}
calSrv(<-st.Seqns)
// Skip ahead alpha steps so that the registrar can provide a
// meaningful cluster.
for i := 0; i < alpha; i++ {
st.Ops <- store.Op{1 + <-st.Seqns, store.Nop}
}
canWrite <- true
go setReady(pr, self)
} else {
setC(cl, "/ctl/node/"+self+"/addr", listenAddr, store.Clobber)
setC(cl, "/ctl/node/"+self+"/hostname", hostname, store.Clobber)
setC(cl, "/ctl/node/"+self+"/version", Version, store.Clobber)
rev, err := cl.Rev()
if err != nil {
panic(err)
}
stop := make(chan bool, 1)
go follow(st, cl, rev+1, stop)
err = doozer.Walk(cl, rev, "/", func(path string, f *doozer.FileInfo, e error) (err error) {
if f.IsDir {
return
}
if e != nil {
return e
}
// store.Clobber is okay here because the event
// has already passed through another store
body, _, err := cl.Get(path, &rev)
if err != nil {
return
}
mut := store.MustEncodeSet(path, string(body), store.Clobber)
st.Ops <- store.Op{f.Rev, mut}
return
})
if err != nil {
panic(err)
}
st.Flush()
ch, err := st.Wait(store.Any, rev+1)
if err == nil {
<-ch
}
go func() {
n := activate(st, self, cl)
calSrv(n)
advanceUntil(cl, st.Seqns, n+alpha)
stop <- true
canWrite <- true
go setReady(pr, self)
if buri != "" {
b, err := doozer.DialUri(buri, "")
if err != nil {
panic(err)
}
setC(
b,
"/ctl/ns/"+clusterName+"/"+self,
listenAddr,
store.Missing,
)
}
}()
}
shun := make(chan string, 3) // sufficient for a cluster of 7
go member.Clean(shun, st, pr)
go server.ListenAndServe(listener, canWrite, st, pr, rwsk, rosk)
if rwsk == "" && rosk == "" && webListener != nil {
web.Store = st
web.ClusterName = clusterName
go web.Serve(webListener)
}
go func() {
for p := range out {
n, err := udpConn.WriteTo(p.Data, p.Addr)
if err != nil {
log.Println(err)
continue
}
if n != len(p.Data) {
log.Println("packet len too long:", len(p.Data))
continue
}
}
}()
selfAddr, ok := udpConn.LocalAddr().(*net.UDPAddr)
if !ok {
panic("no UDP addr")
}
lv := liveness{
timeout: kickTimeout,
ival: kickTimeout / 2,
self: selfAddr,
shun: shun,
}
for {
t := time.Now().UnixNano()
buf := make([]byte, maxUDPLen)
n, addr, err := udpConn.ReadFromUDP(buf)
if err == syscall.EINVAL {
return
}
if err != nil {
log.Println(err)
continue
}
buf = buf[:n]
lv.mark(addr, t)
lv.check(t)
in <- consensus.Packet{addr, buf}
}
}
func set(st *store.Store, path, body string, rev int64) {
mut := store.MustEncodeSet(path, body, rev)
st.Ops <- store.Op{1 + <-st.Seqns, mut}
}
func TestConsensusTwo(t *testing.T) {
a := "a"
b := "b"
x := &net.UDPAddr{net.IP{1, 2, 3, 4}, 5}
xs := "1.2.3.4:5"
y := &net.UDPAddr{net.IP{2, 3, 4, 5}, 6}
ys := "2.3.4.5:6"
const alpha = 1
st := store.New()
st.Ops <- store.Op{1, store.Nop}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/node/a/addr", xs, 0)}
st.Ops <- store.Op{3, store.MustEncodeSet("/ctl/cal/1", a, 0)}
st.Ops <- store.Op{4, store.MustEncodeSet("/ctl/node/b/addr", ys, 0)}
st.Ops <- store.Op{5, store.MustEncodeSet("/ctl/cal/2", b, 0)}
ain := make(chan Packet)
aout := make(chan Packet)
aseqns := make(chan int64, alpha)
aprops := make(chan *Prop)
am := &Manager{
Self: a,
DefRev: 5,
Alpha: alpha,
In: ain,
Out: aout,
Ops: st.Ops,
PSeqn: aseqns,
Props: aprops,
TFill: 10e9,
Store: st,
Ticker: time.Tick(10e6),
}
go am.Run()
bin := make(chan Packet)
bout := make(chan Packet)
bseqns := make(chan int64, alpha)
bprops := make(chan *Prop)
bm := &Manager{
Self: b,
DefRev: 5,
Alpha: alpha,
In: bin,
Out: bout,
Ops: st.Ops,
PSeqn: bseqns,
Props: bprops,
TFill: 10e9,
Store: st,
Ticker: time.Tick(10e6),
}
go bm.Run()
go func() {
for o := range aout {
o := o
if o.Addr.Port == x.Port && o.Addr.IP.Equal(x.IP) {
go func() { ain <- o }()
} else {
o.Addr = x
go func() { bin <- o }()
}
}
}()
go func() {
for o := range bout {
if o.Addr.Port == y.Port && o.Addr.IP.Equal(y.IP) {
go func() { bin <- o }()
} else {
o.Addr = y
go func() { ain <- o }()
}
}
}()
n := <-aseqns
assert.Equal(t, int64(6), n)
w, err := st.Wait(store.Any, n)
if err != nil {
panic(err)
}
aprops <- &Prop{n, []byte("foo")}
e := <-w
exp := store.Event{
Seqn: 6,
Path: "/ctl/err",
Body: "bad mutation",
Rev: 6,
Mut: "foo",
Err: errors.New("bad mutation"),
}
e.Getter = nil
assert.Equal(t, exp, e)
}