func TestSession(t *testing.T) {
st := store.New()
defer close(st.Ops)
fp := &test.FakeProposer{Store: st}
go Clean(st, fp)
ch := make(chan store.Event, 100)
go func(c <-chan store.Event) {
for e := range c {
ch <- e
}
close(ch)
}(st.Watch("/session/*"))
// check-in with less than a nanosecond to live
body := strconv.Itoa64(time.Nanoseconds() + 1)
fp.Propose(store.MustEncodeSet("/session/a", body, store.Clobber))
// Throw away the set
assert.T(t, (<-ch).IsSet())
ev := <-ch
assert.T(t, ev.IsDel())
assert.Equal(t, "/session/a", ev.Path)
}
func TestManagerEvent(t *testing.T) {
const alpha = 2
runs := make(map[int64]*run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "1.2.3.4:5", 0),
}
st.Ops <- store.Op{
Seqn: 2,
Mut: store.MustEncodeSet(cal+"/1", "a", 0),
}
ch, err := st.Wait(store.Any, 2)
if err != nil {
panic(err)
}
x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5")
pseqn := make(chan int64, 1)
m := &Manager{
Alpha: alpha,
Self: "a",
PSeqn: pseqn,
Ops: st.Ops,
Out: make(chan Packet),
run: runs,
}
m.event(<-ch)
exp := &run{
self: "a",
seqn: 2 + alpha,
cals: []string{"a"},
addr: []*net.UDPAddr{x},
ops: st.Ops,
out: m.Out,
bound: initialWaitBound,
}
exp.c = coordinator{
crnd: 1,
size: 1,
quor: exp.quorum(),
}
exp.l = learner{
round: 1,
size: 1,
quorum: int64(exp.quorum()),
votes: map[string]int64{},
voted: []bool{false},
}
assert.Equal(t, 1, len(runs))
assert.Equal(t, exp, runs[exp.seqn])
assert.Equal(t, exp.seqn, <-pseqn)
assert.Equal(t, exp.seqn+1, m.next)
}
func TestManagerProposeFill(t *testing.T) {
props := make(chan *Prop)
q := new(vector.Vector)
st := store.New()
out := make(chan Packet, 100)
cfg := &Config{
Self: "a",
Store: st,
Out: out,
DefRev: 2,
Alpha: 1,
Props: props,
}
m := NewManager(cfg)
m.run = map[int64]*run{
6: &run{seqn: 6, cals: []string{"a", "b", "c"}},
7: &run{seqn: 7, cals: []string{"a", "b", "c"}},
8: &run{seqn: 8, cals: []string{"a", "b", "c"}},
}
exp := vector.Vector{
trigger{123, 7},
trigger{123, 8},
}
m.propose(q, &Prop{Seqn: 9, Mut: []byte("foo")}, 123)
assert.Equal(t, exp, m.fill)
}
func TestMemberSimple(t *testing.T) {
st := store.New()
defer close(st.Ops)
fp := &test.FakeProposer{Store: st}
c := make(chan string)
go Clean(c, fp.Store, fp)
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/x", "a", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/y", "b", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/node/a/addr", "1.2.3.4", store.Missing)))
fp.Propose([]byte(store.MustEncodeSet("/ctl/cal/0", "a", store.Missing)))
calCh := fp.Watch(store.MustCompileGlob("/ctl/cal/0"))
nodeCh := fp.Watch(store.MustCompileGlob("/ctl/node/a/?"))
// indicate that this peer is inactive
go func() { c <- "1.2.3.4" }()
ev := <-calCh
assert.T(t, ev.IsSet())
assert.Equal(t, "", ev.Body)
cs := []int{}
ev = <-nodeCh
assert.T(t, ev.IsDel())
cs = append(cs, int(ev.Path[len(ev.Path)-1]))
ev = <-nodeCh
assert.T(t, ev.IsDel())
cs = append(cs, int(ev.Path[len(ev.Path)-1]))
sort.SortInts(cs)
assert.Equal(t, []int{'x', 'y'}, cs)
}
func TestManagerTickQueue(t *testing.T) {
ticker := make(chan int64)
st := store.New()
defer close(st.Ops)
in := make(chan Packet)
cfg := &Config{
Alpha: 1,
Store: st,
In: in,
Ticker: ticker,
Out: make(chan Packet, 100),
}
m := NewManager(cfg)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "x", 0),
}
for (<-m.Stats).Runs < 1 {
}
// get it to tick for seqn 2
in <- Packet{Data: mustMarshal(&msg{Seqn: proto.Int64(2), Cmd: propose})}
assert.Equal(t, 1, (<-m.Stats).WaitTicks)
ticker <- time.Nanoseconds() + initialWaitBound*2
assert.Equal(t, int64(1), (<-m.Stats).TotalTicks)
}
func TestManagerDeletesSuccessfulRun(t *testing.T) {
st := store.New()
defer close(st.Ops)
in := make(chan Packet)
out := make(chan Packet, 100)
cfg := &Config{
Alpha: 1,
Store: st,
In: in,
Out: out,
Ops: st.Ops,
}
m := NewManager(cfg)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "x", 0),
}
for (<-m.Stats).TotalRuns < 1 {
}
in <- Packet{
Data: mustMarshal(&msg{Seqn: proto.Int64(2), Cmd: learn, Value: []byte("foo")}),
Addr: "127.0.0.1:9999",
}
for (<-m.Stats).TotalRuns < 2 {
}
assert.Equal(t, 1, (<-m.Stats).Runs)
}
func TestManagerFilterPropSeqn(t *testing.T) {
ps := make(chan int64, 100)
st := store.New()
defer close(st.Ops)
m := &Manager{
DefRev: 2,
Alpha: 1,
Self: "b",
PSeqn: ps,
Store: st,
}
go m.Run()
st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", "b", 0)}
st.Ops <- store.Op{3, store.Nop}
st.Ops <- store.Op{4, store.Nop}
assert.Equal(t, int64(3), <-ps)
assert.Equal(t, int64(5), <-ps)
st.Ops <- store.Op{5, store.Nop}
st.Ops <- store.Op{6, store.Nop}
assert.Equal(t, int64(7), <-ps)
}
func TestRegistrarEmptySlot(t *testing.T) {
st := store.New()
rg := NewRegistrar(st, 0, 2)
go func() {
(st.Ops <- store.Op{3, mustEncodeSet(slotDir+"0", "c")})
(st.Ops <- store.Op{2, mustEncodeSet(slotDir+"1", "")})
(st.Ops <- store.Op{1, mustEncodeSet(slotDir+"2", "a")})
}()
func TestRegistrarMembers(t *testing.T) {
st := store.New()
rg := NewRegistrar(st, 0, 2)
go func() {
(st.Ops <- store.Op{3, mustEncodeSet(membersKey+"/c", "1")})
(st.Ops <- store.Op{2, mustEncodeSet(membersKey+"/b", "1")})
(st.Ops <- store.Op{1, mustEncodeSet(membersKey+"/a", "1")})
}()
func TestAddRun(t *testing.T) {
const alpha = 2
runs := make(map[int64]*run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "x", 0),
}
st.Ops <- store.Op{
Seqn: 2,
Mut: store.MustEncodeSet(cal+"/1", "a", 0),
}
ch, err := st.Wait(store.Any, 2)
if err != nil {
panic(err)
}
pseqn := make(chan int64, 1)
c := &Config{
Alpha: alpha,
Self: "a",
PSeqn: pseqn,
Ops: st.Ops,
Out: make(chan Packet),
}
m := &Manager{cfg: *c, run: runs}
got := m.addRun(<-ch)
exp := &run{
self: "a",
seqn: 2 + alpha,
cals: []string{"a"},
addr: []string{"x"},
ops: st.Ops,
out: c.Out,
bound: initialWaitBound,
}
exp.c = coordinator{
crnd: 1,
size: 1,
quor: exp.quorum(),
}
exp.l = learner{
round: 1,
quorum: int64(exp.quorum()),
votes: map[string]int64{},
voted: map[string]bool{},
}
assert.Equal(t, exp, got)
assert.Equal(t, exp, runs[got.seqn])
assert.Equal(t, 1, len(runs))
assert.Equal(t, exp.seqn, <-pseqn)
}
func selfRefNewManager(self string, alpha int) (*Manager, *store.Store) {
p := make(FakePutterFrom, 1)
st := store.New()
st.Ops <- store.Op{1, mustEncodeSet(membersDir+"a", "x")}
st.Ops <- store.Op{2, mustEncodeSet(slotDir+"0", "a")}
m := NewManager(self, alpha, st, putFromWrapperTo{p, "x"})
p[0] = m
return m, st
}
func TestManagerProposalQueue(t *testing.T) {
props := make(chan *Prop)
st := store.New()
out := make(chan Packet, 100)
m := newManager("", 0, nil, nil, nil, props, nil, 0, st, out)
props <- &Prop{Seqn: 1, Mut: []byte("foo")}
assert.Equal(t, 1, (<-m).WaitPackets)
}
func TestManagerFill(t *testing.T) {
st := store.New()
p := make(ChanPutCloserTo)
st.Ops <- store.Op{1, mustEncodeSet(membersDir+"a", "x")}
st.Ops <- store.Op{2, mustEncodeSet(slotDir+"0", "a")}
mg := NewManager("a", 1, st, p)
mg.fillUntil <- 4
assert.Equal(t, uint64(3), (<-p).Msg.Seqn())
}
func TestRegistrarSlotInitFirst(t *testing.T) {
st := store.New()
(st.Ops <- store.Op{1, mustEncodeSet(slotDir+"0", "a")})
st.Sync(1)
rg := NewRegistrar(st, 1, 0)
members, active := rg.setsForVersion(1)
assert.Equal(t, 0, len(members))
assert.Equal(t, []string{"a"}, active)
}
func TestRegistrarInitFirst(t *testing.T) {
st := store.New()
(st.Ops <- store.Op{1, mustEncodeSet(membersKey+"/a", "1")})
st.Sync(1)
rg := NewRegistrar(st, 1, 0)
members, active := rg.setsForVersion(1)
assert.Equal(t, 1, len(members))
assert.Equal(t, 0, len(active))
}
func TestRegistrarSlotInitTwo(t *testing.T) {
st := store.New()
(st.Ops <- store.Op{1, mustEncodeSet(slotDir+"0", "b")})
(st.Ops <- store.Op{2, mustEncodeSet(slotDir+"1", "a")})
rg := NewRegistrar(st, 2, 0)
members, active := rg.setsForVersion(2)
assert.Equal(t, 0, len(members))
assert.Equal(t, []string{"a", "b"}, active)
}
func TestConsensusOne(t *testing.T) {
self := "test"
const alpha = 1
st := store.New()
st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/node/"+self+"/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", self, 0)}
<-st.Seqns
in := make(chan Packet)
out := make(chan Packet)
seqns := make(chan int64, alpha)
props := make(chan *Prop)
m := &Manager{
Self: self,
DefRev: 2,
Alpha: alpha,
In: in,
Out: out,
Ops: st.Ops,
PSeqn: seqns,
Props: props,
TFill: 10e9,
Store: st,
Ticker: time.Tick(10e6),
}
go m.Run()
go func() {
for o := range out {
in <- o
}
}()
n := <-seqns
w, err := st.Wait(store.Any, n)
if err != nil {
panic(err)
}
props <- &Prop{n, []byte("foo")}
e := <-w
exp := store.Event{
Seqn: 3,
Path: "/ctl/err",
Body: "bad mutation",
Rev: 3,
Mut: "foo",
Err: errors.New("bad mutation"),
}
e.Getter = nil
assert.Equal(t, exp, e)
}
func TestConsensusOne(t *testing.T) {
self := "test"
const alpha = 1
st := store.New()
st.Ops <- store.Op{1, store.MustEncodeSet("/ctl/node/"+self+"/addr", "x", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/1", self, 0)}
<-st.Seqns
in := make(chan Packet)
out := make(chan Packet)
seqns := make(chan int64, alpha)
props := make(chan *Prop)
cfg := &Config{
self,
2,
alpha,
in,
out,
st.Ops,
seqns,
props,
10e9,
st,
time.Tick(10e6),
}
NewManager(cfg)
go func() {
for o := range out {
in <- o
}
}()
n := <-seqns
w, err := st.Wait(store.Any, n)
if err != nil {
panic(err)
}
props <- &Prop{n, []byte("foo")}
e := <-w
exp := store.Event{
Seqn: 3,
Path: "/ctl/err",
Body: "bad mutation",
Rev: 3,
Mut: "foo",
Err: os.NewError("bad mutation"),
}
e.Getter = nil
assert.Equal(t, exp, e)
}
func TestManagerAppliedNeverStarted(t *testing.T) {
st := store.New()
st.Ops <- store.Op{1, mustEncodeSet(membersDir+"a", "x")}
st.Ops <- store.Op{2, mustEncodeSet(slotDir+"0", "a")}
mg := NewManager("a", 1, st, nil)
st.Ops <- store.Op{3, store.Nop}
<-st.Seqns // give mg a chance to get the store.Event for seqn 3
assert.Equal(t, instance(nil), mg.getInstance(3))
}
func TestManagerPacketQueue(t *testing.T) {
in := make(chan Packet)
st := store.New()
out := make(chan Packet, 100)
m := newManager("", 0, nil, in, nil, nil, nil, 0, st, out)
in <- Packet{"x", mustMarshal(&msg{Seqn: proto.Int64(1)})}
assert.Equal(t, 1, (<-m).WaitPackets)
}
func TestGetCalsPartial(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{Seqn: 1, Mut: store.MustEncodeSet(cal+"/1", "a", 0)}
st.Ops <- store.Op{Seqn: 2, Mut: store.MustEncodeSet(cal+"/2", "", 0)}
st.Ops <- store.Op{Seqn: 3, Mut: store.MustEncodeSet(cal+"/3", "", 0)}
<-st.Seqns
assert.Equal(t, []string{"a"}, getCals(st))
}
func TestGetAddrs(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/1/addr", "x", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet(node+"/2/addr", "y", 0)}
st.Ops <- store.Op{3, store.MustEncodeSet(node+"/3/addr", "z", 0)}
<-st.Seqns
assert.Equal(t, map[string]bool{"x": true, "y": true, "z": true}, getAddrs(st))
}
func TestManagerClosesInstance(t *testing.T) {
st := store.New()
st.Ops <- store.Op{1, mustEncodeSet(membersDir+"a", "x")}
st.Ops <- store.Op{2, mustEncodeSet(slotDir+"0", "a")}
mg := NewManager("a", 1, st, nil)
it := mg.getInstance(3)
st.Ops <- store.Op{3, store.Nop}
<-it
assert.T(t, closed(it))
}
func TestRegistrarTooOld(t *testing.T) {
st := store.New()
(st.Ops <- store.Op{1, mustEncodeSet(membersKey+"/a", "1")})
(st.Ops <- store.Op{2, mustEncodeSet(membersKey+"/a", "1")})
st.Sync(2)
rg := NewRegistrar(st, 2, 0)
members, active := rg.setsForVersion(1)
assert.Equal(t, map[string]string{}, members, "members 1")
assert.Equal(t, []string{}, active, "active 1")
}
func alphaTest(t *testing.T, alpha int64) {
runs := make(chan *run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/a/addr", "x", 0),
}
st.Ops <- store.Op{
Seqn: 2,
Mut: store.MustEncodeSet(cal+"/1", "a", 0),
}
for 2 != <-st.Seqns {
}
tr := run{
self: "a",
ops: st.Ops,
out: make(chan Packet),
bound: initialWaitBound,
}
go generateRuns(alpha, st.Watch(store.Any), runs, tr)
// The only way to generate a run is on an event.
// Send a nop here to get things started.
st.Ops <- store.Op{3, store.Nop}
exp := &run{
self: "a",
seqn: 3 + alpha,
cals: []string{"a"},
addrs: map[string]bool{"x": true},
ops: st.Ops,
out: tr.out,
bound: initialWaitBound,
}
exp.c = coordinator{
crnd: 1,
size: 1,
quor: exp.quorum(),
}
exp.l = learner{
round: 1,
quorum: int64(exp.quorum()),
votes: map[string]int64{},
voted: map[string]bool{},
}
assert.Equal(t, exp, <-runs)
}
func TestGetAddrs(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/1/addr", "x", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet(node+"/2/addr", "y", 0)}
st.Ops <- store.Op{3, store.MustEncodeSet(node+"/3/addr", "z", 0)}
<-st.Seqns
addrs := getAddrs(st, []string{"1", "2", "3"})
assert.Equal(t, []string{"x", "y", "z"}, addrs)
}
func TestRunAfterWatch(t *testing.T) {
alpha := int64(3)
runs := make(chan *run)
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{
Seqn: 1,
Mut: store.MustEncodeSet(node+"/b/addr", "y", 0),
}
for 1 != <-st.Seqns {
}
tr := run{
self: "b",
ops: st.Ops,
out: make(chan Packet),
bound: initialWaitBound,
}
go generateRuns(alpha, st.Watch(store.Any), runs, tr)
st.Ops <- store.Op{
Seqn: 2,
Mut: store.MustEncodeSet(cal+"/1", "b", 0),
}
exp := &run{
self: "b",
seqn: 2 + alpha,
cals: []string{"b"},
addrs: map[string]bool{"y": true},
ops: st.Ops,
out: tr.out,
bound: initialWaitBound,
}
exp.c = coordinator{
crnd: 1,
size: 1,
quor: exp.quorum(),
}
exp.l = learner{
round: 1,
quorum: int64(exp.quorum()),
votes: map[string]int64{},
voted: map[string]bool{},
}
assert.Equal(t, exp, <-runs)
}
func TestManagerPumpDropsOldPackets(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/a/addr", "x", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet("/ctl/cal/0", "a", 0)}
var m Manager
m.run = make(map[int64]*run)
m.event(<-mustWait(st, 2))
m.pump()
recvPacket(&m.packet, Packet{"x", mustMarshal(&msg{Seqn: proto.Int64(1)})})
m.pump()
assert.Equal(t, 0, m.Stats.WaitPackets)
}
func TestReadFromStore(t *testing.T) {
// The cluster initially has 1 node (quorum of 1).
p := make(ChanPutCloserTo)
self := "a"
addr := "x"
st := store.New()
st.Ops <- store.Op{1, mustEncodeSet(membersDir+self, addr)}
st.Ops <- store.Op{2, mustEncodeSet(slotDir+"0", self)}
ch := make(chan store.Event, 100)
go func(c <-chan store.Event) {
for e := range c {
ch <- e
}
close(ch)
}(st.Watch("**"))
m := NewManager(self, 1, st, p)
// Fire up a new instance with a vote message. This instance should block
// trying to read the list of members. If it doesn't wait, it'll
// immediately learn the value `x`.
in := newVote(1, "x")
in.SetSeqn(5)
go m.PutFrom(addr, in)
// Satisfy the sync read of data members above. After this, there will be
// 2 nodes in the cluster, making the quorum 2.
bAddr := "y"
st.Ops <- store.Op{3, mustEncodeSet(membersDir+"b", bAddr)}
st.Ops <- store.Op{4, mustEncodeSet(slotDir+"1", "b")}
// Now try to make it learn a new value with 2 votes to meet the new
// quorum.
exp := "y"
in = newVote(2, exp)
in.SetSeqn(5)
m.PutFrom(addr, in)
in = newVote(2, exp)
in.SetSeqn(5)
m.PutFrom(bAddr, in)
<-ch
<-ch
got := <-ch
assert.Equal(t, uint64(5), got.Seqn)
assert.Equal(t, exp, got.Mut)
}
func TestGetAddrs(t *testing.T) {
st := store.New()
defer close(st.Ops)
st.Ops <- store.Op{1, store.MustEncodeSet(node+"/1/addr", "1.2.3.4:5", 0)}
st.Ops <- store.Op{2, store.MustEncodeSet(node+"/2/addr", "2.3.4.5:6", 0)}
st.Ops <- store.Op{3, store.MustEncodeSet(node+"/3/addr", "3.4.5.6:7", 0)}
<-st.Seqns
x, _ := net.ResolveUDPAddr("udp", "1.2.3.4:5")
y, _ := net.ResolveUDPAddr("udp", "2.3.4.5:6")
z, _ := net.ResolveUDPAddr("udp", "3.4.5.6:7")
addrs := getAddrs(st, []string{"1", "2", "3"})
assert.Equal(t, []*net.UDPAddr{x, y, z}, addrs)
}