func TestDoozerSimple(t *testing.T) {
l := mustListen()
defer l.Close()
u := mustListenPacket(l.Addr().String())
defer u.Close()
go Main("a", "", u, l, nil)
cl, err := client.Dial(l.Addr().String())
assert.Equal(t, nil, err)
assert.Equal(t, nil, cl.Noop())
}
func TestGoroutines(t *testing.T) {
gs := runtime.Goroutines()
func() {
l := mustListen()
defer l.Close()
u := mustListenPacket(l.Addr().String())
defer u.Close()
go Main("a", "", u, l, nil)
cl, err := client.Dial(l.Addr().String())
assert.Equal(t, nil, err)
cl.Noop()
}()
assert.T(t, gs+leaked >= runtime.Goroutines(), gs+leaked)
}
func TestDoozerFiveNodeFailure(t *testing.T) {
d0 := mustRunDoozer("8040", "8080", "")
defer syscall.Kill(d0.Pid, 9)
time.Sleep(1e9)
d1 := mustRunDoozer("8041", "8081", "8040")
defer syscall.Kill(d1.Pid, 9)
d2 := mustRunDoozer("8042", "8082", "8040")
defer syscall.Kill(d2.Pid, 9)
d3 := mustRunDoozer("8043", "8083", "8040")
defer syscall.Kill(d3.Pid, 9)
d4 := mustRunDoozer("8044", "8084", "8040")
defer syscall.Kill(d4.Pid, 9)
cl, err := client.Dial("127.0.0.1:8040")
assert.Equal(t, nil, err)
ch, err := cl.Watch("/ctl/cal/*")
assert.Equal(t, nil, err)
cl.Set("/ctl/cal/2", "", "")
<-ch
<-ch
cl.Set("/ctl/cal/3", "", "")
<-ch
<-ch
// Give doozer time to get through initial Nops
time.Sleep(1e9 * 60)
// Kill an attached doozer
syscall.Kill(d1.Pid, 9)
// We should get something here
ev := <-ch
assert.NotEqual(t, nil, ev)
for i := 0; i < 1000; i++ {
cl.Noop()
}
}
func Main(clusterName, attachAddr string, udpConn net.PacketConn, listener, webListener net.Listener) {
logger := util.NewLogger("main")
var err os.Error
listenAddr := listener.Addr().String()
outs := make(paxos.ChanPutCloserTo)
cal := make(chan int)
var cl *client.Client
self := util.RandId()
st := store.New()
if attachAddr == "" { // we are the only node in a new cluster
set(st, "/doozer/info/"+self+"/public-addr", listenAddr, store.Missing)
set(st, "/doozer/info/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing)
set(st, "/doozer/members/"+self, listenAddr, store.Missing)
set(st, "/doozer/slot/"+"1", self, store.Missing)
set(st, "/doozer/leader", self, store.Missing)
set(st, "/ping", "pong", store.Missing)
close(cal)
cl, err = client.Dial(listenAddr)
if err != nil {
panic(err)
}
} else {
cl, err = client.Dial(attachAddr)
if err != nil {
panic(err)
}
path := "/doozer/info/" + self + "/public-addr"
_, err = cl.Set(path, listenAddr, store.Clobber)
if err != nil {
panic(err)
}
path = "/doozer/info/" + self + "/hostname"
_, err = cl.Set(path, os.Getenv("HOSTNAME"), store.Clobber)
if err != nil {
panic(err)
}
joinSeqn, snap, err := cl.Join(self, listenAddr)
if err != nil {
panic(err)
}
done := make(chan int)
st.Ops <- store.Op{1, snap}
go advanceUntil(cl, done)
go func() {
st.Sync(joinSeqn + alpha)
close(done)
activate(st, self, cl, cal)
}()
// TODO sink needs a way to pick up missing values if there are any
// gaps in its sequence
}
mg := paxos.NewManager(self, alpha, st, outs)
if attachAddr == "" {
// Skip ahead alpha steps so that the registrar can provide a
// meaningful cluster.
n := <-st.Seqns
for i := n + 1; i < n+alpha; i++ {
st.Ops <- store.Op{i, store.Nop}
}
}
go func() {
<-cal
go lock.Clean(st, mg)
go session.Clean(st, mg)
go member.Clean(st, mg)
go gc.Pulse(self, st.Seqns, cl, pulseInterval)
go gc.Clean(st)
}()
sv := &server.Server{udpConn, listenAddr, st, mg, self}
go func() {
cas := store.Missing
for _ = range time.Tick(checkinInterval) {
_, cas, err = cl.Checkin(self, cas)
if err != nil {
logger.Println(err)
}
}
}()
go func() {
err := sv.Serve(listener, cal)
if err != nil {
panic(err)
}
}()
if webListener != nil {
web.Store = st
web.ClusterName = clusterName
go web.Serve(webListener)
}
sv.ServeUdp(outs)
}