// start is the seqn at which this member was defined.
// start+alpha is the first seqn this manager is expected to participate in.
func NewManager(self string, alpha int, st *store.Store, outs PutterTo) *Manager {
start := <-st.Seqns
m := &Manager{
st: st,
ops: st.Ops,
rg: NewRegistrar(st, start, alpha),
seqns: make(chan uint64),
fillUntil: make(chan uint64),
reqs: make(chan instReq),
logger: util.NewLogger("manager"),
Self: self,
alpha: alpha,
outs: outs,
}
go m.gen(start + uint64(alpha))
go m.fill(start + uint64(alpha))
go m.process()
// Wait until process is ready
// TODO: is there something we can do to avoid this?
m.getInstance(0)
return m
}
func Dial(addr string) (*Client, os.Error) {
c, err := net.Dial("tcp", "", addr)
if err != nil {
return nil, err
}
pr := proto.NewConn(c)
go pr.ReadResponses()
return &Client{pr: pr, lg: util.NewLogger(addr)}, nil
}
func Clean(st *store.Store) {
cl := &cleaner{
st: st,
table: make(map[string]uint64),
logger: util.NewLogger("clean"),
}
for ev := range st.Watch("/doozer/info/*/applied") {
cl.update(ev)
cl.check()
}
}
func (st *Store) process(ops <-chan Op, seqns chan<- uint64, watches chan<- int) {
logger := util.NewLogger("store")
defer st.closeWatches()
var head uint64
for {
ver, values := st.state.ver, st.state.root
// Take any incoming requests and queue them up.
select {
case a := <-ops:
if closed(ops) {
return
}
if a.Seqn > ver {
st.todo[a.Seqn] = a
}
case w := <-st.watchCh:
st.watches = append(st.watches, w)
case seqn := <-st.cleanCh:
for ; head <= seqn; head++ {
st.log[head] = Event{}, false
}
case seqns <- ver:
// nothing to do here
case watches <- len(st.watches):
// nothing to do here
case st.notices[0].ch <- st.notices[0].ev:
st.notices = st.notices[1:]
if len(st.notices) < 1 {
st.notices = make([]notice, 1)
}
}
// If we have any mutations that can be applied, do them.
for t, ok := st.todo[ver+1]; ok; t, ok = st.todo[ver+1] {
var ev Event
values, ev = values.apply(t.Seqn, t.Mut)
logger.Printf("apply %s %v %v %v %v %v", ev.Desc(), ev.Seqn, ev.Path, ev.Body, ev.Cas, ev.Err)
st.state = &state{ev.Seqn, values}
st.log[t.Seqn] = ev
st.notify(ev)
for ver < ev.Seqn {
ver++
st.todo[ver] = Op{}, false
}
}
}
}
func newService(id, name string, mon *monitor) *service {
sv := &service{
id: id,
name: name,
st: mon.st,
self: mon.self,
cl: mon.cl,
mon: mon,
logger: util.NewLogger(id),
}
sv.logger.Println("new")
return sv
}
func (c *conn) serve() {
logger := util.NewLogger("%v", c.c.RemoteAddr())
logger.Println("accepted connection")
for {
rid, verb, data, err := c.ReadRequest()
if err != nil {
if err == os.EOF {
logger.Println("connection closed by peer")
} else {
logger.Println(err)
}
return
}
rlogger := util.NewLogger("%v - req [%d]", c.c.RemoteAddr(), rid)
if o, ok := ops[verb]; ok {
rlogger.Printf("%s %v", verb, data)
err := proto.Fit(data, o.p)
if err != nil {
c.SendResponse(rid, proto.Last, err)
continue
}
if o.redirect && !c.cal {
c.redirect(rid)
continue
}
go c.handle(rid, o.f, indirect(o.p))
continue
}
rlogger.Printf("unknown command <%s>", verb)
c.SendResponse(rid, proto.Last, os.ErrorString(proto.InvalidCommand+" "+verb))
}
}
// This is a little subtle. We want to follow redirects while still pipelining
// requests, and we want to allow as many requests as possible to succeed
// without retrying unnecessarily.
//
// In particular, reads never need to redirect, and writes must always go to
// the leader. So we want that read requests never retry, and write requests
// retry if and only if necessary. Here's how it works:
//
// In the proto.Conn, when we get a redirect response, we raise a flag noting
// the new address. This flag only goes up, never down. This flag effectively
// means the connection is deprecated. Any pending requests can go ahead, but
// new requests should use the new address.
//
// In the Client, when we notice that a redirect has occurred (i.e. the flag is
// set), we establish a new connection to the new address. Calls in the future
// will use the new connection. But we also allow the old connection to
// continue functioning as it was. Any writes on the old connection will retry,
// and then they are guaranteed to pick up the new connection. Any reads on the
// old connection will just succeed directly.
func (cl *Client) proto() (*proto.Conn, os.Error) {
cl.lk.Lock()
defer cl.lk.Unlock()
if cl.pr.RedirectAddr != "" {
conn, err := net.Dial("tcp", "", cl.pr.RedirectAddr)
if err != nil {
return nil, err
}
cl.lg = util.NewLogger(cl.pr.RedirectAddr)
cl.pr = proto.NewConn(conn)
go cl.pr.ReadResponses()
}
return cl.pr, nil
}
func Clean(st *store.Store, p paxos.Proposer) {
logger := util.NewLogger("member")
for ev := range st.Watch("/session/*") {
if !ev.IsDel() {
continue
}
parts := strings.Split(ev.Path, "/", 3)
name := parts[2]
logger.Printf("lost session %s", name)
clearSlot(p, ev, name)
removeMember(p, ev, name)
removeInfo(p, ev, name)
}
}
func activate(st *store.Store, self string, c *client.Client, cal chan int) {
logger := util.NewLogger("activate")
ch := make(chan store.Event)
st.GetDirAndWatch("/doozer/slot", ch)
for ev := range ch {
// TODO ev.IsEmpty()
if ev.IsSet() && ev.Body == "" {
_, err := c.Set(ev.Path, self, ev.Cas)
if err != nil {
logger.Println(err)
continue
}
close(cal)
close(ch)
}
}
}
func newSocket(id, name string, mon *monitor) *socket {
sv := mon.increfService(name + ".service")
if sv == nil {
return nil
}
so := &socket{
id: id,
name: name,
st: mon.st,
self: mon.self,
cl: mon.cl,
sv: sv,
mon: mon,
logger: util.NewLogger(id),
}
so.logger.Println("new")
return so
}
func evServer(w http.ResponseWriter, r *http.Request) {
wevs := make(chan store.Event)
logger := util.NewLogger(w.RemoteAddr())
path := r.URL.Path[len(evPrefix):]
logger.Println("new", path)
evs := Store.Watch(path + "**")
// TODO convert store.Snapshot to json and use that
go func() {
walk(path, Store, wevs)
close(wevs)
}()
websocket.Handler(func(ws *websocket.Conn) {
send(ws, path, wevs, logger)
send(ws, path, evs, logger)
ws.Close()
}).ServeHTTP(w, r)
}
func Pulse(node string, seqns <-chan uint64, s Setter, sleep int64) {
logger := util.NewLogger("pulse")
var err os.Error
cas := store.Missing
for {
seqn := strconv.Uitoa64(<-seqns)
if closed(seqns) {
break
}
cas, err = s.Set("/doozer/info/"+node+"/applied", seqn, cas)
if err != nil {
logger.Println(err)
}
time.Sleep(sleep)
}
}
func Clean(st *store.Store, pp paxos.Proposer) {
logger := util.NewLogger("lock")
for ev := range st.Watch("/session/*") {
if !ev.IsDel() {
continue
}
parts := strings.Split(ev.Path, "/", 3)
name := parts[2]
logger.Printf("lost session %s", name)
ch, err := store.Walk(ev, "/lock/**")
if err != nil {
continue
}
for ev := range ch {
if ev.Body == name {
paxos.Del(pp, ev.Path, ev.Cas)
}
}
}
}
errPrefix = "ERR:"
)
var (
ErrClosed = os.NewError("response was closed")
)
// Response flags
const (
Closed = 1 << iota
Last
)
var crnl = []byte{'\r', '\n'}
var logger = util.NewLogger("proto")
type Line string
type Redirect string
// This is to satisfy os.Error.
func (r Redirect) String() string {
return string(r)
}
// This needs to be refactored. There is client stuff and server stuff mixed up
// in here. This type should contain only symmetric low-level connection stuff.
type Conn struct {
c io.ReadWriteCloser
r *bufio.Reader
func Monitor(self string, st *store.Store, cl SetDeler) os.Error {
mon := &monitor{
self: self,
st: st,
cl: cl,
clock: make(chan ticker),
units: make(map[string]unit),
refs: make(map[string]int),
exitCh: make(chan exit),
readyCh: make(chan ready),
logger: util.NewLogger("monitor"),
}
mon.logger.Println("reading units")
evs := make(chan store.Event)
st.GetDirAndWatch(ctlKey, evs)
go func(c <-chan store.Event) {
for e := range c {
evs <- e
}
close(evs)
}(st.Watch(lockKey + "/*"))
for {
select {
case t := <-mon.clock:
t.tick()
case ev := <-evs:
prefix, id := path.Split(ev.Path)
switch prefix {
case ctlDir:
if ev.IsDel() {
mon.logger.Println("\n\n\ndel", id)
mon.decrefUnit(id)
break
}
ut := mon.increfUnit(id)
if ut == nil {
break
}
switch ev.Body {
case "start":
ut.start()
case "stop":
ut.stop()
case "auto", "":
fallthrough
default:
// nothing
}
case lockDir:
ut := mon.units[id]
if ut == nil {
break
}
ut.dispatchLockEvent(ev)
}
case e := <-mon.exitCh:
e.e.exited(e.w)
case r := <-mon.readyCh:
r.r.ready(r.f)
}
}
panic("unreachable")
}
func Main(clusterName, attachAddr string, udpConn net.PacketConn, listener, webListener net.Listener) {
logger := util.NewLogger("main")
var err os.Error
listenAddr := listener.Addr().String()
outs := make(paxos.ChanPutCloserTo)
cal := make(chan int)
var cl *client.Client
self := util.RandId()
st := store.New()
if attachAddr == "" { // we are the only node in a new cluster
set(st, "/doozer/info/"+self+"/public-addr", listenAddr, store.Missing)
set(st, "/doozer/info/"+self+"/hostname", os.Getenv("HOSTNAME"), store.Missing)
set(st, "/doozer/members/"+self, listenAddr, store.Missing)
set(st, "/doozer/slot/"+"1", self, store.Missing)
set(st, "/doozer/leader", self, store.Missing)
set(st, "/ping", "pong", store.Missing)
close(cal)
cl, err = client.Dial(listenAddr)
if err != nil {
panic(err)
}
} else {
cl, err = client.Dial(attachAddr)
if err != nil {
panic(err)
}
path := "/doozer/info/" + self + "/public-addr"
_, err = cl.Set(path, listenAddr, store.Clobber)
if err != nil {
panic(err)
}
path = "/doozer/info/" + self + "/hostname"
_, err = cl.Set(path, os.Getenv("HOSTNAME"), store.Clobber)
if err != nil {
panic(err)
}
joinSeqn, snap, err := cl.Join(self, listenAddr)
if err != nil {
panic(err)
}
done := make(chan int)
st.Ops <- store.Op{1, snap}
go advanceUntil(cl, done)
go func() {
st.Sync(joinSeqn + alpha)
close(done)
activate(st, self, cl, cal)
}()
// TODO sink needs a way to pick up missing values if there are any
// gaps in its sequence
}
mg := paxos.NewManager(self, alpha, st, outs)
if attachAddr == "" {
// Skip ahead alpha steps so that the registrar can provide a
// meaningful cluster.
n := <-st.Seqns
for i := n + 1; i < n+alpha; i++ {
st.Ops <- store.Op{i, store.Nop}
}
}
go func() {
<-cal
go lock.Clean(st, mg)
go session.Clean(st, mg)
go member.Clean(st, mg)
go gc.Pulse(self, st.Seqns, cl, pulseInterval)
go gc.Clean(st)
}()
sv := &server.Server{udpConn, listenAddr, st, mg, self}
go func() {
cas := store.Missing
for _ = range time.Tick(checkinInterval) {
_, cas, err = cl.Checkin(self, cas)
if err != nil {
logger.Println(err)
}
}
}()
go func() {
err := sv.Serve(listener, cal)
if err != nil {
panic(err)
}
}()
if webListener != nil {
web.Store = st
web.ClusterName = clusterName
go web.Serve(webListener)
}
sv.ServeUdp(outs)
}
"doozer/paxos"
"doozer/util"
"math"
"net"
"os"
"time"
)
const (
interval = 1e8 // ns == 100ms
timeout = 1e10 // ns == 10s
max = 3000 // bytes. Definitely big enough for UDP over Ethernet.
)
var logger = util.NewLogger("net")
type Conn interface {
ReadFrom([]byte) (int, net.Addr, os.Error)
WriteTo([]byte, net.Addr) (int, os.Error)
LocalAddr() net.Addr
}
type check struct {
paxos.Packet
at, until int64
}
func (k check) Less(y interface{}) bool {
return k.at < y.(check).at
}
Conn net.PacketConn
Addr string
St *store.Store
Mg Manager
Self string
}
func (sv *Server) ServeUdp(outs chan paxos.Packet) {
r := dnet.Ackify(sv.Conn, outs)
for p := range r {
sv.Mg.PutFrom(p.Addr, p.Msg)
}
}
var clg = util.NewLogger("cal")
func (s *Server) Serve(l net.Listener, cal chan int) os.Error {
for {
rw, err := l.Accept()
if err != nil {
log.Printf("%#v", err)
if e, ok := err.(*net.OpError); ok && e.Error == os.EINVAL {
return nil
}
return err
}
c := &conn{proto.NewConn(rw), rw, s, closed(cal)}
go c.serve()
}
