// Send sends the message to the broker, and blocks until an acknowledgement is
// received. If the max number of retries is exceeded, returns the last error.
func (p *Producer) Send(topic string, payload []byte) error {
seqnum := atomic.AddInt64(&p.seqnum, 1)
message := protocol.Message{seqnum, payload, crc32.ChecksumIEEE(payload)}
request := &protocol.ProduceRequest{p.id, topic, message}
log.Debug("Sending %v", request)
p.lock.Lock()
defer p.lock.Unlock()
for {
if _, err := p.socket.Send(request, MAX_RETRIES, origin()); nil != err {
p.socket.Reset(p.register)
continue
}
log.Debug("Acknowledgement received.")
return nil
}
return nil
}
// SyncFollower streams updates to a follower through the given connection.
// Once the follower has fully caught up, add it to the follower set.
func (b *Broker) SyncFollower(conn *websocket.Conn, tails Offsets, hostport protocol.HostPort) error {
follower := &Follower{
conn: conn,
tails: tails,
hostport: hostport,
quit: make(chan interface{}, 1),
}
if err := b.ackFollower(follower); nil != err {
return err
}
log.Debug("Begin synchronizing follower %v.", follower.hostport)
for !follower.caughtUp(b) {
if err := follower.catchUp(b); nil != err {
return err
}
}
log.Info("Follower %v has fully caught up.", follower.hostport)
<-follower.quit
return nil
}
// catchUpLog synchronizes a single log with the follower. Returns an error if
// it votes to abort the synchronization.
func (f *Follower) catchUpLog(broker *Broker, topic string) error {
file, err := OpenLog(broker.config, topic, f.tails[topic])
if nil != err {
log.Warn("Could not open log file for topic: %s.", topic)
return err
}
defer file.Close()
var total uint32 = uint32(f.tails[topic])
log.Debug("Started with %d.", total)
for {
// read next entry
entry, err := file.ReadNext()
if nil != err {
return nil // EOF
}
// send to follower
sync := &protocol.Sync{topic, entry.Message, entry.RequestId}
if err = websocket.JSON.Send(f.conn, sync); nil != err {
return err
}
total += entry.length() + 4
log.Debug("Send %v.", entry.RequestId)
log.Debug("Sent %d to %s.", total, f.conn.RemoteAddr())
// wait for ack
var ack protocol.SyncACK
if err = websocket.JSON.Receive(f.conn, &ack); nil != err {
return err
}
f.tails[topic] = ack.Offset
log.Debug("%s is at %d.", f.conn.RemoteAddr(), ack.Offset)
}
return nil
}
// catchUp tries to bring _this_ broker up to date with its leader.
func (b *Broker) catchUp() error {
write := func(request *protocol.Sync) (int64, error) {
b.lock.Lock()
defer b.lock.Unlock()
file, err := b.getOrOpenLog(request.Topic)
if nil != err {
return 0, err
}
entry := &LogEntry{request.Message, request.RequestId}
if err = file.WriteNext(entry); nil != err {
return 0, err
}
stat, err := file.Stat()
log.Debug("File size on %s is %d.", b.Origin(), stat.Size())
if nil != err {
return 0, err
}
return stat.Size(), nil
}
for {
var request protocol.Sync
if err := b.leader.Receive(&request); nil != err {
log.Warn("Unable to receive from leader.")
return err
}
offset, err := write(&request)
if nil != err {
log.Warn("Unable to open log file for %s.", request.Topic)
continue
}
ack := &protocol.SyncACK{request.Topic, offset}
if err := b.leader.Acknowledge(ack); nil != err {
log.Warn("Unable to ack leader: %s", err.Error())
}
b.cond.Broadcast()
}
return nil
}
// next reads the next message from the associated log file. When it reaches
// the end of the file, it waits (using a conditional variable) for more
// messages from the broker. As a consequence of this design, a waiting
// subscription cannot be closed until a new message is published, waking it
// up. Returns nil or associated error.
func (s *Subscription) next() error {
entry, err := s.log.ReadNext()
switch err {
case nil:
case io.EOF: // wait for more
log.Debug("Reached end of log.")
s.broker.wait(s)
return nil
default: // abort
return err
}
return websocket.JSON.Send(s.conn, &entry.Message)
}
// Serve blocks until either the websocket connection is closed, or until a
// message is received on the `quit` channel. This method may be invoked at
// most once; after it returns, the subscription is closed.
func (s *Subscription) Serve() error {
defer s.log.Close()
defer log.Debug("Stopped serving subscription %p.", s)
for {
select {
case <-s.quit:
return nil
default:
if err := s.next(); nil != err {
log.Error("Unable to serve subscription: %s", err.Error())
return err
}
}
}
return nil
}
// caughtUp checks if the follower has really caught up, and adds it to the
// broker's follower set.
func (f *Follower) caughtUp(broker *Broker) bool {
broker.lock.Lock()
defer broker.lock.Unlock()
//log.Info("Obtained lock for %v", f.hostport)
expected := broker.tails()
for topic, offset := range expected {
if offset != f.tails[topic] {
log.Debug("Not fully caught up yet for %s. %d -> %d", topic, f.tails[topic], offset)
return false
}
}
// check if follower already in set. if so, delete prev entry.
for follower, _ := range broker.followers {
if f.hostport == follower.hostport {
delete(broker.followers, follower)
}
}
// add to set of followers
broker.followers[f] = true
// create struct to communicate with register
var addFollow protocol.InsyncChange
addFollow.Type = protocol.ADD
addFollow.HostPort = f.hostport
// add in-sync follower
// check if disconnect from register. if so, exit.
if err := websocket.JSON.Send(broker.regConn, addFollow); nil != err {
log.Warn("Unable to update register: %s.", err.Error())
}
return true
}
// BecomeLeader returns only after successfully declaring leadership with the
// register. It locks down the broker, declares leadership with the register,
// and checkpoints the tails of all open logs.
func (b *Broker) BecomeLeader() error {
endpoint := "ws://" + b.config.Register() + "/" + protocol.LEADER
origin := b.Origin()
log.Debug("Resetting...")
b.leader.Reset(origin)
b.lock.Lock()
defer b.lock.Unlock()
for {
var err error
b.regConn, err = websocket.Dial(endpoint, "", origin)
if nil != err {
log.Warn("Error dialing %s: %s", endpoint, err.Error())
backoff()
continue
}
err = websocket.JSON.Send(b.regConn, origin)
if nil != err {
backoff()
continue
}
break
}
b.checkpoints = b.tails()
b.role = LEADER
return nil
}
func backoff() {
duration := time.Duration(rand.Intn(protocol.MAX_RETRY_INTERVAL))
log.Debug("Backing off %d milliseconds.", duration)
time.Sleep(duration * time.Millisecond)
}