// catchUp tries to bring _this_ broker up to date with its leader.
func (b *Broker) catchUp() error {
write := func(request *protocol.Sync) (int64, error) {
b.lock.Lock()
defer b.lock.Unlock()
file, err := b.getOrOpenLog(request.Topic)
if nil != err {
return 0, err
}
entry := &LogEntry{request.Message, request.RequestId}
if err = file.WriteNext(entry); nil != err {
return 0, err
}
stat, err := file.Stat()
log.Debug("File size on %s is %d.", b.Origin(), stat.Size())
if nil != err {
return 0, err
}
return stat.Size(), nil
}
for {
var request protocol.Sync
if err := b.leader.Receive(&request); nil != err {
log.Warn("Unable to receive from leader.")
return err
}
offset, err := write(&request)
if nil != err {
log.Warn("Unable to open log file for %s.", request.Topic)
continue
}
ack := &protocol.SyncACK{request.Topic, offset}
if err := b.leader.Acknowledge(ack); nil != err {
log.Warn("Unable to ack leader: %s", err.Error())
}
b.cond.Broadcast()
}
return nil
}
// ackFollower sends an acknowledgement to the follower.
func (b *Broker) ackFollower(f *Follower) error {
b.lock.Lock()
defer b.lock.Unlock()
ack := new(protocol.Ack)
if b.role != LEADER || f.conn.RemoteAddr().String() == b.Origin() {
log.Warn("Denying follow requests from %s.", f.conn.RemoteAddr())
ack.Status = protocol.StatusFailure
} else {
inner := new(protocol.FollowACK)
inner.Truncate = make(Offsets)
for topic, checkpoint := range b.checkpoints {
log.Info("Checkpoint is %v", checkpoint)
if f.tails[topic] > checkpoint {
inner.Truncate[topic] = checkpoint
f.tails[topic] = checkpoint
}
}
ack.Status = protocol.StatusSuccess
ack.Payload, _ = json.Marshal(inner)
}
return websocket.JSON.Send(f.conn, ack)
}
// producer handles incoming produce requests. Producers may send multiple
// produce requests on the same persistent connection. The function exits when
// an `io.EOF` is received on the connection.
func producer(conn *websocket.Conn) {
defer conn.Close()
for {
var request protocol.ProduceRequest
err := websocket.JSON.Receive(conn, &request)
if err == io.EOF { // graceful shutdown
break
}
if nil != err {
log.Warn("Ignoring invalid message from %v.", conn.RemoteAddr())
continue
}
ack := new(protocol.Ack)
if err := broker.Publish(request.Topic, request.ID, &request.Message); nil != err {
log.Error(err.Error())
ack.Status = protocol.StatusFailure
} else {
ack.Status = protocol.StatusSuccess
}
// TODO: should redirect if this node is not the leader
websocket.JSON.Send(conn, &ack)
}
log.Info("Closed producer connection from %v.", conn.RemoteAddr())
}
// LeaderDisconnect empties out the leader and notifies followers
// of a change in leader
func (r *Register) LeaderDisconnect() {
r.lock.Lock()
// create a copy to release lock earlier
tmpSet := make(map[string]bool)
for hp, _ := range r.insync {
tmpSet[hp] = true
}
r.lock.Unlock()
if len(tmpSet) == 0 {
r.lock.Lock()
log.Warn("Set of followers is 0! Notify all seen brokers!!")
for hp, _ := range r.seenBrokers {
tmpSet[hp] = true
}
r.lock.Unlock()
}
// notify all followers with the same set for consistency
// and only remove from original set if fail to contact
for hp, _ := range tmpSet {
go r.notifyFollower(hp, tmpSet)
}
}
func main() {
var broker = flag.String("register", "localhost:12345", "host and port number of broker")
var user = flag.String("user", "octopx", "username")
flag.Parse()
tp, err := twitproducer.NewTwitProducer(*user, "octopioctopus", *broker, nil)
if nil != err {
log.Warn("Did not receive a correct twitproducer")
}
err = tp.RelayMessages()
if nil != err {
log.Warn("%v", err)
}
}
// tails returns the sizes of all log files, organized by their topics.
func (b *Broker) tails() Offsets {
tails := make(Offsets, len(b.logs))
for topic, file := range b.logs {
stat, err := file.Stat()
if nil != err {
log.Warn("Unable to get stats from log file: %s.", file.Name())
continue
}
tails[topic] = stat.Size()
}
return tails
}
// catchUpLog synchronizes a single log with the follower. Returns an error if
// it votes to abort the synchronization.
func (f *Follower) catchUpLog(broker *Broker, topic string) error {
file, err := OpenLog(broker.config, topic, f.tails[topic])
if nil != err {
log.Warn("Could not open log file for topic: %s.", topic)
return err
}
defer file.Close()
var total uint32 = uint32(f.tails[topic])
log.Debug("Started with %d.", total)
for {
// read next entry
entry, err := file.ReadNext()
if nil != err {
return nil // EOF
}
// send to follower
sync := &protocol.Sync{topic, entry.Message, entry.RequestId}
if err = websocket.JSON.Send(f.conn, sync); nil != err {
return err
}
total += entry.length() + 4
log.Debug("Send %v.", entry.RequestId)
log.Debug("Sent %d to %s.", total, f.conn.RemoteAddr())
// wait for ack
var ack protocol.SyncACK
if err = websocket.JSON.Receive(f.conn, &ack); nil != err {
return err
}
f.tails[topic] = ack.Offset
log.Debug("%s is at %d.", f.conn.RemoteAddr(), ack.Offset)
}
return nil
}
// caughtUp checks if the follower has really caught up, and adds it to the
// broker's follower set.
func (f *Follower) caughtUp(broker *Broker) bool {
broker.lock.Lock()
defer broker.lock.Unlock()
//log.Info("Obtained lock for %v", f.hostport)
expected := broker.tails()
for topic, offset := range expected {
if offset != f.tails[topic] {
log.Debug("Not fully caught up yet for %s. %d -> %d", topic, f.tails[topic], offset)
return false
}
}
// check if follower already in set. if so, delete prev entry.
for follower, _ := range broker.followers {
if f.hostport == follower.hostport {
delete(broker.followers, follower)
}
}
// add to set of followers
broker.followers[f] = true
// create struct to communicate with register
var addFollow protocol.InsyncChange
addFollow.Type = protocol.ADD
addFollow.HostPort = f.hostport
// add in-sync follower
// check if disconnect from register. if so, exit.
if err := websocket.JSON.Send(broker.regConn, addFollow); nil != err {
log.Warn("Unable to update register: %s.", err.Error())
}
return true
}
// BecomeLeader returns only after successfully declaring leadership with the
// register. It locks down the broker, declares leadership with the register,
// and checkpoints the tails of all open logs.
func (b *Broker) BecomeLeader() error {
endpoint := "ws://" + b.config.Register() + "/" + protocol.LEADER
origin := b.Origin()
log.Debug("Resetting...")
b.leader.Reset(origin)
b.lock.Lock()
defer b.lock.Unlock()
for {
var err error
b.regConn, err = websocket.Dial(endpoint, "", origin)
if nil != err {
log.Warn("Error dialing %s: %s", endpoint, err.Error())
backoff()
continue
}
err = websocket.JSON.Send(b.regConn, origin)
if nil != err {
backoff()
continue
}
break
}
b.checkpoints = b.tails()
b.role = LEADER
return nil
}