// main starts a broker instance.
// Configuration Options:
// - port number
// - host:port of register/leader
func main() {
log.SetVerbose(log.DEBUG)
defer func() {
if r := recover(); nil != r {
log.Error("%v", r)
}
}()
// parse command line args
var configFile = flag.String("conf", "conf.json", "configuration file")
flag.Parse()
log.Info("Initializing broker with options from %s.", *configFile)
// init configuration
config, err := config.Init(*configFile)
checkError(err)
log.Info("Options read were: %v", config)
port, err := strconv.Atoi(config.Get("port", PORT))
checkError(err)
log.SetPrefix(fmt.Sprintf("broker@%d: ", port))
broker, err = brokerimpl.New(config)
checkError(err)
listenHttp(port)
}
// register sends a follow request to the given leader.
func (b *Broker) register() error {
log.Info("In register()")
follow := &protocol.FollowRequest{b.tails(), protocol.HostPort(b.Origin())}
payload, err := b.leader.Send(follow, math.MaxInt32, b.Origin())
if nil != err {
return err
}
log.Info("Registered with leader.")
var ack protocol.FollowACK
if err := json.Unmarshal(payload, &ack); nil != err {
return err
}
for topic, checkpoint := range ack.Truncate {
if log, exists := b.logs[topic]; exists {
log.Close()
delete(b.logs, topic)
}
if err := truncateLog(b.config, topic, checkpoint); nil != err {
return err
}
}
// successful connection
go b.failSafeCatchUp()
log.Info("Catching up with leader.")
return nil
}
// CheckNewLeader allows re-sending of disconnect requests
// every interval until a leader is connected
func (r *Register) CheckNewLeader() {
select {
case <-r.singleton:
for r.leader == EMPTY {
r.LeaderDisconnect()
time.Sleep(LEADERWAIT * time.Millisecond)
log.Info("CheckNewLeader leader is ", r.leader)
}
r.singleton <- 1
default:
// return if an instance already running
}
log.Info("Returning from CheckNewLeader")
}
// Writes the given entry at the end of the broker log.
func (log *Log) WriteNext(entry *LogEntry) error {
// in case of error, revert
checkpoint, _ := log.Seek(0, os.SEEK_CUR)
bail := func() { log.Seek(checkpoint, os.SEEK_SET) }
if string(entry.RequestId) == string(log.lastWritten) {
return nil
}
if err := log.writeLength(entry); nil != err {
bail()
return err
}
if _, err := log.writeEntry(entry); nil != err {
bail()
return err
}
log.lastWritten = entry.RequestId
debug.Info("wrote request %v.", entry.RequestId)
return nil
}
// initLogs initializes the logs map.
func (b *Broker) initLogs() {
pattern := filepath.Join(b.config.LogDir(), "*"+EXT)
matches, err := filepath.Glob(pattern)
if nil != err {
log.Panic("Unable to read from log directory: %s", b.config.LogDir())
}
for _, name := range matches {
topic := filepath.Base(name)
topic = topic[0 : len(topic)-len(EXT)]
file, err := OpenLog(b.config, topic, -1)
if nil != err {
log.Error("Ignoring bad log file: %s", name)
continue
}
b.logs[topic] = file
log.Info("Found log file for %s.", topic)
}
}
// ackFollower sends an acknowledgement to the follower.
func (b *Broker) ackFollower(f *Follower) error {
b.lock.Lock()
defer b.lock.Unlock()
ack := new(protocol.Ack)
if b.role != LEADER || f.conn.RemoteAddr().String() == b.Origin() {
log.Warn("Denying follow requests from %s.", f.conn.RemoteAddr())
ack.Status = protocol.StatusFailure
} else {
inner := new(protocol.FollowACK)
inner.Truncate = make(Offsets)
for topic, checkpoint := range b.checkpoints {
log.Info("Checkpoint is %v", checkpoint)
if f.tails[topic] > checkpoint {
inner.Truncate[topic] = checkpoint
f.tails[topic] = checkpoint
}
}
ack.Status = protocol.StatusSuccess
ack.Payload, _ = json.Marshal(inner)
}
return websocket.JSON.Send(f.conn, ack)
}
// SyncFollower streams updates to a follower through the given connection.
// Once the follower has fully caught up, add it to the follower set.
func (b *Broker) SyncFollower(conn *websocket.Conn, tails Offsets, hostport protocol.HostPort) error {
follower := &Follower{
conn: conn,
tails: tails,
hostport: hostport,
quit: make(chan interface{}, 1),
}
if err := b.ackFollower(follower); nil != err {
return err
}
log.Debug("Begin synchronizing follower %v.", follower.hostport)
for !follower.caughtUp(b) {
if err := follower.catchUp(b); nil != err {
return err
}
}
log.Info("Follower %v has fully caught up.", follower.hostport)
<-follower.quit
return nil
}
// producer handles incoming produce requests. Producers may send multiple
// produce requests on the same persistent connection. The function exits when
// an `io.EOF` is received on the connection.
func producer(conn *websocket.Conn) {
defer conn.Close()
for {
var request protocol.ProduceRequest
err := websocket.JSON.Receive(conn, &request)
if err == io.EOF { // graceful shutdown
break
}
if nil != err {
log.Warn("Ignoring invalid message from %v.", conn.RemoteAddr())
continue
}
ack := new(protocol.Ack)
if err := broker.Publish(request.Topic, request.ID, &request.Message); nil != err {
log.Error(err.Error())
ack.Status = protocol.StatusFailure
} else {
ack.Status = protocol.StatusSuccess
}
// TODO: should redirect if this node is not the leader
websocket.JSON.Send(conn, &ack)
}
log.Info("Closed producer connection from %v.", conn.RemoteAddr())
}
func (r *Register) SetLeader(hostport string) {
r.lock.Lock()
defer r.lock.Unlock()
r.leader = hostport
r.seenBrokers[hostport] = true
log.Info("SetLeader setting leader to be %v", r.leader)
}
// listenHttp starts a http server at the given port and listens for incoming
// websocket messages.
func listenHttp(port int) {
http.Handle("/"+protocol.PUBLISH, websocket.Handler(producer))
http.Handle("/"+protocol.FOLLOW, websocket.Handler(follower))
http.Handle("/"+protocol.SUBSCRIBE, websocket.Handler(consumer))
http.Handle("/"+protocol.SWAP, websocket.Handler(register))
log.Info("HTTP server started on %d.", port)
http.ListenAndServe(":"+strconv.Itoa(port), nil)
}
// ChangeLeader closes the current leader connection and re-registers.
func (b *Broker) ChangeLeader() error {
b.leader.Reset(b.config.Register())
log.Info("Reset to be %v", b.config.Register())
b.lock.Lock()
defer b.lock.Unlock()
return b.register()
}
// readEntry reads the next n bytes and decodes them into a log entry.
func (log *Log) readEntry(n uint32) (*LogEntry, error) {
debug.Info("Making slice of size n=%v", n)
buf := make([]byte, n)
total := uint32(0)
for { // read exactly n bytes, or until error
read, err := log.Read(buf[total:])
if nil != err {
return nil, err
}
total += uint32(read)
if total >= n {
break
}
}
var entry LogEntry
return &entry, entry.decode(buf)
}
// notifyFollowers notifies the followers of a change in leader
func (r *Register) notifyFollower(follower string, is map[string]bool) {
conn, err := websocket.Dial("ws://"+follower+"/"+protocol.SWAP, "", "http://"+follower+"/")
log.Info("Notifying %v", follower)
// failed to contact the follower
if nil != err {
// remove from follower set
r.RemoveFollower(follower)
return
}
err = websocket.JSON.Send(conn, is)
// failed to send to the follower
if nil != err {
// remove from follower set
r.RemoveFollower(follower)
return
}
}
// removeFollower disconnects follower from followers set.
func (b *Broker) removeFollower(follower *Follower) {
_, exists := b.followers[follower]
if !exists {
return
}
delete(b.followers, follower)
// create struct to communicate with register
var removeFollow protocol.InsyncChange
removeFollow.Type = protocol.REMOVE
removeFollow.HostPort = follower.hostport
// add in-sync follower
// check if disconnect from register. if so, exit.
websocket.JSON.Send(b.regConn, removeFollow)
// checkError(err) // FIXME: exiting is not the correct thing to do
follower.quit <- nil
log.Info("Removed follower %v from follower set.", follower.hostport)
}
// replicate replicates the given log entry across all followers.
func (b *Broker) replicate(topic string, entry *LogEntry) error {
// send message to all followers
for follower, _ := range b.followers {
sync := &protocol.Sync{topic, entry.Message, entry.RequestId}
log.Info("Sending %v to %v", entry.Message.ID, follower.hostport)
if err := websocket.JSON.Send(follower.conn, sync); nil != err {
// lost
b.removeFollower(follower)
}
}
// wait for ACK
for follower, _ := range b.followers {
var ack protocol.SyncACK
if err := websocket.JSON.Receive(follower.conn, &ack); nil != err {
// lost
b.removeFollower(follower)
}
}
return nil
}