// StartMapper.Start is a worker function that launches a mapper worker.
// firehoseConfig specifies the credentials for connecting to the Tumblr Firehose.
// reducer is an array of circuit cross-runtime pointers, listing all available reducers.
func (StartMapper) Start(firehoseConfig *firehose.Request, reducer []circuit.X) {
circuit.Daemonize(func() {
f := firehose.Redial(firehoseConfig)
var n int64
// Repeat forever: Read an event from the Firehose and pass is on to an appropriate reducer
for {
event := f.Read()
p := &Post{}
if event.Post != nil {
p.ID = event.Post.ID
p.Name = event.Post.BlogName
}
if event.Like != nil {
p.ID = event.Like.RootPostID
}
p.Score = 1
// XXX panic-protect
reducer[int(p.ID%int64(len(reducer)))].Call("Add", p)
n++
if n%100 == 0 {
println("Consumed", n, "events from the firehose")
}
}
})
}
func (StartMapper) Start(firehoseConfig *firehose.Request, reducer []circuit.X) {
circuit.Daemonize(func() {
f := firehose.Redial(firehoseConfig)
for {
poll(reducer, f)
}
})
}
func StreamFirehose(freq *firehose.Request) <-chan *createRequest {
ch := make(chan *createRequest)
go func() {
conn := firehose.Redial(freq)
for {
q := filter(conn.Read())
if q == nil {
continue
}
println(fmt.Sprintf("CREATE blogID=%d postID=%d", q.TimelineID, q.PostID))
ch <- &createRequest{
Forwarded: false,
Post: q,
ReturnResponse: func(err error) {
if err != nil {
println("Firehose->XCreatePost error:", err.Error())
return
}
},
}
}
}()
return ch
}