示例#1
0
// StartMapper.Start is a worker function that launches a mapper worker.
// firehoseConfig specifies the credentials for connecting to the Tumblr Firehose.
// reducer is an array of circuit cross-runtime pointers, listing all available reducers.
func (StartMapper) Start(firehoseConfig *firehose.Request, reducer []circuit.X) {
	circuit.Daemonize(func() {
		f := firehose.Redial(firehoseConfig)
		var n int64
		// Repeat forever: Read an event from the Firehose and pass is on to an appropriate reducer
		for {
			event := f.Read()
			p := &Post{}
			if event.Post != nil {
				p.ID = event.Post.ID
				p.Name = event.Post.BlogName
			}
			if event.Like != nil {
				p.ID = event.Like.RootPostID
			}
			p.Score = 1
			// XXX panic-protect
			reducer[int(p.ID%int64(len(reducer)))].Call("Add", p)
			n++
			if n%100 == 0 {
				println("Consumed", n, "events from the firehose")
			}
		}
	})
}
示例#2
0
func (StartMapper) Start(firehoseConfig *firehose.Request, reducer []circuit.X) {
	circuit.Daemonize(func() {
		f := firehose.Redial(firehoseConfig)
		for {
			poll(reducer, f)
		}
	})
}
示例#3
0
func StreamFirehose(freq *firehose.Request) <-chan *createRequest {
	ch := make(chan *createRequest)
	go func() {
		conn := firehose.Redial(freq)
		for {
			q := filter(conn.Read())
			if q == nil {
				continue
			}
			println(fmt.Sprintf("CREATE blogID=%d postID=%d", q.TimelineID, q.PostID))
			ch <- &createRequest{
				Forwarded: false,
				Post:      q,
				ReturnResponse: func(err error) {
					if err != nil {
						println("Firehose->XCreatePost error:", err.Error())
						return
					}
				},
			}
		}
	}()
	return ch
}