func monitor(reg *registry.Registry, feed *Feed) {
db := reg.DB()
defer db.Session.Close()
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
for {
var fields map[string]interface{}
form := make(url.Values)
select {
case <-sig:
return
case event := <-feed.stream.Events:
if len(event.Data()) == 0 {
continue
}
if err := json.Unmarshal([]byte(event.Data()), &fields); err != nil {
glog.Errorln(feed.String(), err)
continue
}
feed.DocId++
id := &document.DocumentID{
Doctype: feed.DocType,
Docid: feed.DocId,
}
for k, v := range fields {
switch v.(type) {
case string:
form.Add(k, v.(string))
case []string:
form.Add(k, strings.Join(v.([]string), "|"))
}
}
doc, err := document.NewDocument(id, &form)
if err != nil {
glog.Errorln(err)
continue
}
if _, err := queue.NewQueueItem(r, "Add Document", nil, id, "", "", strings.NewReader(form.Encode())); err != nil {
glog.Infoln("Queueing add document:", err)
continue
}
glog.Infof("Received: %s %s", feed, doc.Title)
feed.LastEventId = event.Id()
if _, err := db.C("feeds").UpsertId(feed.DocType, feed); err != nil {
glog.Fatalln("Updating feeds:", err)
}
case err := <-feed.stream.Errors:
glog.Errorln(err)
}
}
}
func AddDocument(item *QueueItem, registry *registry.Registry, client *posting.Client, c chan *QueueItemRun) {
values, err := item.PayloadValues()
if err != nil {
c <- runFailure(item, "Get Payload", err)
return
}
doc, err := document.NewDocument(item.Target, values)
if err != nil {
c <- runFailure(item, "New Document", err)
return
}
if err = doc.Save(registry); err != nil {
c <- runFailure(item, "Save Document", err)
return
}
if err = client.CallMultiple("Posting.Add", &document.DocumentArg{Id: &doc.Id}); err != nil {
c <- runFailure(item, "RPC Call", err)
return
}
c <- runSuccess(item)
}
