func MonitorFeeds(reg *registry.Registry) {
if reg.Feeds == "" {
return
}
f, err := os.Open(reg.Feeds)
if err != nil {
glog.Fatalln("Reading feeds:", err)
}
defer f.Close()
var feeds []Feed
if err := json.NewDecoder(f).Decode(&feeds); err != nil {
glog.Fatalln("Decoding feeds:", err)
}
db := reg.DB()
defer db.Session.Close()
for i := range feeds {
if err := db.C("feeds").FindId(feeds[i].DocType).One(&feeds[i]); err != nil && err != mgo.ErrNotFound {
glog.Fatalln("Finding existing feeds:", err)
}
feeds[i].stream, err = eventsource.Subscribe(feeds[i].Url, feeds[i].LastEventId)
if err == nil {
glog.Infof("Monitoring: %s", &feeds[i])
go monitor(reg, &feeds[i])
} else {
glog.Fatalln("Eventsource:", err)
}
}
}
func monitor(reg *registry.Registry, feed *Feed) {
db := reg.DB()
defer db.Session.Close()
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
for {
var fields map[string]interface{}
form := make(url.Values)
select {
case <-sig:
return
case event := <-feed.stream.Events:
if len(event.Data()) == 0 {
continue
}
if err := json.Unmarshal([]byte(event.Data()), &fields); err != nil {
glog.Errorln(feed.String(), err)
continue
}
feed.DocId++
id := &document.DocumentID{
Doctype: feed.DocType,
Docid: feed.DocId,
}
for k, v := range fields {
switch v.(type) {
case string:
form.Add(k, v.(string))
case []string:
form.Add(k, strings.Join(v.([]string), "|"))
}
}
doc, err := document.NewDocument(id, &form)
if err != nil {
glog.Errorln(err)
continue
}
if _, err := queue.NewQueueItem(r, "Add Document", nil, id, "", "", strings.NewReader(form.Encode())); err != nil {
glog.Infoln("Queueing add document:", err)
continue
}
glog.Infof("Received: %s %s", feed, doc.Title)
feed.LastEventId = event.Id()
if _, err := db.C("feeds").UpsertId(feed.DocType, feed); err != nil {
glog.Fatalln("Updating feeds:", err)
}
case err := <-feed.stream.Errors:
glog.Errorln(err)
}
}
}
func getQueueItem(id bson.ObjectId, registry *registry.Registry) (*QueueItem, error) {
var item QueueItem
db := registry.DB()
defer db.Session.Close()
if err := db.C("queue").FindId(id).Select(bson.M{"payload": 0}).One(&item); err != nil {
return nil, fmt.Errorf("Queue item not found: %s", err)
}
return &item, nil
}
func GetDocument(id *DocumentID, registry *registry.Registry) (*Document, error) {
doc := &Document{Id: *id}
db := registry.DB()
defer db.Session.Close()
if err := db.C("documents").FindId(doc.Id).One(doc); err != nil {
return nil, err
}
return doc.init(), nil
}
func Start(registry *registry.Registry) {
glog.Infoln("Starting Queue Processor")
registry.Queue = make(chan bool)
client, err := posting.NewClient(registry)
if err != nil {
panic(err)
}
defer client.Close()
if err = client.Initialise(); err != nil {
panic(err)
}
db := registry.DB()
defer db.Session.Close()
queue := db.C("queue")
registry.Routines.Add(1)
var items QueueItemSlice
for {
start := time.Now()
if err := queue.Find(bson.M{"status": "Queued"}).Sort("_id").Limit(10).All(&items); err != nil {
panic(err)
}
for i, item := range items {
if item.Command != items[0].Command {
items = items[:i]
break
}
}
if err := items.Execute(registry, client); err != nil {
glog.Errorln(err)
}
if len(items) > 0 {
glog.Infof("Executed %d Queue items in %.2f secs", len(items), time.Now().Sub(start).Seconds())
continue
}
select {
case <-registry.Queue:
glog.Infoln("Queue Processor Stopped")
registry.Routines.Done()
return
default:
time.Sleep(time.Second)
}
}
}
func GetQueue(values url.Values, registry *registry.Registry) (*QueueResult, error) {
var items []QueueItem
db := registry.DB()
defer db.Session.Close()
if err := db.C("queue").Find(nil).Select(bson.M{"payload": 0}).Sort("_id").All(&items); err != nil {
return nil, fmt.Errorf("Queue item not found: %s", err)
}
return &QueueResult{
Rows: items,
TotalRows: len(items),
Success: true,
}, nil
}
func Stats(registry *registry.Registry) (map[string]int, error) {
job := &mgo.MapReduce{
Map: "function() { emit(this.status, 1) }",
Reduce: "function(key, values) { return Array.sum(values) }",
}
var result []struct {
Id string "_id"
Value int
}
db := registry.DB()
defer db.Session.Close()
_, err := db.C("queue").Find(nil).MapReduce(job, &result)
if err != nil {
return nil, newQueueError("Queue Map Reduce Stats:", err)
}
stats := make(map[string]int)
for _, item := range result {
stats[item.Id] = item.Value
}
return stats, nil
}
func (q *QueueItem) UpdateStatus(registry *registry.Registry, status string) error {
db := registry.DB()
defer db.Session.Close()
return db.C("queue").UpdateId(q.Id, bson.M{"$set": bson.M{"status": status}})
}
func (q *QueueItem) Save(registry *registry.Registry) error {
db := registry.DB()
defer db.Session.Close()
_, err := db.C("queue").UpsertId(q.Id, q)
return err
}
func (m ThemeMap) Save(registry *registry.Registry) error {
db := registry.DB()
defer db.Session.Close()
return db.C("theme").Insert(reflect.ValueOf(m).MapKeys())
}
func (document *Document) Delete(registry *registry.Registry) error {
db := registry.DB()
defer db.Session.Close()
return db.C("documents").RemoveId(document.Id)
}
func (document *Document) Save(registry *registry.Registry) error {
db := registry.DB()
defer db.Session.Close()
_, err := db.C("documents").UpsertId(document.Id, document)
return err
}
