func (q *UniqueQueue) pushUnique(item interface{}) bool {
id := q.getID(item)
log.WithFields(log.Fields{
"ID": id,
"Item": item,
}).Debug("Incoming Item")
// We don't have this item queued
if q.uniqueIDs[id] {
log.WithFields(log.Fields{
"ID": id,
"item": item,
}).Debug("Item is already queued")
return false
}
q.feeder <- item
q.uniqueIDs[id] = true
log.WithFields(log.Fields{
"feeder": len(q.feeder),
"uniqueIDs": len(q.uniqueIDs),
}).Debug("Incoming Queue Lengths")
return true
}
func (q *UniqueQueue) popTo(sendTo func(interface{})) {
select {
case item, feederOk := <-q.feeder:
if !feederOk {
log.Debug("Feeder Queue is closed")
q.feederOk = false
return
}
id := q.getID(item)
log.WithFields(log.Fields{
"ID": id,
"Item": item,
}).Debug("Outgoing Item")
sendTo(item)
delete(q.uniqueIDs, id) // drop it from the set
log.WithFields(log.Fields{
"feeder": len(q.feeder),
"uniqueIDs": len(q.uniqueIDs),
}).Debug("Outgoing Queue Lengths")
default:
// We don't want this to block the rest of the loop
}
}
// Push the data from one Q to the next, uniquifying on the URL
func (q *UniqueQueue) fifo() {
// If the In is closed, there's nothing incoming:
defer close(q.feeder)
defer q.wg.Done()
inQueueOk := true
counter := 0
for q.feederOk && (inQueueOk || len(q.feeder) > 0) {
select {
case newItem, inQueueOk := <-q.In:
if inQueueOk {
counter++
log.WithFields(log.Fields{
"Count": counter,
}).Debug("Incoming item")
q.pushUnique(newItem)
}
default:
// We don't want this to block the rest of the loop
}
// If the Out has room:
if len(q.Out) < cap(q.Out) {
q.popTo(func(newItem interface{}) {
q.Out <- newItem
})
}
}
}
func Warn(fields map[string]interface{}, msg string) {
if fields == nil {
jsonLog.Warn(msg)
logrus.Warn(msg)
} else {
jsonLog.WithFields(fields).Warn(msg)
logrus.WithFields(fields).Warn(msg)
}
}
// wrapper for Fatal
func Fatal(fields map[string]interface{}, msg string) {
if fields == nil {
logrus.Error(msg)
jsonLog.Fatal(msg)
} else {
logrus.WithFields(fields).Error(msg)
jsonLog.WithFields(fields).Fatal(msg)
}
}