Exemplo n.º 1
0
func (q *UniqueQueue) pushUnique(item interface{}) bool {
	id := q.getID(item)

	log.WithFields(log.Fields{
		"ID":   id,
		"Item": item,
	}).Debug("Incoming Item")

	// We don't have this item queued
	if q.uniqueIDs[id] {
		log.WithFields(log.Fields{
			"ID":   id,
			"item": item,
		}).Debug("Item is already queued")
		return false
	}

	q.feeder <- item
	q.uniqueIDs[id] = true

	log.WithFields(log.Fields{
		"feeder":    len(q.feeder),
		"uniqueIDs": len(q.uniqueIDs),
	}).Debug("Incoming Queue Lengths")

	return true
}
Exemplo n.º 2
0
func (q *UniqueQueue) popTo(sendTo func(interface{})) {
	select {
	case item, feederOk := <-q.feeder:
		if !feederOk {
			log.Debug("Feeder Queue is closed")
			q.feederOk = false
			return
		}

		id := q.getID(item)

		log.WithFields(log.Fields{
			"ID":   id,
			"Item": item,
		}).Debug("Outgoing Item")

		sendTo(item)
		delete(q.uniqueIDs, id) // drop it from the set

		log.WithFields(log.Fields{
			"feeder":    len(q.feeder),
			"uniqueIDs": len(q.uniqueIDs),
		}).Debug("Outgoing Queue Lengths")

	default:
		// We don't want this to block the rest of the loop
	}
}
Exemplo n.º 3
0
// Push the data from one Q to the next, uniquifying on the URL
func (q *UniqueQueue) fifo() {
	// If the In is closed, there's nothing incoming:
	defer close(q.feeder)
	defer q.wg.Done()

	inQueueOk := true
	counter := 0

	for q.feederOk && (inQueueOk || len(q.feeder) > 0) {
		select {
		case newItem, inQueueOk := <-q.In:
			if inQueueOk {
				counter++
				log.WithFields(log.Fields{
					"Count": counter,
				}).Debug("Incoming item")
				q.pushUnique(newItem)
			}

		default:
			// We don't want this to block the rest of the loop
		}

		// If the Out has room:
		if len(q.Out) < cap(q.Out) {
			q.popTo(func(newItem interface{}) {
				q.Out <- newItem
			})
		}
	}
}
Exemplo n.º 4
0
func Warn(fields map[string]interface{}, msg string) {
	if fields == nil {
		jsonLog.Warn(msg)
		logrus.Warn(msg)
	} else {
		jsonLog.WithFields(fields).Warn(msg)
		logrus.WithFields(fields).Warn(msg)
	}
}
Exemplo n.º 5
0
// wrapper for Fatal
func Fatal(fields map[string]interface{}, msg string) {
	if fields == nil {
		logrus.Error(msg)
		jsonLog.Fatal(msg)
	} else {
		logrus.WithFields(fields).Error(msg)
		jsonLog.WithFields(fields).Fatal(msg)
	}
}