Пример #1
0
func Chew(chewChan <-chan *messaging.Food, swallowChan chan *messaging.Food, wg *sync.WaitGroup) {
	log.Info("Let the chewing begin!")
	defer close(swallowChan)
	r := rep.NewReporter()
	r.RegisterStatWIndex("chew", "good")
	for msg := range chewChan {
		if nil != msg {
			//parsing work here probably change what our message type looks like when swallowed

			date := time.Unix(0, msg.GetTimeNano()).UTC()
			fmtDate := date.Format("2006-01-02")
			indexType := "all"
			customerId := "id" //should exist eventually
			index := "documents-" + customerId + "-" + fmtDate
			msg.Index = &index
			msg.IndexType = &indexType
			r.AddStatWIndex("chew", 1, "good")
			swallowChan <- msg
		}
	}
	log.Info("Done chewing")
	log.Flush()
	wg.Done()

}
Пример #2
0
func main() {
	defer log.Flush()
	logger, err := log.LoggerFromConfigAsFile("seelog.xml")
	rep.ReporterConfig("ipc:///temp/testSender.ipc", 0)
	r := rep.NewReporter()
	defer r.Close()
	if err != nil {
		log.Warn("Failed to load config", err)
	}

	log.ReplaceLogger(logger)
	start.Run()
}
Пример #3
0
//Swallow data and insert it into the db
func swallow(swallowChan <-chan *messaging.Food, stomach Stomach, wg *sync.WaitGroup) {
	log.Info("Ready to swallow!")
	r := rep.NewReporter()
	r.RegisterStatWIndex("swallow", "RFC3164Bad")
	r.RegisterStatWIndex("swallow", "RFC3164Good")
	r.RegisterStatWIndex("swallow", "RFC5424Bad")
	r.RegisterStatWIndex("swallow", "RFC5424Good")
	r.RegisterStatWIndex("swallow", "JSONBad")
	r.RegisterStatWIndex("swallow", "JSONGood")
	r.RegisterStatWIndex("swallow", "Invalid")

	for food := range swallowChan {
		fType := food.GetType()
		switch fType {
		case messaging.RFC3164:
			{
				for _, v := range food.Rfc3164 {
					err := stomach.IndexDocument(food.GetIndex(), food.GetIndexType(), v)
					if nil != err {
						log.Error(err)
						r.AddStatWIndex("swallow", 1, "RFC3164Bad")
					} else {
						r.AddStatWIndex("swallow", 1, "RFC3164Good")
					}
				}
			}

		case messaging.RFC5424:
			{
				// log.Trace("RFC5424 :", food)
				// stomach.IndexDocument(food)
			}

		case messaging.JSON:
			{
				var buf *bytes.Buffer
				var msg interface{}
				var err error
				for _, v := range food.Json {
					buf = bytes.NewBuffer(v.GetJson())
					err = json.NewDecoder(buf).Decode(&msg)
					if nil != err {
						log.Error("Failed to decode json: ", err)
						continue
					}
					err = stomach.IndexDocumentDynamic(food.GetIndex(), food.GetIndexType(), &msg, v.GetId())
					if nil != err {
						log.Error(err)
						r.AddStatWIndex("swallow", 1, "JSONBad")
					} else {
						r.AddStatWIndex("swallow", 1, "JSONGood")
					}
				}
			}

		default:
			{
				r.AddStatWIndex("swallow", 1, "Invalid")
			}
		}
	}
	log.Info("Done Swallowing")
	log.Flush()
	wg.Done()

}
Пример #4
0
//Injest data from queue and ship the data off to be swallowed
func OpenWide(chewChan chan *messaging.Food, done chan interface{}, wg *sync.WaitGroup, port int) {
	var (
		msg []byte
		err error
	)

	defer close(chewChan)
	socket, err := nano.NewPullSocket()
	if nil != err {
		log.Error(err)
	}
	defer socket.Close()
	r := rep.NewReporter()
	//repeat stats with 0 if nothing is reported
	r.RegisterStatWIndex("lips", "timeout")
	r.RegisterStatWIndex("lips", "good")
	r.RegisterStatWIndex("lips", "bad")

	socket.SetRecvTimeout(1000 * time.Millisecond)
	sport := strconv.Itoa(port)
	_, err = socket.Bind("tcp://*:" + sport)
	if nil != err {
		log.Error(err)
	}

	log.Info("Connected and ready to receive data")
main:
	for {
		select {
		case <-done:
			{
				log.Info("Got done signal")
				break main
			}
		default:
			{
				msg, err = socket.Recv(0)
				if nil != err {
					r.AddStatWIndex("lips", 1, "timeout")
					//we hit timeout
				}
				if nil != msg {
					food := new(messaging.Food)
					err = food.Unmarshal(msg)
					if nil != err {
						log.Error("Invalid message: ", err)
						r.AddStatWIndex("lips", 1, "bad")
						continue
					}
					r.AddStatWIndex("lips", 1, "good")
					chewChan <- food

				}
			}
		}
	}

	log.Info("Closing lips")
	log.Flush()
	wg.Done()

}