func Chew(chewChan <-chan *messaging.Food, swallowChan chan *messaging.Food, wg *sync.WaitGroup) {
log.Info("Let the chewing begin!")
defer close(swallowChan)
r := rep.NewReporter()
r.RegisterStatWIndex("chew", "good")
for msg := range chewChan {
if nil != msg {
//parsing work here probably change what our message type looks like when swallowed
date := time.Unix(0, msg.GetTimeNano()).UTC()
fmtDate := date.Format("2006-01-02")
indexType := "all"
customerId := "id" //should exist eventually
index := "documents-" + customerId + "-" + fmtDate
msg.Index = &index
msg.IndexType = &indexType
r.AddStatWIndex("chew", 1, "good")
swallowChan <- msg
}
}
log.Info("Done chewing")
log.Flush()
wg.Done()
}
func main() {
defer log.Flush()
logger, err := log.LoggerFromConfigAsFile("seelog.xml")
rep.ReporterConfig("ipc:///temp/testSender.ipc", 0)
r := rep.NewReporter()
defer r.Close()
if err != nil {
log.Warn("Failed to load config", err)
}
log.ReplaceLogger(logger)
start.Run()
}
//Swallow data and insert it into the db
func swallow(swallowChan <-chan *messaging.Food, stomach Stomach, wg *sync.WaitGroup) {
log.Info("Ready to swallow!")
r := rep.NewReporter()
r.RegisterStatWIndex("swallow", "RFC3164Bad")
r.RegisterStatWIndex("swallow", "RFC3164Good")
r.RegisterStatWIndex("swallow", "RFC5424Bad")
r.RegisterStatWIndex("swallow", "RFC5424Good")
r.RegisterStatWIndex("swallow", "JSONBad")
r.RegisterStatWIndex("swallow", "JSONGood")
r.RegisterStatWIndex("swallow", "Invalid")
for food := range swallowChan {
fType := food.GetType()
switch fType {
case messaging.RFC3164:
{
for _, v := range food.Rfc3164 {
err := stomach.IndexDocument(food.GetIndex(), food.GetIndexType(), v)
if nil != err {
log.Error(err)
r.AddStatWIndex("swallow", 1, "RFC3164Bad")
} else {
r.AddStatWIndex("swallow", 1, "RFC3164Good")
}
}
}
case messaging.RFC5424:
{
// log.Trace("RFC5424 :", food)
// stomach.IndexDocument(food)
}
case messaging.JSON:
{
var buf *bytes.Buffer
var msg interface{}
var err error
for _, v := range food.Json {
buf = bytes.NewBuffer(v.GetJson())
err = json.NewDecoder(buf).Decode(&msg)
if nil != err {
log.Error("Failed to decode json: ", err)
continue
}
err = stomach.IndexDocumentDynamic(food.GetIndex(), food.GetIndexType(), &msg, v.GetId())
if nil != err {
log.Error(err)
r.AddStatWIndex("swallow", 1, "JSONBad")
} else {
r.AddStatWIndex("swallow", 1, "JSONGood")
}
}
}
default:
{
r.AddStatWIndex("swallow", 1, "Invalid")
}
}
}
log.Info("Done Swallowing")
log.Flush()
wg.Done()
}
//Injest data from queue and ship the data off to be swallowed
func OpenWide(chewChan chan *messaging.Food, done chan interface{}, wg *sync.WaitGroup, port int) {
var (
msg []byte
err error
)
defer close(chewChan)
socket, err := nano.NewPullSocket()
if nil != err {
log.Error(err)
}
defer socket.Close()
r := rep.NewReporter()
//repeat stats with 0 if nothing is reported
r.RegisterStatWIndex("lips", "timeout")
r.RegisterStatWIndex("lips", "good")
r.RegisterStatWIndex("lips", "bad")
socket.SetRecvTimeout(1000 * time.Millisecond)
sport := strconv.Itoa(port)
_, err = socket.Bind("tcp://*:" + sport)
if nil != err {
log.Error(err)
}
log.Info("Connected and ready to receive data")
main:
for {
select {
case <-done:
{
log.Info("Got done signal")
break main
}
default:
{
msg, err = socket.Recv(0)
if nil != err {
r.AddStatWIndex("lips", 1, "timeout")
//we hit timeout
}
if nil != msg {
food := new(messaging.Food)
err = food.Unmarshal(msg)
if nil != err {
log.Error("Invalid message: ", err)
r.AddStatWIndex("lips", 1, "bad")
continue
}
r.AddStatWIndex("lips", 1, "good")
chewChan <- food
}
}
}
}
log.Info("Closing lips")
log.Flush()
wg.Done()
}