// processData processes data incoming from devices and sends them over to Fluentd
func processData(accessToken string) {
var err error
// Connect to Fluentd
logger := connectToFluentd()
fluentdConnected = true
// The stream object reconnects with exponential backoff.
stream := connectToParticle(accessToken)
particleAPIConnected = true
// Now actually process events.
for {
// Block on the data/error channels.
select {
case event := <-stream.Events:
// Unmarshall the JSON data from the Particle API.
//////////////////////////////////////////////////////////////////
var m Message
jsonData := event.Data()
// The particle API often sends newlines.
// Perhaps as a keep-alive mechanism.
if jsonData == "" {
continue
}
err = json.Unmarshal([]byte(jsonData), &m)
if err != nil {
log.Printf("Could not parse message data: %v", err)
continue
}
// Read LTSV data from the device into map[string]string
//////////////////////////////////////////////////////////////////
reader := ltsv.NewReader(bytes.NewBufferString(m.Data))
records, err := reader.ReadAll()
if err != nil || len(records) != 1 {
log.Printf("Error reading LTSV data: %v", err)
continue
}
data := records[0]
log.Printf("Got data: %v", data)
// Put the data into jsonValue and send to Fluentd
//////////////////////////////////////////////////////////////////
jsonValue := make(map[string]interface{})
jsonValue["deviceid"] = m.Id
timestamp, err := strconv.ParseInt(data["timestamp"], 10, 64)
if err != nil {
log.Printf("Error reading timestamp: %v", err)
continue
}
jsonValue["timestamp"] = timestamp
addFloatValue("temp", jsonValue, data)
addFloatValue("humidity", jsonValue, data)
addFloatValue("pressure", jsonValue, data)
addFloatValue("windspeed", jsonValue, data)
addFloatValue("winddirection", jsonValue, data)
addFloatValue("rainfall", jsonValue, data)
DeviceChan <- jsonValue
// Send data directly to Fluentd
if err = logger.Post("aggre_mod.sensordata", jsonValue); err != nil {
log.Printf("Could not send data from %s to Fluentd: %v", m.Id, err)
} else {
log.Printf("Data processed (%s): %s", m.Id, data)
}
case err := <-stream.Errors:
log.Printf("Stream error: %v", err)
}
}
}
func main() {
kingpin.CommandLine.Help = "Access Log Profiler for LTSV (read from file or stdin)."
kingpin.Version("0.3.1")
kingpin.Parse()
var f *os.File
var err error
var c Config
if *config != "" {
c, err = LoadYAML(*config)
if err != nil {
log.Fatal(err)
}
}
option := Config{
File: *file,
Reverse: *reverse,
QueryString: *queryString,
Tsv: *tsv,
ApptimeLabel: *apptimeLabel,
ReqtimeLabel: *reqtimeLabel,
StatusLabel: *statusLabel,
SizeLabel: *sizeLabel,
MethodLabel: *methodLabel,
UriLabel: *uriLabel,
TimeLabel: *timeLabel,
Limit: *limit,
IncludesStr: *includes,
ExcludesStr: *excludes,
IncludeStatusesStr: *includeStatuses,
ExcludeStatusesStr: *excludeStatuses,
NoHeaders: *noHeaders,
AggregatesStr: *aggregates,
StartTime: *startTime,
EndTime: *endTime,
StartTimeDuration: *startTimeDuration,
EndTimeDuration: *endTimeDuration,
}
if *max {
c.Sort = "max"
} else if *min {
c.Sort = "min"
} else if *avg {
c.Sort = "avg"
} else if *sum {
c.Sort = "sum"
} else if *cnt {
c.Sort = "cnt"
} else if *sortUri {
c.Sort = "uri"
} else if *method {
c.Sort = "method"
} else if *maxBody {
c.Sort = "max-body"
} else if *minBody {
c.Sort = "min-body"
} else if *avgBody {
c.Sort = "avg-body"
} else if *sumBody {
c.Sort = "sum-body"
} else if *p1 {
c.Sort = "p1"
} else if *p50 {
c.Sort = "p50"
} else if *p99 {
c.Sort = "p99"
} else if *stddev {
c.Sort = "stddev"
} else {
if c.Sort == "" {
c.Sort = "max"
}
}
c = SetConfig(c, option)
if *load != "" {
accessLog, err = LoadProfiles(*load)
if err != nil {
log.Fatal(err)
}
SortProfiles(accessLog, c)
return
}
fileinfo, err := os.Stdin.Stat()
if err != nil {
log.Fatal(err)
}
if fileinfo.Mode()&os.ModeNamedPipe == 0 {
f, err = os.Open(c.File)
defer f.Close()
if err != nil {
log.Fatal(err)
}
} else {
f = os.Stdin
}
accessLog = make(Profiles, 0, c.Limit)
var includeRegexps []*regexp.Regexp
if len(c.Includes) > 0 {
includeRegexps = make([]*regexp.Regexp, 0, len(c.Includes))
for _, pattern := range c.Includes {
re, rerr := regexp.Compile(pattern)
if rerr != nil {
log.Fatal(err)
}
includeRegexps = append(includeRegexps, re)
}
}
var excludeRegexps []*regexp.Regexp
if len(c.Excludes) > 0 {
excludeRegexps = make([]*regexp.Regexp, 0, len(c.Excludes))
for _, pattern := range c.Excludes {
re, rerr := regexp.Compile(pattern)
if rerr != nil {
log.Fatal(err)
}
excludeRegexps = append(excludeRegexps, re)
}
}
var includeStatusRegexps []*regexp.Regexp
if len(c.IncludeStatuses) > 0 {
includeRegexps = make([]*regexp.Regexp, 0, len(c.Includes))
for _, pattern := range c.IncludeStatuses {
re, rerr := regexp.Compile(pattern)
if rerr != nil {
log.Fatal(err)
}
includeStatusRegexps = append(includeStatusRegexps, re)
}
}
var excludeStatusRegexps []*regexp.Regexp
if len(c.ExcludeStatuses) > 0 {
excludeRegexps = make([]*regexp.Regexp, 0, len(c.Excludes))
for _, pattern := range c.ExcludeStatuses {
re, rerr := regexp.Compile(pattern)
if rerr != nil {
log.Fatal(err)
}
excludeStatusRegexps = append(excludeStatusRegexps, re)
}
}
var aggregateRegexps []*regexp.Regexp
if len(c.Aggregates) > 0 {
aggregateRegexps = make([]*regexp.Regexp, 0, len(c.Aggregates))
for _, pattern := range c.Aggregates {
re, rerr := regexp.Compile(pattern)
if rerr != nil {
log.Fatal(err)
}
aggregateRegexps = append(aggregateRegexps, re)
}
}
var p parsetime.ParseTime
p, err = parsetime.NewParseTime(*location)
var sTimeNano int64
if c.StartTime != "" {
sTime, err := p.Parse(c.StartTime)
if err != nil {
log.Fatal(err)
}
sTimeNano = sTime.UnixNano()
}
if c.StartTimeDuration != "" {
sTime, err := TimeDurationSub(c.StartTimeDuration)
if err != nil {
log.Fatal(err)
}
sTimeNano = sTime.UnixNano()
}
var eTimeNano int64
if c.EndTime != "" {
eTime, err := p.Parse(c.EndTime)
if err != nil {
log.Fatal(err)
}
eTimeNano = eTime.UnixNano()
}
if c.EndTimeDuration != "" {
eTime, err := TimeDurationSub(c.EndTimeDuration)
if err != nil {
log.Fatal(err)
}
eTimeNano = eTime.UnixNano()
}
r := ltsv.NewReader(f)
Loop:
for {
line, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
resTime, err := strconv.ParseFloat(line[c.ApptimeLabel], 64)
if err != nil {
var reqTime float64
reqTime, err = strconv.ParseFloat(line[c.ReqtimeLabel], 64)
if err != nil {
continue
}
resTime = reqTime
}
bodySize, err := strconv.ParseFloat(line[c.SizeLabel], 64)
if err != nil {
continue
}
if sTimeNano != 0 || eTimeNano != 0 {
t, err := p.Parse(line[c.TimeLabel])
if err != nil {
continue
}
timeNano := t.UnixNano()
if !TimeCmp(sTimeNano, eTimeNano, timeNano) {
continue
}
}
u, err := url.Parse(line[c.UriLabel])
if err != nil {
continue
}
if c.QueryString {
v := url.Values{}
values := u.Query()
for q, _ := range values {
v.Set(q, "xxx")
}
uri = fmt.Sprintf("%s?%s", u.Path, v.Encode())
index = fmt.Sprintf("%s_%s?%s", line[c.MethodLabel], u.Path, v.Encode())
} else {
uri = u.Path
index = fmt.Sprintf("%s_%s", line[c.MethodLabel], u.Path)
}
if len(c.Includes) > 0 {
isnotMatched := true
for _, re := range includeRegexps {
if ok := re.Match([]byte(uri)); ok && err == nil {
isnotMatched = false
} else if err != nil {
log.Fatal(err)
}
}
if isnotMatched {
continue Loop
}
}
if len(c.Excludes) > 0 {
for _, re := range excludeRegexps {
if ok := re.Match([]byte(uri)); ok && err == nil {
continue Loop
} else if err != nil {
log.Fatal(err)
}
}
}
if len(c.IncludeStatuses) > 0 {
isnotMatched := true
for _, re := range includeStatusRegexps {
if ok := re.Match([]byte(line[c.StatusLabel])); ok && err == nil {
isnotMatched = false
} else if err != nil {
log.Fatal(err)
}
}
if isnotMatched {
continue Loop
}
}
if len(c.ExcludeStatuses) > 0 {
for _, re := range excludeStatusRegexps {
if ok := re.Match([]byte(line[c.StatusLabel])); ok && err == nil {
continue Loop
} else if err != nil {
log.Fatal(err)
}
}
}
isMatched := false
if len(c.Aggregates) > 0 {
for _, re := range aggregateRegexps {
if ok := re.Match([]byte(uri)); ok && err == nil {
isMatched = true
pattern := re.String()
index = fmt.Sprintf("%s_%s", line[c.MethodLabel], pattern)
uri = pattern
SetCursor(index, uri)
} else if err != nil {
log.Fatal(err)
}
}
}
if !isMatched {
SetCursor(index, uri)
}
if len(uriHints) > c.Limit {
log.Fatal(fmt.Sprintf("Too many uri (%d or less)", c.Limit))
}
if accessLog[cursor].Max < resTime {
accessLog[cursor].Max = resTime
}
if accessLog[cursor].Min >= resTime || accessLog[cursor].Min == 0 {
accessLog[cursor].Min = resTime
}
accessLog[cursor].Cnt++
accessLog[cursor].Sum += resTime
accessLog[cursor].Method = line[c.MethodLabel]
accessLog[cursor].Percentails = append(accessLog[cursor].Percentails, Percentail{RequestTime: resTime})
if accessLog[cursor].MaxBody < bodySize {
accessLog[cursor].MaxBody = bodySize
}
if accessLog[cursor].MinBody >= bodySize || accessLog[cursor].MinBody == 0 {
accessLog[cursor].MinBody = bodySize
}
accessLog[cursor].SumBody += bodySize
}
for i, _ := range accessLog {
sort.Sort(ByRequestTime{accessLog[i].Percentails})
accessLog[i].Avg = accessLog[i].Sum / float64(accessLog[i].Cnt)
accessLog[i].AvgBody = accessLog[i].SumBody / float64(accessLog[i].Cnt)
p1Len := LenPercentail(len(accessLog[i].Percentails), 1)
accessLog[i].P1 = accessLog[i].Percentails[p1Len].RequestTime
p50Len := LenPercentail(len(accessLog[i].Percentails), 50)
accessLog[i].P50 = accessLog[i].Percentails[p50Len].RequestTime
p99Len := LenPercentail(len(accessLog[i].Percentails), 99)
accessLog[i].P99 = accessLog[i].Percentails[p99Len].RequestTime
accessLog[i].Stddev = RequestTimeStddev(accessLog[i].Percentails, accessLog[i].Sum, accessLog[i].Avg)
}
SortProfiles(accessLog, c)
if *dump != "" {
err = DumpProfiles(*dump, accessLog)
if err != nil {
log.Fatal(err)
}
}
}