func (this *SelfSysInput) Run(r engine.InputRunner, h engine.PluginHelper) error { var ( globals = engine.Globals() stats = newSysStat() inChan = r.InChan() pack *engine.PipelinePack jsonString string err error stopped = false ) for !stopped { select { case <-this.stopChan: stopped = true case <-r.Ticker(): // same effect as sleep } if stopped { break } stats.gatherStats() jsonString, err = stats.jsonString() if err != nil { globals.Println(err) continue } pack = <-inChan if err = pack.Message.FromLine(fmt.Sprintf("als,%d,%s", time.Now().Unix(), jsonString)); err != nil { globals.Printf("invalid sys stat: %s\n", jsonString) pack.Recycle() continue } pack.Project = "als" pack.Ident = this.ident pack.EsIndex = "fun_als" pack.EsType = "sys" r.Inject(pack) } return nil }
func (this *ArchiveInput) doRunSingleLogfile(path string) { reader := als.NewAlsReader(path) if e := reader.Open(); e != nil { panic(e) } defer func() { reader.Close() this.workersWg.Done() atomic.AddInt32(&this.leftN, -1) <-this.workerNChan // release the lock }() var ( line []byte lineN int inChan = this.runner.InChan() err error project = this.h.Project(this.project) pack *engine.PipelinePack globals = engine.Globals() ) for !this.stopping { line, err = reader.ReadLine() switch err { case nil: lineN += 1 atomic.AddInt64(&this.lineN, 1) if globals.Verbose && lineN == 1 { project.Printf("[%s]started\n", path) } pack = <-inChan if err = pack.Message.FromLine(string(line)); err != nil { if project.ShowError && err != als.ErrEmptyLine { project.Printf("[%s]%v: %s", path, err, string(line)) } pack.Recycle() continue } pack.Ident = this.ident pack.Project = this.project pack.Logfile.SetPath(path) if globals.Debug { globals.Println(*pack) } this.runner.Inject(pack) case io.EOF: if globals.Verbose { project.Printf("[%s]done, lines: %d\n", path, lineN) } this.chkpnt.Put(path) this.chkpnt.Dump() return default: // unknown error panic(err) } } }
func (this *EsFilter) handlePack(pack *engine.PipelinePack, project *engine.ConfProject) bool { if pack.EsType == "" { pack.EsType = pack.Logfile.CamelCaseName() } if pack.EsIndex == "" { pack.EsIndex = indexName(project, this.indexPattern, time.Unix(int64(pack.Message.Timestamp), 0)) } // each ES item has area and ts fields pack.Ident = this.ident pack.Message.SetField("_area", pack.Message.Area) pack.Message.SetField("_t", pack.Message.Timestamp) for _, conv := range this.converters { for _, key := range conv.keys { if conv.normalizers != nil { for _, norm := range conv.normalizers { val, err := pack.Message.FieldValue(key, als.KEY_TYPE_STRING) if err != nil { // no such field break } normed := normalizers[norm].ReplaceAll([]byte(val.(string)), []byte("?")) val = string(normed) pack.Message.SetField(key+"_norm", val) } continue } switch conv.typ { case "money": amount, err := pack.Message.FieldValue(key, als.KEY_TYPE_MONEY) if err != nil { // has no such field continue } currency, err := pack.Message.FieldValue(conv.currency, als.KEY_TYPE_STRING) if err != nil { // has money field, but no currency field? return false } pack.Message.SetField("_usd", als.MoneyInUsdCents(currency.(string), amount.(int))) case "ip": ip, err := pack.Message.FieldValue(key, als.KEY_TYPE_IP) if err != nil { continue } pack.Message.SetField("_cntry", als.IpToCountry(ip.(string))) case "range": if len(conv.rang) < 2 { continue } val, err := pack.Message.FieldValue(key, als.KEY_TYPE_INT) if err != nil { continue } pack.Message.SetField(key+"_rg", als.GroupInt(val.(int), conv.rang)) case "del": pack.Message.DelField(key) } } } return true }