func (input *FilePollingInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error { input.runner = runner input.hostname = helper.PipelineConfig().Hostname() tickChan := runner.Ticker() sRunner := runner.NewSplitterRunner("") if !sRunner.UseMsgBytes() { sRunner.SetPackDecorator(input.packDecorator) } for { select { case <-input.stop: return nil case <-tickChan: } f, err := os.Open(input.FilePath) if err != nil { runner.LogError(fmt.Errorf("Error opening file: %s", err.Error())) continue } for err == nil { err = sRunner.SplitStream(f, nil) if err != io.EOF && err != nil { runner.LogError(fmt.Errorf("Error reading file: %s", err.Error())) } } } return nil }
func (s *SandboxInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) (err error) { s.sb.InjectMessage(func(payload, payload_type, payload_name string) int { pack := <-ir.InChan() if err := proto.Unmarshal([]byte(payload), pack.Message); err != nil { pack.Recycle() return 1 } if s.tz != time.UTC { const layout = "2006-01-02T15:04:05.999999999" // remove the incorrect UTC tz info t := time.Unix(0, pack.Message.GetTimestamp()) t = t.In(time.UTC) ct, _ := time.ParseInLocation(layout, t.Format(layout), s.tz) pack.Message.SetTimestamp(ct.UnixNano()) } ir.Inject(pack) atomic.AddInt64(&s.processMessageCount, 1) atomic.AddInt64(&s.processMessageBytes, int64(len(payload))) return 0 }) ticker := ir.Ticker() for true { retval := s.sb.ProcessMessage(nil) if retval <= 0 { // Sandbox is in polling mode if retval < 0 { atomic.AddInt64(&s.processMessageFailures, 1) em := s.sb.LastError() if len(em) > 0 { ir.LogError(errors.New(em)) } } if ticker == nil { ir.LogMessage("single run completed") break } select { // block until stop or poll interval case <-s.stopChan: case <-ticker: } } else { // Sandbox is shutting down em := s.sb.LastError() if !strings.HasSuffix(em, "shutting down") { ir.LogError(errors.New(em)) } break } } s.reportLock.Lock() if s.sbc.PreserveData { err = s.sb.Destroy(s.preservationFile) } else { err = s.sb.Destroy("") } s.sb = nil s.reportLock.Unlock() return }
func (input *FilePollingInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error { var ( data []byte pack *pipeline.PipelinePack dRunner pipeline.DecoderRunner ok bool err error ) if input.DecoderName != "" { if dRunner, ok = helper.DecoderRunner(input.DecoderName, fmt.Sprintf("%s-%s", runner.Name(), input.DecoderName)); !ok { return fmt.Errorf("Decoder not found: %s", input.DecoderName) } input.decoderChan = dRunner.InChan() } input.runner = runner hostname := helper.PipelineConfig().Hostname() packSupply := runner.InChan() tickChan := runner.Ticker() for { select { case <-input.stop: return nil case <-tickChan: } data, err = ioutil.ReadFile(input.FilePath) if err != nil { runner.LogError(fmt.Errorf("Error reading file: %s", err)) continue } pack = <-packSupply pack.Message.SetUuid(uuid.NewRandom()) pack.Message.SetTimestamp(time.Now().UnixNano()) pack.Message.SetType("heka.file.polling") pack.Message.SetHostname(hostname) pack.Message.SetPayload(string(data)) if field, err := message.NewField("TickerInterval", int(input.TickerInterval), ""); err != nil { runner.LogError(err) } else { pack.Message.AddField(field) } if field, err := message.NewField("FilePath", input.FilePath, ""); err != nil { runner.LogError(err) } else { pack.Message.AddField(field) } input.sendPack(pack) } return nil }
func (input *Sqs3Input) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error { // initialize input.runner = runner input.sqs = sqs.New(session.New()) input.s3 = s3.New(session.New()) queue_url, err := get_queue(input.sqs, input.SqsQueue) if err != nil { return err } input.queue_url = queue_url //input.hostname = helper.PipelineConfig().Hostname() tickChan := runner.Ticker() sRunner := runner.NewSplitterRunner("") if !sRunner.UseMsgBytes() { sRunner.SetPackDecorator(input.packDecorator) } defer sRunner.Done() for { select { case <-input.stop: return nil case <-tickChan: } receipt_handle, bucket, key, err := receive_from_queue(input.sqs, input.queue_url) if err != nil { runner.LogError(fmt.Errorf("Error reading queue: %s", err.Error())) continue } o, _, err := get_object(input.s3, bucket, key) if err != nil { runner.LogError(fmt.Errorf("Error opening file: %s", err.Error())) if aws_err := awserr.Error(err); aws_err != nil { f aws_err.Code == "NoSuchBucket" or aws_err.Code == "NoSuchKey" { delete_message(input.sqs, input.queue_url, receipt_handle) } } continue } for err == nil { err = sRunner.SplitStream(o, nil) if err != io.EOF && err != nil { runner.LogError(fmt.Errorf("Error reading file: %s", err.Error())) } } o.Close() } }
func (input *DockerStatsInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error { var pack *pipeline.PipelinePack input.runner = runner packSupply := runner.InChan() tickChan := runner.Ticker() hostname := helper.PipelineConfig().Hostname() for { select { case <-input.stop: return nil case <-tickChan: } var ( // test chan bool //err error previousCPU, previousSystem uint64 mstats *dockerStat preCPUStats, stats *docker.Stats ) endpoint := "unix:///var/run/docker.sock" client, _ := docker.NewClient(endpoint) containers, _ := client.ListContainers(docker.ListContainersOptions{Filters: map[string][]string{"status": {"running"}}}) for _, container := range containers { if containerName, exists := input.cacheHostnames[container.ID]; !exists { containerName = strings.Replace(container.Names[0], "/", "", -1) input.cacheHostnames[container.ID] = containerName if input.NameFromEnv != "" { con, _ := client.InspectContainer(container.ID) for _, value := range con.Config.Env { parts := strings.SplitN(value, "=", 2) if len(parts) == 2 { if input.NameFromEnv == parts[0] { containerName = parts[1] input.cacheHostnames[container.ID] = containerName break } } } } } opts := docker.StatsStaticOptions{ID: container.ID, Stream: false} preCPUStats, _ = client.StatsStatic(opts) if preCPUStats == nil { continue } previousCPU = preCPUStats.CPUStats.CPUUsage.TotalUsage previousSystem = preCPUStats.CPUStats.SystemCPUUsage stats, _ = client.StatsStatic(opts) if stats == nil { continue } mstats = &dockerStat{} mstats.CPUPercent = calculateCPUPercent(previousCPU, previousSystem, stats) mstats.MemPercent = calculateMemPercent(stats) mstats.MemUsage = stats.MemoryStats.Usage mstats.MemLimit = stats.MemoryStats.Limit mstats.BlockRead, mstats.BlockWrite = calculateBlockIO(stats) for _, networkstat := range stats.Networks { mstats.NetworkRx = networkstat.RxBytes mstats.NetworkTx = networkstat.TxBytes } pack = <-packSupply pack.Message.SetUuid(uuid.NewRandom()) pack.Message.SetTimestamp(time.Now().Unix()) pack.Message.SetType("DockerStats") pack.Message.SetHostname(hostname) containerName, _ := message.NewField("ContainerName", string(strings.Replace(input.cacheHostnames[container.ID], "-", "_", -1)), "") pack.Message.AddField(containerName) cpuPercent, _ := message.NewField("CPUPercent", float64(mstats.CPUPercent), "") pack.Message.AddField(cpuPercent) memPercent, _ := message.NewField("MemoryPercent", float64(mstats.MemPercent), "") pack.Message.AddField(memPercent) memLimit, _ := message.NewField("MemoryLimit", int64(mstats.MemLimit), "") pack.Message.AddField(memLimit) memUsage, _ := message.NewField("MemoryUsage", int64(mstats.MemUsage), "") pack.Message.AddField(memUsage) netInput, _ := message.NewField("NetworkInput", int64(mstats.NetworkRx), "") pack.Message.AddField(netInput) netOutput, _ := message.NewField("NetworkOutput", int64(mstats.NetworkTx), "") pack.Message.AddField(netOutput) blockInput, _ := message.NewField("BlockInput", int64(mstats.BlockRead), "") pack.Message.AddField(blockInput) blockOutput, _ := message.NewField("BlockOutput", int64(mstats.BlockWrite), "") pack.Message.AddField(blockOutput) pack.Message.SetPayload(fmt.Sprintf("container_name %s\ncpu %.2f\nmem_usage %d\nmem_limit %d\nmem %.2f\nnet_input %d\nnet_output %d\nblock_input %d\nblock_output %d", strings.Replace(input.cacheHostnames[container.ID], "-", "_", -1), mstats.CPUPercent, mstats.MemUsage, mstats.MemLimit, mstats.MemPercent, mstats.NetworkRx, mstats.NetworkTx, mstats.BlockRead, mstats.BlockWrite)) runner.Deliver(pack) } } return nil }