func write(log map[string]string) error { if logFileName != "" { f, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, logFilePermit) if err != nil { logger.Println("Open log failed.") return nil } defer f.Close() ltsvLog := convertLTSV(log) if _, err = f.WriteString(ltsvLog); err != nil { logger.Println("Write log failed.") } } if fluentTag != "" { logger, err := fluent.New(fluent.Config{FluentPort: fluentPort, FluentHost: fluentHost}) if err != nil { return err } defer logger.Close() error := logger.Post(fluentTag, log) if error != nil { panic(error) } } return nil }
// connectToFluentd continuously tries to connect to Fluentd. func connectToFluentd() *fluent.Fluent { var err error var logger *fluent.Fluent // Continuously try to connect to Fluentd. backoff := time.Duration(*fluentdRetryWait) * time.Millisecond for { log.Printf("Connecting to Fluentd (%s:%d)...", *fluentdHost, *fluentdPort) logger, err = fluent.New(fluent.Config{ FluentHost: *fluentdHost, FluentPort: *fluentdPort, // Once we have a connection, the library will reconnect automatically // if the connection is lost. However, it panics if it fails to connect // more than MaxRetry times. To avoid panics crashing the server, retry // many times before panicking. MaxRetry: 240, RetryWait: *fluentdRetryWait, }) if err != nil { log.Printf("Could not connect to Fluentd: %v", err) time.Sleep(backoff) backoff *= 2 } else { log.Printf("Connected to Fluentd (%s:%d)...", *fluentdHost, *fluentdPort) return logger } } }
// Reads configuration and connects to fluentd func Init() { var err error var timeout time.Duration = 0 timeStr, exists := revel.Config.String("revfluent.timeout") if exists { timeout, err = time.ParseDuration(timeStr) if err != nil { revel.ERROR.Panic(err) } } var retryWait int = 0 retryWaitStr, exists := revel.Config.String("revfluent.retryWait") if exists { tmp, err := time.ParseDuration(retryWaitStr) if err != nil { revel.ERROR.Panic(err) } retryWait = int(tmp.Nanoseconds() / 1e6) } port, _ := revel.Config.Int("revfluent.port") host, _ := revel.Config.String("revfluent.host") network, _ := revel.Config.String("revfluent.network") socketPath, _ := revel.Config.String("revfluent.socketPath") bufferLimit, _ := revel.Config.Int("revfluent.bufferLimit") maxRetry, _ := revel.Config.Int("revfluent.maxRetry") appName, _ := revel.Config.String("app.name") tagPrefix := revel.Config.StringDefault("revfluent.tagPrefix", appName) config := fluent.Config{ FluentPort: port, FluentHost: host, FluentNetwork: network, FluentSocketPath: socketPath, Timeout: timeout, BufferLimit: bufferLimit, RetryWait: retryWait, MaxRetry: maxRetry, TagPrefix: tagPrefix, } var hostUsed string if hostUsed = host; host == "" { hostUsed = "127.0.0.1" } var portUsed int if portUsed = port; port == 0 { portUsed = 24224 } revel.INFO.Printf("Connecting Fluentd: %s:%d", hostUsed, portUsed) Logger, err = fluent.New(config) if err != nil { revel.ERROR.Panic("Failed to connect Fluentd: %s", err) } }
func (hook *fluentHook) Fire(entry *logrus.Entry) error { logger, err := fluent.New(fluent.Config{ FluentHost: hook.host, FluentPort: hook.port, }) if err != nil { return err } defer logger.Close() // Create a map for passing to FluentD data := make(logrus.Fields) for k, v := range entry.Data { data[k] = v } setLevelString(entry, data) tag := getTagAndDel(entry, data) if tag != entry.Message { setMessage(entry, data) } fluentData := ConvertToValue(data, TagName) err = logger.PostWithTime(tag, entry.Time, fluentData) return err }
// New creates a fluentd logger using the configuration passed in on // the context. Supported context configuration variables are // fluentd-address & fluentd-tag. func New(ctx logger.Context) (logger.Logger, error) { host, port, err := parseAddress(ctx.Config["fluentd-address"]) if err != nil { return nil, err } tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}") if err != nil { return nil, err } extra := ctx.ExtraAttributes(nil) logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s, extra:%v.", ctx.ContainerID, host, port, tag, extra) // logger tries to reconnect 2**32 - 1 times // failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds] log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32}) if err != nil { return nil, err } return &fluentd{ tag: tag, containerID: ctx.ContainerID, containerName: ctx.ContainerName, writer: log, extra: extra, }, nil }
func main() { var tag = flag.String("tag", "syslog", "fleuntd tag for logging") var fluent_socket = flag.String("socket", "/tmp/td-agent.sock", "fleuntd socket for logging") var task_name = flag.String("task", "test", "task name") var message_subject = flag.String("subject", "subject", "message subject") var message_enable = flag.Bool("debug", false, "send email") flag.Parse() // fmt.Println(*tag) logger, err := fluent.New(fluent.Config{FluentSocketPath: *fluent_socket, FluentNetwork: "unix"}) if err != nil { log.Fatal(err) } var message []string bio := bufio.NewReader(os.Stdin) for { line, _, err := bio.ReadLine() if err != nil { break } message = append(message, string(line)) } // fmt.Println(time.Now(), "\n", strings.Join(message, "\n") ) defer logger.Close() var data = map[string]string{ "message": strings.Join(message, "\n"), "timestamp": time.Now().String()} if err = logger.Post(*tag, data); err != nil { log.Fatal(err) } d := gomail.NewPlainDialer("127.0.0.1", 25, "", "") s, err := d.Dial() if err != nil { log.Fatal(err) } m := gomail.NewMessage() m.SetHeader("From", "from@mail") m.SetHeader("To", "to@mail") m.SetHeader("Subject", "go-fluentd-stdin") m.SetBody("text/html", strings.Join(message, "\n")) if err := gomail.Send(s, m); err != nil { log.Printf("Could not send email to %q: %v", "to@mail", err) } m.Reset() }
func main() { var dbid = flag.String("database", "", "database identifier") var path = flag.String("path", "", "log file path") var fluentHost = flag.String("fluent-host", "", "fluentd hostname") var fluentPort = flag.Int("fluent-port", 24224, "fluentd forward port (default 24224)") var fluentTag = flag.String("fluent-tag", "mysql.slowquery", "fluentd tag") var raw = flag.Bool("raw", false, "output raw data (stdout only)") flag.Parse() if *dbid == "" || *path == "" { fmt.Println("Usage: rds-throwlog --database=[database identifier] --path=[log file path]") return } stream, err := Fetch(dbid, path) if err != nil { log.Println(err) return } defer stream.Close() log.Println("download completed") // output raw if *raw { io.Copy(os.Stdout, stream) return } // prepare fluent-logger var logger *fluent.Fluent if *fluentHost != "" { logger, err = fluent.New(fluent.Config{ FluentPort: *fluentPort, FluentHost: *fluentHost, }) if err != nil { log.Println("fluent.New returned error:", err) return } } records := mysqlslow.Parse(stream) for _, r := range records { if logger != nil { t, msg := r.ToFluentLog() log.Println(t) logger.PostWithTime(*fluentTag, t, msg) } else { data, err := json.Marshal(r) if err != nil { fmt.Println(err) } fmt.Println(string(data)) } } }
func (p *Output) Start() error { gigo.Debugf(p.logger, "out_fluent: start") if p.output != nil { return fmt.Errorf("already started") } output, err := fluent.New(p.config) if err != nil { return err } p.output = output return nil }
func NewHook(host string, port int) (*fluentHook, error) { logger, err := fluent.New(fluent.Config{ FluentHost: host, FluentPort: port, }) if err != nil { return nil, err } return &fluentHook{ Logger: logger, levels: defaultLevels, }, nil }
// New returns initialized logrus hook for fluentd with persistent fluentd logger. func New(host string, port int) (*FluentHook, error) { fd, err := fluent.New(fluent.Config{FluentHost: host, FluentPort: port}) if err != nil { return nil, err } return &FluentHook{ levels: defaultLevels, Fluent: fd, ignoreFields: make(map[string]struct{}), filters: make(map[string]func(interface{}) interface{}), }, nil }
func main() { var err error logger, err = fluent.New(fluent.Config{ FluentHost: "localhost", FluentPort: 24224, }) if err != nil { log.Fatal(err) } http.HandleFunc("/demo", demoHandler) http.ListenAndServe(":8080", nil) }
func SetupFluentLogger(cfg *Config) error { if cfg.Fluent.FluentHost == "" { logger.Infof(mylog, "The fluentd host is not specified. Skipping fluent logger instantiation.") return nil } logger.Infof(mylog, "Initializing fluent logger based on config: %+v", cfg.Fluent) fcli, err := gfluent.New(cfg.Fluent) if err != nil { return err } logger.Registry().AddOutput(fluent.FluentLogger{fcli}) return nil }
func print(s severity, args ...interface{}) { //connect fluent server f, err := fluent.New(fluent.Config{ FluentPort: 24224, FluentHost: "localhost", TagPrefix: "goluent." + getHostname(), }) message := fmt.Sprint(args...) stdLog.Println(message) if err == nil { f.Post(severityName[s], map[string]string{"message": message}) } }
func main() { logger, err := fluent.New(fluent.Config{FluentPort: 24224, FluentHost: "127.0.0.1"}) if err != nil { fmt.Println(err) } defer logger.Close() tag := "myapp.access" var data = map[string]string{ "foo": "bar", "hoge": "hoge"} i := 0 for i < 100 { logger.Post(tag, data) i = i + 1 } }
func main() { lambda_proc.Run(func(context *lambda_proc.Context, eventJSON json.RawMessage) (interface{}, error) { fluentURL, err := getfluentURL(context.FunctionName) fmt.Fprintf(os.Stderr, "fluentd connection config=%s %d\n", fluentURL.Host, fluentURL.Port) logger, err := fluent.New(fluent.Config{FluentPort: fluentURL.Port, FluentHost: fluentURL.Host}) if err != nil { fmt.Fprintf(os.Stderr, "fluentd connection error=%s\n", err) return nil, err } defer logger.Close() var event cloudwatchlogs.Event err = json.Unmarshal([]byte(eventJSON), &event) if err != nil { fmt.Fprintf(os.Stderr, "json.Unmarshal err=%s\n", err) return nil, err } d, err := event.AWSLogs.DecodedData() if err != nil { fmt.Fprintf(os.Stderr, "AWSLogs.DecodedData err=%s\n", err) return nil, err } logs, errs := 0, 0 for _, e := range d.LogEvents { event, err := decodeLogLine(e.Message) if err != nil { fmt.Fprintf(os.Stderr, "Error decoding log line err=%s\n", err) continue } tag := d.LogGroup err = logger.Post(tag, event) if err != nil { fmt.Fprintf(os.Stderr, "FluentD Post: %s\n", err) return nil, err } } return fmt.Sprintf("LogGroup=%s LogStream=%s MessageType=%s NumLogEvents=%d logs=%d errs=%d", d.LogGroup, d.LogStream, d.MessageType, len(d.LogEvents), logs, errs), nil }) }
func main() { defer util.Run()() var err error f_config := fluent.Config{FluentSocketPath: *fluent_socket, FluentNetwork: "unix"} logger, err = fluent.New(f_config) defer logger.Close() // log.Printf("starting capture on interface %q", *iface) // Set up pcap packet capture handle, err := pcap.OpenLive(*iface, int32(*snaplen), true, pcap.BlockForever) if err != nil { panic(err) } if err := handle.SetBPFFilter(*filter); err != nil { panic(err) } // Set up assembly streamFactory := &myFactory{bidiMap: make(map[key]*bidi)} streamPool := tcpassembly.NewStreamPool(streamFactory) assembler := tcpassembly.NewAssembler(streamPool) // log.Println("reading in packets") // Read in packets, pass to assembler. packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) packets := packetSource.Packets() ticker := time.Tick(timeout / 4) for { select { case packet := <-packets: if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP { //log.Println("Unusable packet") continue } tcp := packet.TransportLayer().(*layers.TCP) assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp) case <-ticker: // Every minute, flush connections that haven't seen activity in the past minute. // log.Println("---- FLUSHING ----") assembler.FlushOlderThan(time.Now().Add(-timeout)) streamFactory.collectOldStreams() } } }
func (hook *fluentHook) Fire(entry *logrus.Entry) error { logger, err := fluent.New(fluent.Config{ FluentHost: hook.host, FluentPort: hook.port, }) if err != nil { return err } defer logger.Close() setLevelString(entry) tag := getTagAndDel(entry) if tag != entry.Message { setMessage(entry) } data := ConvertToValue(entry.Data, TagName) err = logger.PostWithTime(tag, entry.Time, data) return err }
// New creates a fluentd logger using the configuration passed in on // the context. Supported context configuration variables are // fluentd-address & fluentd-tag. func New(ctx logger.Context) (logger.Logger, error) { host, port, tag, err := parseConfig(ctx) if err != nil { return nil, err } logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s.", ctx.ContainerID, host, port, tag) // logger tries to recoonect 2**32 - 1 times // failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds] log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32}) if err != nil { return nil, err } return &fluentd{ tag: tag, containerID: ctx.ContainerID, containerName: ctx.ContainerName, writer: log, }, nil }
func setupFluentd(fluentHost string) func(interface{}) { parts := strings.Split(fluentHost, ":") if len(parts) != 2 { fmt.Fprintf(os.Stderr, "invalid fluentd host format, should be <hostname>:<port>") os.Exit(1) } port, err := strconv.Atoi(parts[1]) if err != nil { fmt.Fprintf(os.Stderr, "invalid port for fluentd") os.Exit(1) } cfg := fluent.Config{FluentHost: parts[0], FluentPort: port} logger, err := fluent.New(cfg) return func(message interface{}) { spew.Sprintln(message) logger.Post("bouncer", message) } }
// Fire is invoked by logrus and sends log to fluentd logger. func (hook *FluentHook) Fire(entry *logrus.Entry) error { var logger *fluent.Fluent var err error switch { case hook.Fluent != nil: logger = hook.Fluent default: logger, err = fluent.New(fluent.Config{ FluentHost: hook.host, FluentPort: hook.port, }) if err != nil { return err } defer logger.Close() } // Create a map for passing to FluentD data := make(logrus.Fields) for k, v := range entry.Data { if _, ok := hook.ignoreFields[k]; ok { continue } if fn, ok := hook.filters[k]; ok { v = fn(v) } data[k] = v } setLevelString(entry, data) tag := hook.getTagAndDel(entry, data) if tag != entry.Message { setMessage(entry, data) } fluentData := ConvertToValue(data, TagName) err = logger.PostWithTime(tag, entry.Time, fluentData) return err }
func main() { logger, _ := fluent.New(fluent.Config{}) tag2 := "myapp.errors" defer func() { if e := recover(); e != nil { stack := make([]byte, 1<<16) sz := runtime.Stack(stack, true) stackstr := string(stack[:sz]) var data3 = map[string]string{ "app_id": "command-underline", "version": "v2.0.0", "message": "A slightly modified error occurred", "error_message": "This is my error other messages", "function": "main", "exception": stackstr, "environment": "prod", } _ = logger.Post(tag2, data3) } }() panic("something went horribly wrong") }
func logPageView(u *ShortenedUrl) { // Setup the logger logger, err := fluent.New(fluent.Config{FluentPort: config.FluentPort, FluentHost: config.FluentHost}) if err != nil { Warning.Println("Failed to connect to Fluent") return } defer logger.Close() // Build the log data var data = map[string]string{ "a": "3", "u": u.Url, "s": u.ShortUrl, } // Send to fluent error := logger.Post(config.FluentTag, data) if error != nil { Warning.Printf("Failed to send data to fluentd") } }
func (f *FluentDriver) Configure(config map[string]string) (err error) { fluentConfig := fluent.Config{} host, ok := config["host"] if ok { fluentConfig.FluentHost = host } port, ok := config["port"] if ok { fluentConfig.FluentPort, err = strconv.Atoi(port) if err != nil { return } } timeout, ok := config["timeout"] if ok { fluentConfig.Timeout, err = time.ParseDuration(timeout) if err != nil { return } } tagPrefix, ok := config["prefix"] if ok { fluentConfig.TagPrefix = tagPrefix } f.Tag, ok = config["tag"] if !ok { return errors.New("tag is required") } f.fluent, err = fluent.New(fluentConfig) return }
// New creates a fluentd logger using the configuration passed in on // the context. Supported context configuration variables are // fluentd-address & fluentd-tag. func New(ctx logger.Context) (logger.Logger, error) { host, port, err := parseAddress(ctx.Config["fluentd-address"]) if err != nil { return nil, err } tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}") if err != nil { return nil, err } failOnStartupError, err := loggerutils.ParseFailOnStartupErrorFlag(ctx) if err != nil { return nil, err } bufferLimit, err := parseBufferLimit(ctx.Config["buffer-limit"]) if err != nil { return nil, err } extra := ctx.ExtraAttributes(nil) logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s, extra:%v.", ctx.ContainerID, host, port, tag, extra) // logger tries to reconnect 2**32 - 1 times // failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds] log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32, BufferLimit: bufferLimit}) if err != nil { if failOnStartupError { return nil, err } logrus.Warnf("fluentd cannot connect to configured endpoint. Ignoring as instructed. Error: %q", err) } return &fluentd{ tag: tag, containerID: ctx.ContainerID, containerName: ctx.ContainerName, writer: log, extra: extra, }, nil }
func NewFluentdLogger(tag, host, pid string) LoggerInterface { logger, _ := fluent.New(fluent.Config{}) return &FluentdLogger{logger: logger, tag: tag, host: host, pid: pid} }
func main() { var tag = flag.String("tag", "syslog", "fleuntd tag for logging") flag.Parse() // fmt.Println(*tag) logger, err := fluent.New(fluent.Config{FluentSocketPath: "/tmp/td-agent.sock", FluentNetwork: "unix"}) if err != nil { fmt.Println(err) } var message []string cmd := exec.Command("./test.sh", "2>&1") output, err := cmd.StdoutPipe() if err != nil { fmt.Println(err) } cmd.Start() bio := bufio.NewReader(output) // for { // line, _, err := bio.ReadLine(); // if err != nil { // break // } // // } done := make(chan error, 1) go func() { done <- cmd.Wait() }() select { case <-time.After(3 * time.Second): output.Close() if err := cmd.Process.Kill(); err != nil { } <-done // allow goroutine to exit case err := <-done: if err != nil { line, _, err := bio.ReadLine() if err != nil { break } message = append(message, string(line)) } } // fmt.Println(time.Now(), "\n", strings.Join(message, "\n") ) defer logger.Close() var data = map[string]string{ "message": strings.Join(message, "\n"), "timestamp": time.Now().String()} error := logger.Post(*tag, data) if error != nil { panic(error) } }
// New creates a fluentd logger using the configuration passed in on // the context. The supported context configuration variable is // fluentd-address. func New(ctx logger.Context) (logger.Logger, error) { loc, err := parseAddress(ctx.Config[addressKey]) if err != nil { return nil, err } tag, err := loggerutils.ParseLogTag(ctx, loggerutils.DefaultTemplate) if err != nil { return nil, err } extra := ctx.ExtraAttributes(nil) bufferLimit := defaultBufferLimit if ctx.Config[bufferLimitKey] != "" { bl64, err := units.RAMInBytes(ctx.Config[bufferLimitKey]) if err != nil { return nil, err } bufferLimit = int(bl64) } retryWait := defaultRetryWait if ctx.Config[retryWaitKey] != "" { rwd, err := time.ParseDuration(ctx.Config[retryWaitKey]) if err != nil { return nil, err } retryWait = int(rwd.Seconds() * 1000) } maxRetries := defaultMaxRetries if ctx.Config[maxRetriesKey] != "" { mr64, err := strconv.ParseUint(ctx.Config[maxRetriesKey], 10, strconv.IntSize) if err != nil { return nil, err } maxRetries = int(mr64) } asyncConnect := false if ctx.Config[asyncConnectKey] != "" { if asyncConnect, err = strconv.ParseBool(ctx.Config[asyncConnectKey]); err != nil { return nil, err } } fluentConfig := fluent.Config{ FluentPort: loc.port, FluentHost: loc.host, FluentNetwork: loc.protocol, FluentSocketPath: loc.path, BufferLimit: bufferLimit, RetryWait: retryWait, MaxRetry: maxRetries, AsyncConnect: asyncConnect, } logrus.WithField("container", ctx.ContainerID).WithField("config", fluentConfig). Debug("logging driver fluentd configured") log, err := fluent.New(fluentConfig) if err != nil { return nil, err } return &fluentd{ tag: tag, containerID: ctx.ContainerID, containerName: ctx.ContainerName, writer: log, extra: extra, }, nil }