func write(log map[string]string) error {
if logFileName != "" {
f, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, logFilePermit)
if err != nil {
logger.Println("Open log failed.")
return nil
}
defer f.Close()
ltsvLog := convertLTSV(log)
if _, err = f.WriteString(ltsvLog); err != nil {
logger.Println("Write log failed.")
}
}
if fluentTag != "" {
logger, err := fluent.New(fluent.Config{FluentPort: fluentPort, FluentHost: fluentHost})
if err != nil {
return err
}
defer logger.Close()
error := logger.Post(fluentTag, log)
if error != nil {
panic(error)
}
}
return nil
}
// connectToFluentd continuously tries to connect to Fluentd.
func connectToFluentd() *fluent.Fluent {
var err error
var logger *fluent.Fluent
// Continuously try to connect to Fluentd.
backoff := time.Duration(*fluentdRetryWait) * time.Millisecond
for {
log.Printf("Connecting to Fluentd (%s:%d)...", *fluentdHost, *fluentdPort)
logger, err = fluent.New(fluent.Config{
FluentHost: *fluentdHost,
FluentPort: *fluentdPort,
// Once we have a connection, the library will reconnect automatically
// if the connection is lost. However, it panics if it fails to connect
// more than MaxRetry times. To avoid panics crashing the server, retry
// many times before panicking.
MaxRetry: 240,
RetryWait: *fluentdRetryWait,
})
if err != nil {
log.Printf("Could not connect to Fluentd: %v", err)
time.Sleep(backoff)
backoff *= 2
} else {
log.Printf("Connected to Fluentd (%s:%d)...", *fluentdHost, *fluentdPort)
return logger
}
}
}
// Reads configuration and connects to fluentd
func Init() {
var err error
var timeout time.Duration = 0
timeStr, exists := revel.Config.String("revfluent.timeout")
if exists {
timeout, err = time.ParseDuration(timeStr)
if err != nil {
revel.ERROR.Panic(err)
}
}
var retryWait int = 0
retryWaitStr, exists := revel.Config.String("revfluent.retryWait")
if exists {
tmp, err := time.ParseDuration(retryWaitStr)
if err != nil {
revel.ERROR.Panic(err)
}
retryWait = int(tmp.Nanoseconds() / 1e6)
}
port, _ := revel.Config.Int("revfluent.port")
host, _ := revel.Config.String("revfluent.host")
network, _ := revel.Config.String("revfluent.network")
socketPath, _ := revel.Config.String("revfluent.socketPath")
bufferLimit, _ := revel.Config.Int("revfluent.bufferLimit")
maxRetry, _ := revel.Config.Int("revfluent.maxRetry")
appName, _ := revel.Config.String("app.name")
tagPrefix := revel.Config.StringDefault("revfluent.tagPrefix", appName)
config := fluent.Config{
FluentPort: port,
FluentHost: host,
FluentNetwork: network,
FluentSocketPath: socketPath,
Timeout: timeout,
BufferLimit: bufferLimit,
RetryWait: retryWait,
MaxRetry: maxRetry,
TagPrefix: tagPrefix,
}
var hostUsed string
if hostUsed = host; host == "" {
hostUsed = "127.0.0.1"
}
var portUsed int
if portUsed = port; port == 0 {
portUsed = 24224
}
revel.INFO.Printf("Connecting Fluentd: %s:%d", hostUsed, portUsed)
Logger, err = fluent.New(config)
if err != nil {
revel.ERROR.Panic("Failed to connect Fluentd: %s", err)
}
}
func (hook *fluentHook) Fire(entry *logrus.Entry) error {
logger, err := fluent.New(fluent.Config{
FluentHost: hook.host,
FluentPort: hook.port,
})
if err != nil {
return err
}
defer logger.Close()
// Create a map for passing to FluentD
data := make(logrus.Fields)
for k, v := range entry.Data {
data[k] = v
}
setLevelString(entry, data)
tag := getTagAndDel(entry, data)
if tag != entry.Message {
setMessage(entry, data)
}
fluentData := ConvertToValue(data, TagName)
err = logger.PostWithTime(tag, entry.Time, fluentData)
return err
}
// New creates a fluentd logger using the configuration passed in on
// the context. Supported context configuration variables are
// fluentd-address & fluentd-tag.
func New(ctx logger.Context) (logger.Logger, error) {
host, port, err := parseAddress(ctx.Config["fluentd-address"])
if err != nil {
return nil, err
}
tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}")
if err != nil {
return nil, err
}
extra := ctx.ExtraAttributes(nil)
logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s, extra:%v.", ctx.ContainerID, host, port, tag, extra)
// logger tries to reconnect 2**32 - 1 times
// failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds]
log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32})
if err != nil {
return nil, err
}
return &fluentd{
tag: tag,
containerID: ctx.ContainerID,
containerName: ctx.ContainerName,
writer: log,
extra: extra,
}, nil
}
func main() {
var tag = flag.String("tag", "syslog", "fleuntd tag for logging")
var fluent_socket = flag.String("socket", "/tmp/td-agent.sock", "fleuntd socket for logging")
var task_name = flag.String("task", "test", "task name")
var message_subject = flag.String("subject", "subject", "message subject")
var message_enable = flag.Bool("debug", false, "send email")
flag.Parse()
// fmt.Println(*tag)
logger, err := fluent.New(fluent.Config{FluentSocketPath: *fluent_socket, FluentNetwork: "unix"})
if err != nil {
log.Fatal(err)
}
var message []string
bio := bufio.NewReader(os.Stdin)
for {
line, _, err := bio.ReadLine()
if err != nil {
break
}
message = append(message, string(line))
}
// fmt.Println(time.Now(), "\n", strings.Join(message, "\n") )
defer logger.Close()
var data = map[string]string{
"message": strings.Join(message, "\n"),
"timestamp": time.Now().String()}
if err = logger.Post(*tag, data); err != nil {
log.Fatal(err)
}
d := gomail.NewPlainDialer("127.0.0.1", 25, "", "")
s, err := d.Dial()
if err != nil {
log.Fatal(err)
}
m := gomail.NewMessage()
m.SetHeader("From", "from@mail")
m.SetHeader("To", "to@mail")
m.SetHeader("Subject", "go-fluentd-stdin")
m.SetBody("text/html", strings.Join(message, "\n"))
if err := gomail.Send(s, m); err != nil {
log.Printf("Could not send email to %q: %v", "to@mail", err)
}
m.Reset()
}
func main() {
var dbid = flag.String("database", "", "database identifier")
var path = flag.String("path", "", "log file path")
var fluentHost = flag.String("fluent-host", "", "fluentd hostname")
var fluentPort = flag.Int("fluent-port", 24224, "fluentd forward port (default 24224)")
var fluentTag = flag.String("fluent-tag", "mysql.slowquery", "fluentd tag")
var raw = flag.Bool("raw", false, "output raw data (stdout only)")
flag.Parse()
if *dbid == "" || *path == "" {
fmt.Println("Usage: rds-throwlog --database=[database identifier] --path=[log file path]")
return
}
stream, err := Fetch(dbid, path)
if err != nil {
log.Println(err)
return
}
defer stream.Close()
log.Println("download completed")
// output raw
if *raw {
io.Copy(os.Stdout, stream)
return
}
// prepare fluent-logger
var logger *fluent.Fluent
if *fluentHost != "" {
logger, err = fluent.New(fluent.Config{
FluentPort: *fluentPort,
FluentHost: *fluentHost,
})
if err != nil {
log.Println("fluent.New returned error:", err)
return
}
}
records := mysqlslow.Parse(stream)
for _, r := range records {
if logger != nil {
t, msg := r.ToFluentLog()
log.Println(t)
logger.PostWithTime(*fluentTag, t, msg)
} else {
data, err := json.Marshal(r)
if err != nil {
fmt.Println(err)
}
fmt.Println(string(data))
}
}
}
func (p *Output) Start() error {
gigo.Debugf(p.logger, "out_fluent: start")
if p.output != nil {
return fmt.Errorf("already started")
}
output, err := fluent.New(p.config)
if err != nil {
return err
}
p.output = output
return nil
}
func NewHook(host string, port int) (*fluentHook, error) {
logger, err := fluent.New(fluent.Config{
FluentHost: host,
FluentPort: port,
})
if err != nil {
return nil, err
}
return &fluentHook{
Logger: logger,
levels: defaultLevels,
}, nil
}
// New returns initialized logrus hook for fluentd with persistent fluentd logger.
func New(host string, port int) (*FluentHook, error) {
fd, err := fluent.New(fluent.Config{FluentHost: host, FluentPort: port})
if err != nil {
return nil, err
}
return &FluentHook{
levels: defaultLevels,
Fluent: fd,
ignoreFields: make(map[string]struct{}),
filters: make(map[string]func(interface{}) interface{}),
}, nil
}
func main() {
var err error
logger, err = fluent.New(fluent.Config{
FluentHost: "localhost",
FluentPort: 24224,
})
if err != nil {
log.Fatal(err)
}
http.HandleFunc("/demo", demoHandler)
http.ListenAndServe(":8080", nil)
}
func SetupFluentLogger(cfg *Config) error {
if cfg.Fluent.FluentHost == "" {
logger.Infof(mylog, "The fluentd host is not specified. Skipping fluent logger instantiation.")
return nil
}
logger.Infof(mylog, "Initializing fluent logger based on config: %+v", cfg.Fluent)
fcli, err := gfluent.New(cfg.Fluent)
if err != nil {
return err
}
logger.Registry().AddOutput(fluent.FluentLogger{fcli})
return nil
}
func print(s severity, args ...interface{}) {
//connect fluent server
f, err := fluent.New(fluent.Config{
FluentPort: 24224,
FluentHost: "localhost",
TagPrefix: "goluent." + getHostname(),
})
message := fmt.Sprint(args...)
stdLog.Println(message)
if err == nil {
f.Post(severityName[s], map[string]string{"message": message})
}
}
func main() {
logger, err := fluent.New(fluent.Config{FluentPort: 24224, FluentHost: "127.0.0.1"})
if err != nil {
fmt.Println(err)
}
defer logger.Close()
tag := "myapp.access"
var data = map[string]string{
"foo": "bar",
"hoge": "hoge"}
i := 0
for i < 100 {
logger.Post(tag, data)
i = i + 1
}
}
func main() {
lambda_proc.Run(func(context *lambda_proc.Context, eventJSON json.RawMessage) (interface{}, error) {
fluentURL, err := getfluentURL(context.FunctionName)
fmt.Fprintf(os.Stderr, "fluentd connection config=%s %d\n", fluentURL.Host, fluentURL.Port)
logger, err := fluent.New(fluent.Config{FluentPort: fluentURL.Port, FluentHost: fluentURL.Host})
if err != nil {
fmt.Fprintf(os.Stderr, "fluentd connection error=%s\n", err)
return nil, err
}
defer logger.Close()
var event cloudwatchlogs.Event
err = json.Unmarshal([]byte(eventJSON), &event)
if err != nil {
fmt.Fprintf(os.Stderr, "json.Unmarshal err=%s\n", err)
return nil, err
}
d, err := event.AWSLogs.DecodedData()
if err != nil {
fmt.Fprintf(os.Stderr, "AWSLogs.DecodedData err=%s\n", err)
return nil, err
}
logs, errs := 0, 0
for _, e := range d.LogEvents {
event, err := decodeLogLine(e.Message)
if err != nil {
fmt.Fprintf(os.Stderr, "Error decoding log line err=%s\n", err)
continue
}
tag := d.LogGroup
err = logger.Post(tag, event)
if err != nil {
fmt.Fprintf(os.Stderr, "FluentD Post: %s\n", err)
return nil, err
}
}
return fmt.Sprintf("LogGroup=%s LogStream=%s MessageType=%s NumLogEvents=%d logs=%d errs=%d", d.LogGroup, d.LogStream, d.MessageType, len(d.LogEvents), logs, errs), nil
})
}
func main() {
defer util.Run()()
var err error
f_config := fluent.Config{FluentSocketPath: *fluent_socket, FluentNetwork: "unix"}
logger, err = fluent.New(f_config)
defer logger.Close()
// log.Printf("starting capture on interface %q", *iface)
// Set up pcap packet capture
handle, err := pcap.OpenLive(*iface, int32(*snaplen), true, pcap.BlockForever)
if err != nil {
panic(err)
}
if err := handle.SetBPFFilter(*filter); err != nil {
panic(err)
}
// Set up assembly
streamFactory := &myFactory{bidiMap: make(map[key]*bidi)}
streamPool := tcpassembly.NewStreamPool(streamFactory)
assembler := tcpassembly.NewAssembler(streamPool)
// log.Println("reading in packets")
// Read in packets, pass to assembler.
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
packets := packetSource.Packets()
ticker := time.Tick(timeout / 4)
for {
select {
case packet := <-packets:
if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP {
//log.Println("Unusable packet")
continue
}
tcp := packet.TransportLayer().(*layers.TCP)
assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp)
case <-ticker:
// Every minute, flush connections that haven't seen activity in the past minute.
// log.Println("---- FLUSHING ----")
assembler.FlushOlderThan(time.Now().Add(-timeout))
streamFactory.collectOldStreams()
}
}
}
func (hook *fluentHook) Fire(entry *logrus.Entry) error {
logger, err := fluent.New(fluent.Config{
FluentHost: hook.host,
FluentPort: hook.port,
})
if err != nil {
return err
}
defer logger.Close()
setLevelString(entry)
tag := getTagAndDel(entry)
if tag != entry.Message {
setMessage(entry)
}
data := ConvertToValue(entry.Data, TagName)
err = logger.PostWithTime(tag, entry.Time, data)
return err
}
// New creates a fluentd logger using the configuration passed in on
// the context. Supported context configuration variables are
// fluentd-address & fluentd-tag.
func New(ctx logger.Context) (logger.Logger, error) {
host, port, tag, err := parseConfig(ctx)
if err != nil {
return nil, err
}
logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s.", ctx.ContainerID, host, port, tag)
// logger tries to recoonect 2**32 - 1 times
// failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds]
log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32})
if err != nil {
return nil, err
}
return &fluentd{
tag: tag,
containerID: ctx.ContainerID,
containerName: ctx.ContainerName,
writer: log,
}, nil
}
func setupFluentd(fluentHost string) func(interface{}) {
parts := strings.Split(fluentHost, ":")
if len(parts) != 2 {
fmt.Fprintf(os.Stderr, "invalid fluentd host format, should be <hostname>:<port>")
os.Exit(1)
}
port, err := strconv.Atoi(parts[1])
if err != nil {
fmt.Fprintf(os.Stderr, "invalid port for fluentd")
os.Exit(1)
}
cfg := fluent.Config{FluentHost: parts[0], FluentPort: port}
logger, err := fluent.New(cfg)
return func(message interface{}) {
spew.Sprintln(message)
logger.Post("bouncer", message)
}
}
// Fire is invoked by logrus and sends log to fluentd logger.
func (hook *FluentHook) Fire(entry *logrus.Entry) error {
var logger *fluent.Fluent
var err error
switch {
case hook.Fluent != nil:
logger = hook.Fluent
default:
logger, err = fluent.New(fluent.Config{
FluentHost: hook.host,
FluentPort: hook.port,
})
if err != nil {
return err
}
defer logger.Close()
}
// Create a map for passing to FluentD
data := make(logrus.Fields)
for k, v := range entry.Data {
if _, ok := hook.ignoreFields[k]; ok {
continue
}
if fn, ok := hook.filters[k]; ok {
v = fn(v)
}
data[k] = v
}
setLevelString(entry, data)
tag := hook.getTagAndDel(entry, data)
if tag != entry.Message {
setMessage(entry, data)
}
fluentData := ConvertToValue(data, TagName)
err = logger.PostWithTime(tag, entry.Time, fluentData)
return err
}
func main() {
logger, _ := fluent.New(fluent.Config{})
tag2 := "myapp.errors"
defer func() {
if e := recover(); e != nil {
stack := make([]byte, 1<<16)
sz := runtime.Stack(stack, true)
stackstr := string(stack[:sz])
var data3 = map[string]string{
"app_id": "command-underline",
"version": "v2.0.0",
"message": "A slightly modified error occurred",
"error_message": "This is my error other messages",
"function": "main",
"exception": stackstr,
"environment": "prod",
}
_ = logger.Post(tag2, data3)
}
}()
panic("something went horribly wrong")
}
func logPageView(u *ShortenedUrl) {
// Setup the logger
logger, err := fluent.New(fluent.Config{FluentPort: config.FluentPort, FluentHost: config.FluentHost})
if err != nil {
Warning.Println("Failed to connect to Fluent")
return
}
defer logger.Close()
// Build the log data
var data = map[string]string{
"a": "3",
"u": u.Url,
"s": u.ShortUrl,
}
// Send to fluent
error := logger.Post(config.FluentTag, data)
if error != nil {
Warning.Printf("Failed to send data to fluentd")
}
}
func (f *FluentDriver) Configure(config map[string]string) (err error) {
fluentConfig := fluent.Config{}
host, ok := config["host"]
if ok {
fluentConfig.FluentHost = host
}
port, ok := config["port"]
if ok {
fluentConfig.FluentPort, err = strconv.Atoi(port)
if err != nil {
return
}
}
timeout, ok := config["timeout"]
if ok {
fluentConfig.Timeout, err = time.ParseDuration(timeout)
if err != nil {
return
}
}
tagPrefix, ok := config["prefix"]
if ok {
fluentConfig.TagPrefix = tagPrefix
}
f.Tag, ok = config["tag"]
if !ok {
return errors.New("tag is required")
}
f.fluent, err = fluent.New(fluentConfig)
return
}
// New creates a fluentd logger using the configuration passed in on
// the context. Supported context configuration variables are
// fluentd-address & fluentd-tag.
func New(ctx logger.Context) (logger.Logger, error) {
host, port, err := parseAddress(ctx.Config["fluentd-address"])
if err != nil {
return nil, err
}
tag, err := loggerutils.ParseLogTag(ctx, "docker.{{.ID}}")
if err != nil {
return nil, err
}
failOnStartupError, err := loggerutils.ParseFailOnStartupErrorFlag(ctx)
if err != nil {
return nil, err
}
bufferLimit, err := parseBufferLimit(ctx.Config["buffer-limit"])
if err != nil {
return nil, err
}
extra := ctx.ExtraAttributes(nil)
logrus.Debugf("logging driver fluentd configured for container:%s, host:%s, port:%d, tag:%s, extra:%v.", ctx.ContainerID, host, port, tag, extra)
// logger tries to reconnect 2**32 - 1 times
// failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds]
log, err := fluent.New(fluent.Config{FluentPort: port, FluentHost: host, RetryWait: 1000, MaxRetry: math.MaxInt32, BufferLimit: bufferLimit})
if err != nil {
if failOnStartupError {
return nil, err
}
logrus.Warnf("fluentd cannot connect to configured endpoint. Ignoring as instructed. Error: %q", err)
}
return &fluentd{
tag: tag,
containerID: ctx.ContainerID,
containerName: ctx.ContainerName,
writer: log,
extra: extra,
}, nil
}
func NewFluentdLogger(tag, host, pid string) LoggerInterface {
logger, _ := fluent.New(fluent.Config{})
return &FluentdLogger{logger: logger, tag: tag, host: host, pid: pid}
}
func main() {
var tag = flag.String("tag", "syslog", "fleuntd tag for logging")
flag.Parse()
// fmt.Println(*tag)
logger, err := fluent.New(fluent.Config{FluentSocketPath: "/tmp/td-agent.sock", FluentNetwork: "unix"})
if err != nil {
fmt.Println(err)
}
var message []string
cmd := exec.Command("./test.sh", "2>&1")
output, err := cmd.StdoutPipe()
if err != nil {
fmt.Println(err)
}
cmd.Start()
bio := bufio.NewReader(output)
// for {
// line, _, err := bio.ReadLine();
// if err != nil {
// break
// }
//
// }
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-time.After(3 * time.Second):
output.Close()
if err := cmd.Process.Kill(); err != nil {
}
<-done // allow goroutine to exit
case err := <-done:
if err != nil {
line, _, err := bio.ReadLine()
if err != nil {
break
}
message = append(message, string(line))
}
}
// fmt.Println(time.Now(), "\n", strings.Join(message, "\n") )
defer logger.Close()
var data = map[string]string{
"message": strings.Join(message, "\n"),
"timestamp": time.Now().String()}
error := logger.Post(*tag, data)
if error != nil {
panic(error)
}
}
// New creates a fluentd logger using the configuration passed in on
// the context. The supported context configuration variable is
// fluentd-address.
func New(ctx logger.Context) (logger.Logger, error) {
loc, err := parseAddress(ctx.Config[addressKey])
if err != nil {
return nil, err
}
tag, err := loggerutils.ParseLogTag(ctx, loggerutils.DefaultTemplate)
if err != nil {
return nil, err
}
extra := ctx.ExtraAttributes(nil)
bufferLimit := defaultBufferLimit
if ctx.Config[bufferLimitKey] != "" {
bl64, err := units.RAMInBytes(ctx.Config[bufferLimitKey])
if err != nil {
return nil, err
}
bufferLimit = int(bl64)
}
retryWait := defaultRetryWait
if ctx.Config[retryWaitKey] != "" {
rwd, err := time.ParseDuration(ctx.Config[retryWaitKey])
if err != nil {
return nil, err
}
retryWait = int(rwd.Seconds() * 1000)
}
maxRetries := defaultMaxRetries
if ctx.Config[maxRetriesKey] != "" {
mr64, err := strconv.ParseUint(ctx.Config[maxRetriesKey], 10, strconv.IntSize)
if err != nil {
return nil, err
}
maxRetries = int(mr64)
}
asyncConnect := false
if ctx.Config[asyncConnectKey] != "" {
if asyncConnect, err = strconv.ParseBool(ctx.Config[asyncConnectKey]); err != nil {
return nil, err
}
}
fluentConfig := fluent.Config{
FluentPort: loc.port,
FluentHost: loc.host,
FluentNetwork: loc.protocol,
FluentSocketPath: loc.path,
BufferLimit: bufferLimit,
RetryWait: retryWait,
MaxRetry: maxRetries,
AsyncConnect: asyncConnect,
}
logrus.WithField("container", ctx.ContainerID).WithField("config", fluentConfig).
Debug("logging driver fluentd configured")
log, err := fluent.New(fluentConfig)
if err != nil {
return nil, err
}
return &fluentd{
tag: tag,
containerID: ctx.ContainerID,
containerName: ctx.ContainerName,
writer: log,
extra: extra,
}, nil
}
