// Handle is the quicklog handle method for processing a log line func (u *Handler) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error { field := "uuid" if u.FieldName != "" { field = u.FieldName } ok := true fieldIface := config["field"] if fieldIface != nil { field, ok = fieldIface.(string) if !ok { log.Log(ctx).Warn("Could not parse UUID config, using field=uuid") field = "uuid" } } log.Log(ctx).Debug("Starting filter handler", "handler", "uuid", "field", field) go func() { for { select { case line := <-prev: line.Data[field] = uuid.NewV4().String() next <- line case <-ctx.Done(): return } } }() return nil }
func syncToEtcd(ctx context.Context, cfg *config.Config) error { log.Log(ctx).Debug("Connecting to etcd") etcdCfg := client.Config{ Endpoints: strings.Split(etcdEndpoints, ","), } c, err := client.New(etcdCfg) if err != nil { return err } root := "/quicklog/" + instanceName kapi := client.NewKeysAPI(c) input := cfg.Input inputConfig, err := json.Marshal(input.Config) if err != nil { log.Log(ctx).Error("Error converting input config to JSON data", "error", err) return err } output := cfg.Output outputConfig, err := json.Marshal(output.Config) if err != nil { log.Log(ctx).Error("Error converting output config to JSON data", "error", err) return err } filters := cfg.Filters kapi.Set(ctx, root+"/input/driver", input.Driver, nil) kapi.Set(ctx, root+"/input/parser", input.Parser, nil) kapi.Set(ctx, root+"/input/config", string(inputConfig), nil) kapi.Set(ctx, root+"/output/driver", output.Driver, nil) kapi.Set(ctx, root+"/output/config", string(outputConfig), nil) // clear all the filters before re-creating them kapi.Delete(ctx, root+"/filters", &client.DeleteOptions{Recursive: true, Dir: true}) // filters must exist even if empty kapi.Set(ctx, root+"/filters", "", &client.SetOptions{Dir: true}) for idx, filter := range filters { filterConfig, err := json.Marshal(filter.Config) if err != nil { log.Log(ctx).Error("Error converting filter config to JSON data", "error", err) return err } kapi.Set(ctx, root+"/filters/"+strconv.Itoa(idx)+"/driver", filter.Driver, nil) kapi.Set(ctx, root+"/filters/"+strconv.Itoa(idx)+"/config", string(filterConfig), nil) } kapi.Set(ctx, root+"/reload", "1", nil) return nil }
func fromConfig(ctx context.Context, cfg *config.Config) *ql.Chain { inputConfigBody, err := json.Marshal(&cfg.Input.Config) if err != nil { log.Log(ctx).Crit("Error converting input configuration to json", "error", err) return nil } inputHandler, err := ql.GetInput(cfg.Input.Driver).Build(bytes.NewReader(inputConfigBody)) if err != nil { log.Log(ctx).Crit("Error creating input handler", "error", err) return nil } chain := ql.Chain{ Input: inputHandler, InputConfig: cfg.Input.Config, Parser: ql.GetParser(cfg.Input.Parser), Output: ql.GetOutput(cfg.Output.Driver), OutputConfig: cfg.Output.Config, } if len(cfg.Filters) >= 1 { chain.Filter = ql.GetFilter(cfg.Filters[0].Driver) chain.FilterConfig = cfg.Filters[0].Config } return &chain }
func (*hostnameHandler) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error { field := "hostname" ok := true fieldIface := config["field"] if fieldIface != nil { field, ok = fieldIface.(string) if !ok { log.Log(ctx).Warn("Could not parse hostname config, using field=hostname") field = "hostname" } } log.Log(ctx).Debug("Starting filter handler", "handler", "hostname", "field", field) hostname, _ := os.Hostname() go func() { for { select { case line := <-prev: line.Data[field] = hostname next <- line case <-ctx.Done(): return } } }() return nil }
func (out *elHTTP) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error { log.Log(ctx).Debug("Starting output handler", "handler", "elasticsearch-http") url, ok := config["url"].(string) if !ok || url == "" { log.Log(ctx).Error("Could not create elasticsearch-http output, no url defined") return fmt.Errorf("Could not create elasticsearch-http output, no url defined") } index, ok := config["index"].(string) if !ok || index == "" { index = "quicklog" } _type, ok := config["type"].(string) if !ok || _type == "" { _type = "entry" } go func() { for { select { case line := <-prev: data := line.Data data["timestamp"] = line.Timestamp //TODO: eventually provide functions that can determine index, _type from line data destURL := fmt.Sprintf("%s/%s/%s", url, index, _type) r, w := io.Pipe() go func() { defer w.Close() err := json.NewEncoder(w).Encode(&data) if err != nil { log.Log(ctx).Error("error converting line to json", "error", err) } }() resp, err := http.Post(destURL, "application/json", r) if err != nil { log.Log(ctx).Error("error sending to elasticsearch", "error", err) continue } defer resp.Body.Close() case <-ctx.Done(): return } } }() return nil }
// Handle is the quicklog handle method func (d *Handler) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error { printFields := true if d.PrintFields.NotNull { printFields = d.PrintFields.Value } pfInput := config["print-fields"] if pfInput != nil { switch k := pfInput.(type) { case bool: printFields = k case string: printFields = k == "true" default: log.Log(ctx).Warn("Could not parse print-fields variable, falling back to true") } } log.Log(ctx).Debug("Starting output handler", "handler", "debug") go func() { for { select { case line := <-prev: os.Stdout.Write([]byte(fmt.Sprintf("Time: [%v]\n", line.Timestamp))) if line.Data["message"] == nil { line.Data["message"] = "" } os.Stdout.Write([]byte("Message: '" + line.Data["message"].(string) + "'\n")) if !printFields { continue } os.Stdout.Write([]byte("Fields:\n")) for key, val := range line.Data { if key != "message" { os.Stdout.Write([]byte(fmt.Sprintf("\t%s: '%s'\n", key, val))) } } os.Stdout.Write([]byte("\n")) case <-ctx.Done(): return } } }() return nil }
// Start starts the standard input process func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error { log.Log(ctx).Debug("Starting input handler", "handler", "stdin") fact.once.Do(func() { go func() { bio := bufio.NewReader(os.Stdin) for { line, _, err := bio.ReadLine() if err != nil { break } fact.ch <- ql.Buffer{Data: line} } }() }) go func() { for { select { case <-ctx.Done(): return case str := <-fact.ch: next <- str } } }() return nil }
// Handle is the quicklog output handler func (p *Process) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error { if p.W == nil { return errors.New("No writer provided to the output handler") } if p.Name == "" { p.Name = "writer" } l := log.Log(ctx).New("handler", p.Name) l.Debug("Starting output handler") go func() { for { select { case line := <-prev: if _, err := p.W.Write([]byte(fmt.Sprintf("%s\n", line.Data["message"]))); err != nil { l.Error("Error writing", "error", err) } case <-ctx.Done(): return } } }() return nil }
func main() { flag.Parse() if verFlag { fmt.Printf("%s\n", version) return } // Setup context ctx := context.Background() ctx = log.NewContext(ctx) log.Log(ctx).Info("Starting quicklog") // Setup system system := managed.NewSystem("quicklog") system.StartWithContext(ctx) // Register signal listeners system.RegisterForStop(os.Interrupt, os.Kill) switch { case etcdEndpoints != "" && instanceName != "": startEtcdQuicklog(ctx, system) case configFile != "": startFileQuicklog(ctx, system) } system.Wait() }
// Execute executes the chain and waits for its completion func (ch *Chain) Execute(ctx context.Context) { outputHandler := ch.Output var chann chan Line bufferChan := make(chan Buffer) inputChan := make(chan Line) if ch.InputConfig == nil { ch.InputConfig = make(map[string]interface{}) } if ch.OutputConfig == nil { ch.OutputConfig = make(map[string]interface{}) } if ch.FilterConfig == nil { ch.FilterConfig = make(map[string]interface{}) } if err := ch.Input.Start(ctx, bufferChan); err != nil { log.Log(ctx).Crit("Error starting input handler", "error", err) } go ch.parserLoop(ctx, bufferChan, inputChan) if ch.Filter != nil { filterHandler := ch.Filter chann = make(chan Line) if err := filterHandler.Handle(ctx, inputChan, chann, ch.FilterConfig); err != nil { log.Log(ctx).Crit("Error creating filter handler", "error", err) return } } else { chann = inputChan } if err := outputHandler.Handle(ctx, chann, ch.OutputConfig); err != nil { log.Log(ctx).Crit("Error creating output handler", "error", err) return } <-ctx.Done() }
func startFileQuicklog(ctx context.Context, system *managed.System) { log.Log(ctx).Info("Loading config from file", "file", configFile) // load config cfg, err := config.LoadFile(configFile) if err != nil { log.Log(ctx).Error("Error loading configuration", "error", err) os.Exit(255) return } // setup chain chain := fromConfig(ctx, cfg) // execute chain system.Add(managed.Simple("chain", chain.Execute)) return }
func (out *natsOut) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error { log.Log(ctx).Debug("Starting output handler", "handler", "nats") url, ok := config["url"].(string) if !ok || url == "" { log.Log(ctx).Error("Could not create nats output, no url defined") return fmt.Errorf("Could not create nats output, no url defined") } opts := nats.DefaultOptions opts.Url = url servers, ok := config["servers"].([]string) if ok { opts.Servers = servers } publish, ok := config["publish"].(string) if !ok || publish == "" { log.Log(ctx).Error("Could not create nats output, no publish defined") return fmt.Errorf("Could not create nats output, no publish defined") } encoding, ok := config["encoding"].(string) if !ok || encoding == "" { encoding = nats.JSON_ENCODER } nc, err := opts.Connect() if err != nil { log.Log(ctx).Error("Error connecting to nats url", "url", url, "error", err) return err } c, err := nats.NewEncodedConn(nc, encoding) if err != nil { log.Log(ctx).Error("Error creating nats connection", "error", err) return err } go func() { defer c.Close() for { select { case line := <-prev: err = c.Publish(publish, line) if err != nil { log.Log(ctx).Error("Error publishing to nats connection", "error", err) } case <-ctx.Done(): return } } }() return nil }
func (u *uppercase) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error { log.Log(ctx).Debug("Starting filter handler", "handler", "uppercase") go func() { for { select { case line := <-prev: line.Data["message"] = strings.ToUpper(line.Data["message"].(string)) next <- line case <-ctx.Done(): return } } }() return nil }
func (ch *Chain) parserLoop(ctx context.Context, bufferChan <-chan Buffer, inputChan chan<- Line) { parser := ch.Parser if parser == nil { parser = &plainParser{} } for { select { case <-ctx.Done(): return case buffer := <-bufferChan: l := Line{ Data: make(map[string]interface{}), Timestamp: time.Now(), } if len(buffer.Data) == 0 { continue // skip line } if err := parser.Parse(buffer.Data, &l, ch.InputConfig); err != nil { log.Log(ctx).Error("Error parsing incoming data", "error", err) continue } if buffer.Metadata != nil { for k, v := range buffer.Metadata { l.Data[k] = v } } inputChan <- l } } }
func (*rename) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error { log.Log(ctx).Debug("Starting filter handler", "handler", "rename_field") src, exists := config["source"].(string) if !exists { return fmt.Errorf("Error getting source config in rename_field handler") } dst, exists := config["dest"].(string) if !exists { return fmt.Errorf("Error getting dest config in rename_field handler") } cp, exists := config["copy"].(bool) if !exists { cp = false } go func() { for { select { case line := <-prev: line.Data[dst] = line.Data[src] if !cp { // empty out source field line.Data[src] = nil } next <- line case <-ctx.Done(): return } } }() return nil }
// Start reads from the nats queue and pushes onto the given next channel func (i *Process) Start(ctx context.Context, next chan<- ql.Buffer) error { log.Log(ctx).Debug("Starting input handler", "handler", "nats") if i.Config.URL == "" { log.Log(ctx).Error("Could not create nats input, no url defined") return fmt.Errorf("Could not create nats input, no url defined") } opts := nats.DefaultOptions opts.Url = i.Config.URL opts.Servers = i.Config.Servers if i.Config.Publish == "" { log.Log(ctx).Error("Could not create nats input, no publish defined") return fmt.Errorf("Could not create nats input, no publish defined") } if i.Config.Encoding == "" { i.Config.Encoding = nats.JSON_ENCODER } nc, err := opts.Connect() if err != nil { log.Log(ctx).Error("Error connecting to nats url", "url", i.Config.URL, "error", err) return err } c, err := nats.NewEncodedConn(nc, i.Config.Encoding) if err != nil { log.Log(ctx).Error("Error creating nats connection", "error", err) return err } recvCh := make(chan ql.Line) sub, err := c.BindRecvChan(i.Config.Publish, recvCh) if err != nil { log.Log(ctx).Error("Error listening on nats receive channel", "error", err) return err } go func() { defer sub.Unsubscribe() defer c.Close() for { select { case line := <-recvCh: var msg string if m, ok := line.Data["message"]; ok { msg = m.(string) } delete(line.Data, "message") next <- ql.Buffer{ Data: []byte(msg), Metadata: line.Data, } case <-ctx.Done(): return } } }() return nil }
func syncFromEtcd(ctx context.Context, root string, cl client.KeysAPI, cfg *config.Config) error { inputDriverResponse, err := cl.Get(ctx, root+"/input/driver", nil) if err != nil { return err } inputParserResponse, err := cl.Get(ctx, root+"/input/parser", nil) if err != nil { return err } inputDriverCfg, err := cl.Get(ctx, root+"/input/config", nil) if err != nil { return err } outputDriverResponse, err := cl.Get(ctx, root+"/output/driver", nil) if err != nil { return err } outputDriverCfg, err := cl.Get(ctx, root+"/output/config", nil) if err != nil { return err } var input config.Input var output config.Output input.Driver = inputDriverResponse.Node.Value input.Parser = inputParserResponse.Node.Value output.Driver = outputDriverResponse.Node.Value err = json.Unmarshal([]byte(inputDriverCfg.Node.Value), &input.Config) if err != nil { return err } err = json.Unmarshal([]byte(outputDriverCfg.Node.Value), &output.Config) if err != nil { return err } filtersResponse, err := cl.Get(ctx, root+"/filters", &client.GetOptions{Recursive: true}) if err != nil { return err } for idx, node := range filtersResponse.Node.Nodes { var filter config.Filter for _, n := range node.Nodes { if n.Key == root+"/filters/"+strconv.Itoa(idx)+"/driver" { filter.Driver = n.Value } else if n.Key == root+"/filters/"+strconv.Itoa(idx)+"/config" { err = json.Unmarshal([]byte(n.Value), &filter.Config) if err != nil { return err } } else { log.Log(ctx).Warn("Unexpected node", "node", n) } } cfg.Filters = append(cfg.Filters, filter) } cfg.Input = input cfg.Output = output return nil }
func startEtcdQuicklog(mainCtx context.Context, system *managed.System) { log.Log(mainCtx).Info("Loading config from etcd") log.Log(mainCtx).Debug("Connecting to endpoint", "endpoints", strings.Split(etcdEndpoints, ",")) etcdCfg := client.Config{ Endpoints: strings.Split(etcdEndpoints, ","), } c, err := client.New(etcdCfg) if err != nil { log.Log(mainCtx).Crit("Error connecting to etcd server", "error", err) return } root := "/quicklog/" + instanceName log.Log(mainCtx).Debug("Listening for etcd keys", "key", root) kapi := client.NewKeysAPI(c) chainApp := managed.NewSystem("app-chain-" + instanceName) system.SpawnSystem(chainApp) var cfg config.Config err = syncFromEtcd(mainCtx, root, kapi, &cfg) if err != nil { log.Log(mainCtx).Error("Error syncing from etcd", "error", err) } // setup chain chain := fromConfig(mainCtx, &cfg) chainApp.Add(managed.Simple("chain-sub-"+instanceName, chain.Execute)) system.Add(managed.Simple("etcd", func(ctx context.Context) { w := kapi.Watcher(root+"/reload", &client.WatcherOptions{ Recursive: false, }) for { resp, err := w.Next(ctx) if err != nil { if err == context.DeadlineExceeded { continue } else if err == context.Canceled { return } else if cerr, ok := err.(*client.ClusterError); ok { for _, e := range cerr.Errors { if e != context.Canceled { log.Log(ctx).Error("Error getting next etcd watch event", "parentError", err, "error", e) } } } else { log.Log(ctx).Error("Error getting next etcd watch event", "error", err) } return } if resp == nil { return } switch resp.Action { case "get": // do nothing default: log.Log(ctx).Info("Got update on quicklog config", "etcd.action", resp.Action) var newCfg config.Config err = syncFromEtcd(ctx, root, kapi, &newCfg) if err != nil { log.Log(ctx).Error("Error syncing from etcd", "error", err) } else { chainApp.Stop() <-time.After(1 * time.Second) chainApp = managed.NewSystem("app-chain-" + instanceName) system.SpawnSystem(chainApp) // setup chain chain = fromConfig(ctx, &newCfg) chainApp.Add(managed.Simple("chain-sub-"+instanceName, chain.Execute)) } //TODO: (re)load config } } })) }
// Start starts the syslog handler proces func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error { if p.Config.Listen == "" { return errors.New("No syslog listen configuration provided") } log.Log(ctx).Debug("Starting input handler", "handler", "syslog", "listen", p.Config.Listen) ch := make(chan ql.Buffer) listenAddr, err := net.ResolveUDPAddr("udp", p.Config.Listen) if err != nil { return err } ln, err := net.ListenUDP("udp", listenAddr) if err != nil { return err } go func() { buffer := make([]byte, 1024) for { size, addr, err := ln.ReadFromUDP(buffer) if err != nil { // if the context is done, assume the error // is that the listener has been closed. select { case <-ctx.Done(): return default: } log.Log(ctx).Error("Error reading from connection", "error", err) continue } priStart := 0 priEnd := 0 for _, i := range buffer { priEnd++ if i == '>' { break } } hostStart := priEnd + len(timestamp) + 1 hostEnd := priEnd + len(timestamp) + 1 for _, i := range buffer[hostStart:] { if i == ' ' { break } hostEnd++ } tagStart := hostEnd + 1 tagEnd := hostEnd + 1 for _, i := range buffer[tagStart:] { if i == ':' || i == '[' || i == ' ' { break } tagEnd++ } m := make(map[string]interface{}) m["syslog.tag"] = string(buffer[tagStart:tagEnd]) m["syslog.hostname"] = string(buffer[hostStart:hostEnd]) m["syslog.timestamp"] = string(buffer[priEnd : priEnd+len(timestamp)]) m["syslog.pri"] = string(buffer[priStart+1 : priEnd-1]) m["udp.source"] = addr.String() data := buffer[tagEnd+2 : size] ch <- ql.Buffer{ Data: data, Metadata: m, } } }() go func() { defer ln.Close() for { select { case <-ctx.Done(): return case buffer := <-ch: next <- buffer } } }() return nil }
// Start starts the UDP handler proces func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error { if p.Config.Listen == "" { return errors.New("No UDP listen configuration provided") } log.Log(ctx).Debug("Starting input handler", "handler", "udp", "listen", p.Config.Listen) ch := make(chan ql.Buffer) listenAddr, err := net.ResolveUDPAddr("udp", p.Config.Listen) if err != nil { return err } ln, err := net.ListenUDP("udp", listenAddr) if err != nil { return err } go func() { buffer := make([]byte, 1024) for { size, addr, err := ln.ReadFromUDP(buffer) if err != nil { // if the context is done, assume the error // is that the listener has been closed. select { case <-ctx.Done(): return default: } log.Log(ctx).Error("Error reading from connection", "error", err) continue } m := make(map[string]interface{}) m["udp.source"] = addr.String() ch <- ql.Buffer{ Data: buffer[:size], Metadata: m, } } }() go func() { defer ln.Close() for { select { case <-ctx.Done(): return case buffer := <-ch: next <- buffer } } }() return nil }
// Start starts the tcp listener and waits for messages func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error { if p.Config.Listen == "" { return errors.New("No TCP listen option provided") } log.Log(ctx).Debug("Starting input handler", "handler", "tcp", "listen", p.Config.Listen) ch := make(chan ql.Buffer) ln, err := net.Listen("tcp", p.Config.Listen) if err != nil { return err } go func() { for { conn, err := ln.Accept() if err != nil { // if the context is done, assume the error // is that the listener has been closed. select { case <-ctx.Done(): return default: } log.Log(ctx).Error("Error accepting connection", "error", err) continue } go func(conn net.Conn) { bio := bufio.NewReader(conn) for { line, _, err := bio.ReadLine() if err != nil { break } m := make(map[string]interface{}) m["tcp.source"] = conn.RemoteAddr().String() ch <- ql.Buffer{ Data: line, Metadata: m, } } }(conn) } }() go func() { defer ln.Close() for { select { case <-ctx.Done(): return case buffer := <-ch: next <- buffer } } }() return nil }