// Handle is the quicklog handle method for processing a log line
func (u *Handler) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error {
field := "uuid"
if u.FieldName != "" {
field = u.FieldName
}
ok := true
fieldIface := config["field"]
if fieldIface != nil {
field, ok = fieldIface.(string)
if !ok {
log.Log(ctx).Warn("Could not parse UUID config, using field=uuid")
field = "uuid"
}
}
log.Log(ctx).Debug("Starting filter handler", "handler", "uuid", "field", field)
go func() {
for {
select {
case line := <-prev:
line.Data[field] = uuid.NewV4().String()
next <- line
case <-ctx.Done():
return
}
}
}()
return nil
}
func syncToEtcd(ctx context.Context, cfg *config.Config) error {
log.Log(ctx).Debug("Connecting to etcd")
etcdCfg := client.Config{
Endpoints: strings.Split(etcdEndpoints, ","),
}
c, err := client.New(etcdCfg)
if err != nil {
return err
}
root := "/quicklog/" + instanceName
kapi := client.NewKeysAPI(c)
input := cfg.Input
inputConfig, err := json.Marshal(input.Config)
if err != nil {
log.Log(ctx).Error("Error converting input config to JSON data", "error", err)
return err
}
output := cfg.Output
outputConfig, err := json.Marshal(output.Config)
if err != nil {
log.Log(ctx).Error("Error converting output config to JSON data", "error", err)
return err
}
filters := cfg.Filters
kapi.Set(ctx, root+"/input/driver", input.Driver, nil)
kapi.Set(ctx, root+"/input/parser", input.Parser, nil)
kapi.Set(ctx, root+"/input/config", string(inputConfig), nil)
kapi.Set(ctx, root+"/output/driver", output.Driver, nil)
kapi.Set(ctx, root+"/output/config", string(outputConfig), nil)
// clear all the filters before re-creating them
kapi.Delete(ctx, root+"/filters", &client.DeleteOptions{Recursive: true, Dir: true})
// filters must exist even if empty
kapi.Set(ctx, root+"/filters", "", &client.SetOptions{Dir: true})
for idx, filter := range filters {
filterConfig, err := json.Marshal(filter.Config)
if err != nil {
log.Log(ctx).Error("Error converting filter config to JSON data", "error", err)
return err
}
kapi.Set(ctx, root+"/filters/"+strconv.Itoa(idx)+"/driver", filter.Driver, nil)
kapi.Set(ctx, root+"/filters/"+strconv.Itoa(idx)+"/config", string(filterConfig), nil)
}
kapi.Set(ctx, root+"/reload", "1", nil)
return nil
}
func fromConfig(ctx context.Context, cfg *config.Config) *ql.Chain {
inputConfigBody, err := json.Marshal(&cfg.Input.Config)
if err != nil {
log.Log(ctx).Crit("Error converting input configuration to json", "error", err)
return nil
}
inputHandler, err := ql.GetInput(cfg.Input.Driver).Build(bytes.NewReader(inputConfigBody))
if err != nil {
log.Log(ctx).Crit("Error creating input handler", "error", err)
return nil
}
chain := ql.Chain{
Input: inputHandler,
InputConfig: cfg.Input.Config,
Parser: ql.GetParser(cfg.Input.Parser),
Output: ql.GetOutput(cfg.Output.Driver),
OutputConfig: cfg.Output.Config,
}
if len(cfg.Filters) >= 1 {
chain.Filter = ql.GetFilter(cfg.Filters[0].Driver)
chain.FilterConfig = cfg.Filters[0].Config
}
return &chain
}
func (*hostnameHandler) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error {
field := "hostname"
ok := true
fieldIface := config["field"]
if fieldIface != nil {
field, ok = fieldIface.(string)
if !ok {
log.Log(ctx).Warn("Could not parse hostname config, using field=hostname")
field = "hostname"
}
}
log.Log(ctx).Debug("Starting filter handler", "handler", "hostname", "field", field)
hostname, _ := os.Hostname()
go func() {
for {
select {
case line := <-prev:
line.Data[field] = hostname
next <- line
case <-ctx.Done():
return
}
}
}()
return nil
}
func (out *elHTTP) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error {
log.Log(ctx).Debug("Starting output handler", "handler", "elasticsearch-http")
url, ok := config["url"].(string)
if !ok || url == "" {
log.Log(ctx).Error("Could not create elasticsearch-http output, no url defined")
return fmt.Errorf("Could not create elasticsearch-http output, no url defined")
}
index, ok := config["index"].(string)
if !ok || index == "" {
index = "quicklog"
}
_type, ok := config["type"].(string)
if !ok || _type == "" {
_type = "entry"
}
go func() {
for {
select {
case line := <-prev:
data := line.Data
data["timestamp"] = line.Timestamp
//TODO: eventually provide functions that can determine index, _type from line data
destURL := fmt.Sprintf("%s/%s/%s", url, index, _type)
r, w := io.Pipe()
go func() {
defer w.Close()
err := json.NewEncoder(w).Encode(&data)
if err != nil {
log.Log(ctx).Error("error converting line to json", "error", err)
}
}()
resp, err := http.Post(destURL, "application/json", r)
if err != nil {
log.Log(ctx).Error("error sending to elasticsearch", "error", err)
continue
}
defer resp.Body.Close()
case <-ctx.Done():
return
}
}
}()
return nil
}
// Handle is the quicklog handle method
func (d *Handler) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error {
printFields := true
if d.PrintFields.NotNull {
printFields = d.PrintFields.Value
}
pfInput := config["print-fields"]
if pfInput != nil {
switch k := pfInput.(type) {
case bool:
printFields = k
case string:
printFields = k == "true"
default:
log.Log(ctx).Warn("Could not parse print-fields variable, falling back to true")
}
}
log.Log(ctx).Debug("Starting output handler", "handler", "debug")
go func() {
for {
select {
case line := <-prev:
os.Stdout.Write([]byte(fmt.Sprintf("Time: [%v]\n", line.Timestamp)))
if line.Data["message"] == nil {
line.Data["message"] = ""
}
os.Stdout.Write([]byte("Message: '" + line.Data["message"].(string) + "'\n"))
if !printFields {
continue
}
os.Stdout.Write([]byte("Fields:\n"))
for key, val := range line.Data {
if key != "message" {
os.Stdout.Write([]byte(fmt.Sprintf("\t%s: '%s'\n", key, val)))
}
}
os.Stdout.Write([]byte("\n"))
case <-ctx.Done():
return
}
}
}()
return nil
}
// Start starts the standard input process
func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error {
log.Log(ctx).Debug("Starting input handler", "handler", "stdin")
fact.once.Do(func() {
go func() {
bio := bufio.NewReader(os.Stdin)
for {
line, _, err := bio.ReadLine()
if err != nil {
break
}
fact.ch <- ql.Buffer{Data: line}
}
}()
})
go func() {
for {
select {
case <-ctx.Done():
return
case str := <-fact.ch:
next <- str
}
}
}()
return nil
}
// Handle is the quicklog output handler
func (p *Process) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error {
if p.W == nil {
return errors.New("No writer provided to the output handler")
}
if p.Name == "" {
p.Name = "writer"
}
l := log.Log(ctx).New("handler", p.Name)
l.Debug("Starting output handler")
go func() {
for {
select {
case line := <-prev:
if _, err := p.W.Write([]byte(fmt.Sprintf("%s\n", line.Data["message"]))); err != nil {
l.Error("Error writing", "error", err)
}
case <-ctx.Done():
return
}
}
}()
return nil
}
func main() {
flag.Parse()
if verFlag {
fmt.Printf("%s\n", version)
return
}
// Setup context
ctx := context.Background()
ctx = log.NewContext(ctx)
log.Log(ctx).Info("Starting quicklog")
// Setup system
system := managed.NewSystem("quicklog")
system.StartWithContext(ctx)
// Register signal listeners
system.RegisterForStop(os.Interrupt, os.Kill)
switch {
case etcdEndpoints != "" && instanceName != "":
startEtcdQuicklog(ctx, system)
case configFile != "":
startFileQuicklog(ctx, system)
}
system.Wait()
}
// Execute executes the chain and waits for its completion
func (ch *Chain) Execute(ctx context.Context) {
outputHandler := ch.Output
var chann chan Line
bufferChan := make(chan Buffer)
inputChan := make(chan Line)
if ch.InputConfig == nil {
ch.InputConfig = make(map[string]interface{})
}
if ch.OutputConfig == nil {
ch.OutputConfig = make(map[string]interface{})
}
if ch.FilterConfig == nil {
ch.FilterConfig = make(map[string]interface{})
}
if err := ch.Input.Start(ctx, bufferChan); err != nil {
log.Log(ctx).Crit("Error starting input handler", "error", err)
}
go ch.parserLoop(ctx, bufferChan, inputChan)
if ch.Filter != nil {
filterHandler := ch.Filter
chann = make(chan Line)
if err := filterHandler.Handle(ctx, inputChan, chann, ch.FilterConfig); err != nil {
log.Log(ctx).Crit("Error creating filter handler", "error", err)
return
}
} else {
chann = inputChan
}
if err := outputHandler.Handle(ctx, chann, ch.OutputConfig); err != nil {
log.Log(ctx).Crit("Error creating output handler", "error", err)
return
}
<-ctx.Done()
}
func startFileQuicklog(ctx context.Context, system *managed.System) {
log.Log(ctx).Info("Loading config from file", "file", configFile)
// load config
cfg, err := config.LoadFile(configFile)
if err != nil {
log.Log(ctx).Error("Error loading configuration", "error", err)
os.Exit(255)
return
}
// setup chain
chain := fromConfig(ctx, cfg)
// execute chain
system.Add(managed.Simple("chain", chain.Execute))
return
}
func (out *natsOut) Handle(ctx context.Context, prev <-chan ql.Line, config map[string]interface{}) error {
log.Log(ctx).Debug("Starting output handler", "handler", "nats")
url, ok := config["url"].(string)
if !ok || url == "" {
log.Log(ctx).Error("Could not create nats output, no url defined")
return fmt.Errorf("Could not create nats output, no url defined")
}
opts := nats.DefaultOptions
opts.Url = url
servers, ok := config["servers"].([]string)
if ok {
opts.Servers = servers
}
publish, ok := config["publish"].(string)
if !ok || publish == "" {
log.Log(ctx).Error("Could not create nats output, no publish defined")
return fmt.Errorf("Could not create nats output, no publish defined")
}
encoding, ok := config["encoding"].(string)
if !ok || encoding == "" {
encoding = nats.JSON_ENCODER
}
nc, err := opts.Connect()
if err != nil {
log.Log(ctx).Error("Error connecting to nats url", "url", url, "error", err)
return err
}
c, err := nats.NewEncodedConn(nc, encoding)
if err != nil {
log.Log(ctx).Error("Error creating nats connection", "error", err)
return err
}
go func() {
defer c.Close()
for {
select {
case line := <-prev:
err = c.Publish(publish, line)
if err != nil {
log.Log(ctx).Error("Error publishing to nats connection", "error", err)
}
case <-ctx.Done():
return
}
}
}()
return nil
}
func (u *uppercase) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error {
log.Log(ctx).Debug("Starting filter handler", "handler", "uppercase")
go func() {
for {
select {
case line := <-prev:
line.Data["message"] = strings.ToUpper(line.Data["message"].(string))
next <- line
case <-ctx.Done():
return
}
}
}()
return nil
}
func (ch *Chain) parserLoop(ctx context.Context, bufferChan <-chan Buffer, inputChan chan<- Line) {
parser := ch.Parser
if parser == nil {
parser = &plainParser{}
}
for {
select {
case <-ctx.Done():
return
case buffer := <-bufferChan:
l := Line{
Data: make(map[string]interface{}),
Timestamp: time.Now(),
}
if len(buffer.Data) == 0 {
continue // skip line
}
if err := parser.Parse(buffer.Data, &l, ch.InputConfig); err != nil {
log.Log(ctx).Error("Error parsing incoming data", "error", err)
continue
}
if buffer.Metadata != nil {
for k, v := range buffer.Metadata {
l.Data[k] = v
}
}
inputChan <- l
}
}
}
func (*rename) Handle(ctx context.Context, prev <-chan ql.Line, next chan<- ql.Line, config map[string]interface{}) error {
log.Log(ctx).Debug("Starting filter handler", "handler", "rename_field")
src, exists := config["source"].(string)
if !exists {
return fmt.Errorf("Error getting source config in rename_field handler")
}
dst, exists := config["dest"].(string)
if !exists {
return fmt.Errorf("Error getting dest config in rename_field handler")
}
cp, exists := config["copy"].(bool)
if !exists {
cp = false
}
go func() {
for {
select {
case line := <-prev:
line.Data[dst] = line.Data[src]
if !cp { // empty out source field
line.Data[src] = nil
}
next <- line
case <-ctx.Done():
return
}
}
}()
return nil
}
// Start reads from the nats queue and pushes onto the given next channel
func (i *Process) Start(ctx context.Context, next chan<- ql.Buffer) error {
log.Log(ctx).Debug("Starting input handler", "handler", "nats")
if i.Config.URL == "" {
log.Log(ctx).Error("Could not create nats input, no url defined")
return fmt.Errorf("Could not create nats input, no url defined")
}
opts := nats.DefaultOptions
opts.Url = i.Config.URL
opts.Servers = i.Config.Servers
if i.Config.Publish == "" {
log.Log(ctx).Error("Could not create nats input, no publish defined")
return fmt.Errorf("Could not create nats input, no publish defined")
}
if i.Config.Encoding == "" {
i.Config.Encoding = nats.JSON_ENCODER
}
nc, err := opts.Connect()
if err != nil {
log.Log(ctx).Error("Error connecting to nats url", "url", i.Config.URL, "error", err)
return err
}
c, err := nats.NewEncodedConn(nc, i.Config.Encoding)
if err != nil {
log.Log(ctx).Error("Error creating nats connection", "error", err)
return err
}
recvCh := make(chan ql.Line)
sub, err := c.BindRecvChan(i.Config.Publish, recvCh)
if err != nil {
log.Log(ctx).Error("Error listening on nats receive channel", "error", err)
return err
}
go func() {
defer sub.Unsubscribe()
defer c.Close()
for {
select {
case line := <-recvCh:
var msg string
if m, ok := line.Data["message"]; ok {
msg = m.(string)
}
delete(line.Data, "message")
next <- ql.Buffer{
Data: []byte(msg),
Metadata: line.Data,
}
case <-ctx.Done():
return
}
}
}()
return nil
}
func syncFromEtcd(ctx context.Context, root string, cl client.KeysAPI, cfg *config.Config) error {
inputDriverResponse, err := cl.Get(ctx, root+"/input/driver", nil)
if err != nil {
return err
}
inputParserResponse, err := cl.Get(ctx, root+"/input/parser", nil)
if err != nil {
return err
}
inputDriverCfg, err := cl.Get(ctx, root+"/input/config", nil)
if err != nil {
return err
}
outputDriverResponse, err := cl.Get(ctx, root+"/output/driver", nil)
if err != nil {
return err
}
outputDriverCfg, err := cl.Get(ctx, root+"/output/config", nil)
if err != nil {
return err
}
var input config.Input
var output config.Output
input.Driver = inputDriverResponse.Node.Value
input.Parser = inputParserResponse.Node.Value
output.Driver = outputDriverResponse.Node.Value
err = json.Unmarshal([]byte(inputDriverCfg.Node.Value), &input.Config)
if err != nil {
return err
}
err = json.Unmarshal([]byte(outputDriverCfg.Node.Value), &output.Config)
if err != nil {
return err
}
filtersResponse, err := cl.Get(ctx, root+"/filters", &client.GetOptions{Recursive: true})
if err != nil {
return err
}
for idx, node := range filtersResponse.Node.Nodes {
var filter config.Filter
for _, n := range node.Nodes {
if n.Key == root+"/filters/"+strconv.Itoa(idx)+"/driver" {
filter.Driver = n.Value
} else if n.Key == root+"/filters/"+strconv.Itoa(idx)+"/config" {
err = json.Unmarshal([]byte(n.Value), &filter.Config)
if err != nil {
return err
}
} else {
log.Log(ctx).Warn("Unexpected node", "node", n)
}
}
cfg.Filters = append(cfg.Filters, filter)
}
cfg.Input = input
cfg.Output = output
return nil
}
func startEtcdQuicklog(mainCtx context.Context, system *managed.System) {
log.Log(mainCtx).Info("Loading config from etcd")
log.Log(mainCtx).Debug("Connecting to endpoint", "endpoints", strings.Split(etcdEndpoints, ","))
etcdCfg := client.Config{
Endpoints: strings.Split(etcdEndpoints, ","),
}
c, err := client.New(etcdCfg)
if err != nil {
log.Log(mainCtx).Crit("Error connecting to etcd server", "error", err)
return
}
root := "/quicklog/" + instanceName
log.Log(mainCtx).Debug("Listening for etcd keys", "key", root)
kapi := client.NewKeysAPI(c)
chainApp := managed.NewSystem("app-chain-" + instanceName)
system.SpawnSystem(chainApp)
var cfg config.Config
err = syncFromEtcd(mainCtx, root, kapi, &cfg)
if err != nil {
log.Log(mainCtx).Error("Error syncing from etcd", "error", err)
}
// setup chain
chain := fromConfig(mainCtx, &cfg)
chainApp.Add(managed.Simple("chain-sub-"+instanceName, chain.Execute))
system.Add(managed.Simple("etcd", func(ctx context.Context) {
w := kapi.Watcher(root+"/reload", &client.WatcherOptions{
Recursive: false,
})
for {
resp, err := w.Next(ctx)
if err != nil {
if err == context.DeadlineExceeded {
continue
} else if err == context.Canceled {
return
} else if cerr, ok := err.(*client.ClusterError); ok {
for _, e := range cerr.Errors {
if e != context.Canceled {
log.Log(ctx).Error("Error getting next etcd watch event", "parentError", err, "error", e)
}
}
} else {
log.Log(ctx).Error("Error getting next etcd watch event", "error", err)
}
return
}
if resp == nil {
return
}
switch resp.Action {
case "get":
// do nothing
default:
log.Log(ctx).Info("Got update on quicklog config", "etcd.action", resp.Action)
var newCfg config.Config
err = syncFromEtcd(ctx, root, kapi, &newCfg)
if err != nil {
log.Log(ctx).Error("Error syncing from etcd", "error", err)
} else {
chainApp.Stop()
<-time.After(1 * time.Second)
chainApp = managed.NewSystem("app-chain-" + instanceName)
system.SpawnSystem(chainApp)
// setup chain
chain = fromConfig(ctx, &newCfg)
chainApp.Add(managed.Simple("chain-sub-"+instanceName, chain.Execute))
}
//TODO: (re)load config
}
}
}))
}
// Start starts the syslog handler proces
func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error {
if p.Config.Listen == "" {
return errors.New("No syslog listen configuration provided")
}
log.Log(ctx).Debug("Starting input handler", "handler", "syslog", "listen", p.Config.Listen)
ch := make(chan ql.Buffer)
listenAddr, err := net.ResolveUDPAddr("udp", p.Config.Listen)
if err != nil {
return err
}
ln, err := net.ListenUDP("udp", listenAddr)
if err != nil {
return err
}
go func() {
buffer := make([]byte, 1024)
for {
size, addr, err := ln.ReadFromUDP(buffer)
if err != nil {
// if the context is done, assume the error
// is that the listener has been closed.
select {
case <-ctx.Done():
return
default:
}
log.Log(ctx).Error("Error reading from connection", "error", err)
continue
}
priStart := 0
priEnd := 0
for _, i := range buffer {
priEnd++
if i == '>' {
break
}
}
hostStart := priEnd + len(timestamp) + 1
hostEnd := priEnd + len(timestamp) + 1
for _, i := range buffer[hostStart:] {
if i == ' ' {
break
}
hostEnd++
}
tagStart := hostEnd + 1
tagEnd := hostEnd + 1
for _, i := range buffer[tagStart:] {
if i == ':' || i == '[' || i == ' ' {
break
}
tagEnd++
}
m := make(map[string]interface{})
m["syslog.tag"] = string(buffer[tagStart:tagEnd])
m["syslog.hostname"] = string(buffer[hostStart:hostEnd])
m["syslog.timestamp"] = string(buffer[priEnd : priEnd+len(timestamp)])
m["syslog.pri"] = string(buffer[priStart+1 : priEnd-1])
m["udp.source"] = addr.String()
data := buffer[tagEnd+2 : size]
ch <- ql.Buffer{
Data: data,
Metadata: m,
}
}
}()
go func() {
defer ln.Close()
for {
select {
case <-ctx.Done():
return
case buffer := <-ch:
next <- buffer
}
}
}()
return nil
}
// Start starts the UDP handler proces
func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error {
if p.Config.Listen == "" {
return errors.New("No UDP listen configuration provided")
}
log.Log(ctx).Debug("Starting input handler", "handler", "udp", "listen", p.Config.Listen)
ch := make(chan ql.Buffer)
listenAddr, err := net.ResolveUDPAddr("udp", p.Config.Listen)
if err != nil {
return err
}
ln, err := net.ListenUDP("udp", listenAddr)
if err != nil {
return err
}
go func() {
buffer := make([]byte, 1024)
for {
size, addr, err := ln.ReadFromUDP(buffer)
if err != nil {
// if the context is done, assume the error
// is that the listener has been closed.
select {
case <-ctx.Done():
return
default:
}
log.Log(ctx).Error("Error reading from connection", "error", err)
continue
}
m := make(map[string]interface{})
m["udp.source"] = addr.String()
ch <- ql.Buffer{
Data: buffer[:size],
Metadata: m,
}
}
}()
go func() {
defer ln.Close()
for {
select {
case <-ctx.Done():
return
case buffer := <-ch:
next <- buffer
}
}
}()
return nil
}
// Start starts the tcp listener and waits for messages
func (p *Process) Start(ctx context.Context, next chan<- ql.Buffer) error {
if p.Config.Listen == "" {
return errors.New("No TCP listen option provided")
}
log.Log(ctx).Debug("Starting input handler", "handler", "tcp", "listen", p.Config.Listen)
ch := make(chan ql.Buffer)
ln, err := net.Listen("tcp", p.Config.Listen)
if err != nil {
return err
}
go func() {
for {
conn, err := ln.Accept()
if err != nil {
// if the context is done, assume the error
// is that the listener has been closed.
select {
case <-ctx.Done():
return
default:
}
log.Log(ctx).Error("Error accepting connection", "error", err)
continue
}
go func(conn net.Conn) {
bio := bufio.NewReader(conn)
for {
line, _, err := bio.ReadLine()
if err != nil {
break
}
m := make(map[string]interface{})
m["tcp.source"] = conn.RemoteAddr().String()
ch <- ql.Buffer{
Data: line,
Metadata: m,
}
}
}(conn)
}
}()
go func() {
defer ln.Close()
for {
select {
case <-ctx.Done():
return
case buffer := <-ch:
next <- buffer
}
}
}()
return nil
}