func readLog() {
var seek = tail.SeekInfo{Offset: 0, Whence: 2}
tailer, err := tail.TailFile(os.Args[1], tail.Config{
Follow: true,
ReOpen: true,
Location: &seek,
})
if err != nil {
log.Panicln(err)
}
re := regexp.MustCompile(`^(?P<ip>[\d\.]+) - - \[(?P<timestamp>.*)\] "(?P<verb>.*) (?P<query>.*) (?P<proto>.*)" (?P<status>\d+) (?P<bytes>\d+) "(?P<referer>.*)" "(?P<useragent>.*)"`)
for line := range tailer.Lines {
res := re.FindStringSubmatch(line.Text)
ip := res[1]
curtime := time.Now().Local()
verb := res[3]
query := res[4]
proto := res[5]
status, _ := strconv.Atoi(res[6])
bytes, _ := strconv.ParseInt(res[7], 10, 64)
referer := res[8]
useragent := res[9]
logline := RawLogEvent{ip, curtime, verb, query, proto, status, bytes, referer, useragent}
rawlog_output <- logline
logdump_output <- res[0] // Spraynard Kruger
}
}
func main() {
pub, err := nsq.NewProducer(*tHost, nsq.NewConfig())
if err != nil {
log.Fatal(err)
}
t, err := tail.TailFile(*tFile, tail.Config{Follow: true,
//Location: &tail.SeekInfo{Offset: 0, Whence: os.SEEK_END}})
Location: &tail.SeekInfo{Offset: 0, Whence: os.SEEK_CUR}})
if err != nil {
log.Fatal(err)
}
for line := range t.Lines {
if *tTag != "" {
for k, tag := range strings.Split(*tTag, "|") {
var ks = []string{"pv", "click", "other"}
if strings.Contains(line.Text, tag) {
if *tDebug == "1" {
fmt.Println(fmt.Println(line.Text))
}
pub.Publish("log", []byte(ks[k]+"\t"+line.Text))
}
}
}
}
}
func tailFile(ctx context.Context, file string, poll bool, dest *os.File) {
defer wg.Done()
t, err := tail.TailFile(file, tail.Config{
Follow: true,
ReOpen: true,
Poll: poll,
Logger: tail.DiscardingLogger,
})
if err != nil {
log.Fatalf("unable to tail %s: %s", "foo", err)
}
// main loop
for {
select {
// if the channel is done, then exit the loop
case <-ctx.Done():
t.Stop()
t.Cleanup()
return
// get the next log line and echo it out
case line := <-t.Lines:
if line != nil {
fmt.Fprintln(dest, line.Text)
}
}
}
}
func main() {
if len(os.Args) != 3 {
fmt.Println("Usage: canaryeye FILENAME COMMAND")
os.Exit(1)
}
fn, cmd := os.Args[1], os.Args[2]
m := map[string]int{}
begin := time.Now()
go canaryeye.Run(canaryeye.GetConfig(), &m, &begin, cmd)
t, err := tail.TailFile(fn, *canaryeye.GetTailConfig())
canaryeye.HandleError(err)
r, _ := regexp.Compile("^([^ ]+)")
for line := range t.Lines {
matches := r.FindAllStringSubmatch(line.Text, -1)
if len(matches) > 0 {
m[matches[0][1]]++
}
}
}
func tailFile(path string) {
t, err := tail.TailFile(path, tail.Config{
Follow: true,
})
_ = t
if err != nil {
return
}
for {
select {
case line := <-t.Lines:
logstream <- types.Message{
FileName: t.Filename,
Text: line.Text,
}
case fn := <-RemoveTail:
if path == fn {
log.Info("remove file %s\n", fn)
return
} else {
RemoveTail <- fn
}
}
}
}
func follow_and_analyze(filename string, regexes []string, c chan incidentstore.Incident) {
t, err := tail.TailFile(filename,
tail.Config{
Follow: true, // actually follow the logs
ReOpen: true, // allow logs to be rotated
Location: &tail.SeekInfo{Offset: 0, Whence: 2}}) // seek to end of file
if err != nil {
log.Fatal(err)
}
var compiledRegexes []regexp.Regexp
for _, regex := range regexes {
compiledRegex := regexp.MustCompile(regex)
compiledRegexes = append(compiledRegexes, *compiledRegex)
}
for line := range t.Lines {
// match each line against all regexes
for _, regex := range compiledRegexes {
result := regex.FindStringSubmatch(line.Text)
if result != nil {
ip := net.ParseIP(result[1])
if ip != nil {
c <- incidentstore.Incident{Filename: filename, Ip: ip, Time: time.Now(), Line: line.Text}
// break here, this line matched on regex
break
}
}
}
}
}
func (s *apiServer) Logs(r *types.LogsRequest, stream types.API_LogsServer) error {
state, err := s.getState(r.Id)
if err != nil {
return err
}
// if the job has completed or failed then we cannot follow
follow := r.Follow
if string(state) == string(jobFailed) || string(state) == string(jobCompleted) {
follow = false
}
stdout := filepath.Join(s.StateDir, string(jobIDByte(r.Id)), "stdout")
t, err := tail.TailFile(stdout, tail.Config{
Follow: follow,
MustExist: true,
})
if err != nil {
return fmt.Errorf("Tail file %s failed: %v", stdout, err)
}
for line := range t.Lines {
if err := stream.Send(&types.Log{Log: line.Text}); err != nil {
return fmt.Errorf("Sending log to stream failed: %v", err)
}
}
return nil
}
func main() {
ConfigBytes, err := ioutil.ReadFile(*ConfigFile)
if err != nil {
log.Fatalf("Error reading config file %s\n", err)
}
err = json.Unmarshal(ConfigBytes, &Config)
if err != nil {
log.Fatalf("Error parsing config file %s\n", err)
}
for _, endpointConfig := range Config.Endpoints {
log.Printf("Starting up %s handler", endpointConfig.Driver)
endpointConfig.e = endpointDrivers[endpointConfig.Driver](endpointConfig.Options)
endpointConfig.e.HandleMessage(Message)
endpointConfig.e.Run()
}
for name, logConfig := range Config.Logs {
go func(name string, logConfig *Log) {
logfile, err := tail.TailFile(logConfig.File, tail.Config{Location: &tail.SeekInfo{Whence: os.SEEK_END}, Follow: true, ReOpen: true})
if err != nil {
log.Printf("Error tailing file: %s", err)
}
var filter *regexp.Regexp
if logConfig.Regex != "" {
var err error
filter, err = regexp.Compile(logConfig.Regex)
if err != nil {
log.Printf("Error compiling regex: %s", err)
}
}
for line := range logfile.Lines {
if line.Err != nil {
log.Printf("Error tailing file: %s", line.Err)
}
if filter == nil || filter.MatchString(line.Text) {
logConfig.lines = append(logConfig.lines, line)
if len(logConfig.lines) > logConfig.Keep {
logConfig.lines = logConfig.lines[len(logConfig.lines)-logConfig.Keep:]
}
/*if logConfig.Live {
for _, channel := range logConfig.Channels {
c.Privmsg(channel, line.Text)
}
}*/
}
}
}(name, logConfig)
}
for _, monitorConfig := range Config.Monitors {
monitorConfig.monitor = monitorDrivers[monitorConfig.Driver](monitorConfig.Options)
monitorConfig.track = newMonitorTrack()
monitorConfig.track.Start(monitorConfig.monitor)
}
quit := make(chan bool)
<-quit
}
func (recv ChromebusRecordTailReceiver) open() {
var err error
recv.t, err = tail.TailFile(Events, tail.Config{Follow: true})
if err != nil {
log.Fatal(err)
}
recv.opened = true
}
// Creates a Watcher around a watched file.
func WatchLog(watched string) Watcher {
tail, err := tail.TailFile(watched, tail.Config{Follow: true, ReOpen: true})
if err != nil {
log.WithError(err).Fatal()
panic(err)
}
return Watcher{Tail: tail}
}
func (i *Tail) createTailReader(config tail.Config) {
tail, err := tail.TailFile(i.file, config)
if err != nil {
Critical("tail %s: %v", i.file, err)
}
i.tail = tail
}
// printLogs streams the content of a file on the standard output.
// When it stops working, it sends a message in the stopNotifier channel.
func (d *Daemon) printLogs(filename string, stopNotifier chan<- string) {
t, _ := tail.TailFile(filename, tail.Config{Follow: true})
for line := range t.Lines {
fmt.Println(line.Text)
}
stopNotifier <- fmt.Sprintf("could not stream logs from `%s`", filename)
}
// OpenLogfile opens a logfile and passes back a *tail.Tail pointer.
func OpenLogfile(logfile string) *tail.Tail {
t, err := tail.TailFile(logfile, tail.Config{
Location: &tail.SeekInfo{Whence: os.SEEK_END},
ReOpen: true,
Follow: true})
if err != nil {
Log("There was an error opening the file.", "info")
}
return t
}
func (this *TailsInput) Tailer(f string) error {
var seek int
var offset int64
var err error
pointfile := fmt.Sprintf("%s/%d", this.config.JournalDirectory, hash(f))
if offset, err = readPoint(pointfile); err != nil {
seek = os.SEEK_END
} else {
seek = os.SEEK_SET
}
t, err := tail.TailFile(f, tail.Config{
Poll: true,
ReOpen: true,
Follow: true,
MustExist: false,
Location: &tail.SeekInfo{int64(offset), seek},
})
if err != nil {
return err
}
tick := time.NewTicker(time.Second * time.Duration(3))
count := 0
for {
select {
case <-tick.C:
{
if count > 0 {
offset, err := t.Tell()
if err != nil {
log.Println("Tell return error: ", err)
continue
}
if err = writePoint(pointfile, offset); err != nil {
return err
}
count = 0
}
}
case line := <-t.Lines:
{
pack := <-this.runner.InChan()
pack.MsgBytes = []byte(line.Text)
pack.Msg.Tag = this.common.Tag
pack.Msg.Timestamp = time.Now().Unix()
count++
this.runner.RouterChan() <- pack
}
}
}
err = t.Wait()
return err
}
// shows all new lines in cconsole.log
func tailCConsoleLog(inst string) {
folder := getInstanceFolder(inst)
endLocation := tail.SeekInfo{Offset: 0, Whence: os.SEEK_END}
if t, err := tail.TailFile(folder+"/mgr/cconsole.log", tail.Config{Follow: true, Location: &endLocation}); err != nil {
log.Printf("Error while getting content for cconsole.log\n")
log.Printf("ERR: %s.\n", err)
} else {
for line := range t.Lines {
fmt.Println(line.Text)
}
}
}
func logHandler(w http.ResponseWriter, r *http.Request, file string) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
apilog.Println(err)
return
}
defer conn.Close()
var more string
var offset tail.SeekInfo
offset.Whence = 2
fi, err := os.Open(file)
if err != nil {
apilog.Println(err)
}
f, err := fi.Stat()
if err != nil {
apilog.Println(err)
}
if f.Size() < 10000 {
offset.Offset = f.Size() * (-1)
} else {
offset.Offset = -10000
more = "... "
}
fi.Close()
conf := tail.Config{
Follow: true,
ReOpen: true,
Location: &offset,
Logger: tail.DiscardingLogger,
}
t, err := tail.TailFile(file, conf)
if err != nil {
apilog.Println(err)
}
for line := range t.Lines {
if line.Err == nil {
if err = conn.WriteMessage(websocket.TextMessage, []byte(more+line.Text)); err != nil {
apilog.Println(err)
return
}
more = ""
} else {
apilog.Println(err)
}
}
}
func tailFile(wr io.Writer, file string, done *chan bool) error {
defer trace.End(trace.Begin(file))
// By default, seek to EOF (if file doesn't exist)
spos := tail.SeekInfo{
Offset: 0,
Whence: 2,
}
// If the file exists, we want to go back tailLines lines
// and pass that new offset into the TailFile() constructor
// Per @fdawg4l, use bytes.LastIndex() and a 1k buffer to reduce
// seeks/reads
f, err := os.Open(file)
if err == nil {
spos = tail.SeekInfo{
Offset: findSeekPos(f),
Whence: 0,
}
}
tcfg := tail.Config{
Location: &spos,
ReOpen: true,
MustExist: false,
Follow: true,
}
t, err := tail.TailFile(file, tcfg)
if err != nil {
return err
}
// We KNOW there's a data race here.
// But it doesn't break anything, so we just trap it.
defer func() {
t.Stop()
_ = recover()
}()
for true {
select {
case l := <-t.Lines:
if l.Err != nil {
return l.Err
}
fmt.Fprint(wr, l.Text, "\n")
case _ = <-*done:
return nil
}
}
return nil
}
func (l *LogFile) Tail() error {
tail, err := tail.TailFile(l.Path, tail.Config{Follow: true})
if err != nil {
log.Debug(err.Error())
return errors.New("Could not follow Hoverfly log file")
}
for line := range tail.Lines {
fmt.Println(line.Text)
}
return nil
}
func newFileTailChannel(filename string) (<-chan *tail.Line, error) {
tailFile, err := tail.TailFile(filename, tail.Config{
Location: &tail.SeekInfo{Offset: 0, Whence: os.SEEK_END},
ReOpen: true,
MustExist: false,
Follow: true,
MaxLineSize: 4096,
})
if err != nil {
return nil, err
}
return tailFile.Lines, nil
}
func storeSched(dockerLog string) {
t, err := tail.TailFile(dockerLog, tail.Config{Follow: true, Location: &tail.SeekInfo{Offset: 0, Whence: 2}})
ok(err)
for {
select {
case line := <-t.Lines:
storeLine(line)
case <-quit:
return
}
}
}
func tailLog(filename string) ([]string, error) {
result := []string{}
t, err := tail.TailFile(config.FactorioLog, tail.Config{Follow: false})
if err != nil {
log.Printf("Error tailing log %s", err)
return result, err
}
for line := range t.Lines {
result = append(result, line.Text)
}
return result, nil
}
func tailFile(filename string, config tail.Config, done chan bool) {
defer func() { done <- true }()
t, err := tail.TailFile(filename, config)
if err != nil {
fmt.Println(err)
return
}
for line := range t.Lines {
fmt.Println(line.Text)
}
err = t.Wait()
if err != nil {
fmt.Println(err)
}
}
func main() {
iniflags.Parse()
api := slack.New(*token)
//var channel_id string
channel_id := getChannelId(*channel, api)
var include, exclude *regexp.Regexp
var err error
if *includes != "" {
include, err = regexp.Compile(*includes)
if err != nil {
fmt.Println("ERROR: Failed to compile `line_includes` regex.")
fmt.Println(err)
api.PostMessage(channel_id, "==> slackd failed to compile `line_includes` regex.", slack.NewPostMessageParameters())
api.PostMessage(channel_id, err.Error(), slack.NewPostMessageParameters())
os.Exit(2)
}
}
if *excludes != "" {
exclude, err = regexp.Compile(*excludes)
if err != nil {
fmt.Println("ERROR: Failed to compile `line_excludes` regex.")
fmt.Println(err)
api.PostMessage(channel_id, "==> slackd failed to compile `line_excludes` regex.", slack.NewPostMessageParameters())
api.PostMessage(channel_id, err.Error(), slack.NewPostMessageParameters())
os.Exit(2)
}
}
log, err := tail.TailFile(*file, tail.Config{Follow: true, ReOpen: *reopen, Poll: true})
if err != nil {
fmt.Println("ERROR: Could not tail the specified log.")
fmt.Println(err)
api.PostMessage(channel_id, "==> slackd could not tail the specified log.", slack.NewPostMessageParameters())
api.PostMessage(channel_id, err.Error(), slack.NewPostMessageParameters())
os.Exit(2)
}
for line := range log.Lines {
if (include != nil && include.MatchString(line.Text)) || (exclude != nil && !exclude.MatchString(line.Text)) {
api.PostMessage(
channel_id,
fmt.Sprintf("```%s```", line.Text),
slack.NewPostMessageParameters())
}
}
}
// Makes tests over tail implementation choosed easier.
func TestTail(t *testing.T) {
// end of tail
eot := "EOT"
if err := createFile(logfile); err != nil {
log.Fatal(err)
t.Fail()
}
tail, err := tail.TailFile(logfile, tail.Config{
Follow: true,
ReOpen: true,
})
if err != nil {
t.Error(err)
}
go func() {
log.Debug("starting")
for line := range tail.Lines {
log.Debug(line)
var en interface{}
if err := json.Unmarshal([]byte(line.Text), &en); err != nil {
log.Debug(err)
}
if line.Text == eot {
log.Debug("ending")
return
}
}
}()
// writing to watched logs
for i := 0; i < 10; i++ {
if err := writeLine(logfile, fmt.Sprintf("line %#v\n", i)); err != nil {
t.Error(err)
break
}
}
// ZZzzzz..
time.Sleep(2 * time.Second)
// ends the watching
writeLine(logfile, eot)
}
// Tail tails the given file for entries.
func Tail(
filePath string,
unmarshaller lion.Unmarshaller,
entryHandler EntryHandler,
errorHandler ErrorHandler, // if nil, Tail will just return the error
options TailOptions,
) error {
t, err := tail.TailFile(filePath, tailOptionsToConfig(options))
if err != nil {
return err
}
if options.Done != nil {
for {
select {
case <-options.Done:
err = t.Stop()
t.Cleanup()
return err
case line := <-t.Lines:
if err := handleLine(
unmarshaller,
entryHandler,
errorHandler,
line.Text,
); err != nil {
return err
}
}
}
} else {
for line := range t.Lines {
if err := handleLine(
unmarshaller,
entryHandler,
errorHandler,
line.Text,
); err != nil {
return err
}
}
}
t.Cleanup()
return nil
}
func (t *Tail) Start(acc telegraf.Accumulator) error {
t.Lock()
defer t.Unlock()
t.acc = acc
var seek tail.SeekInfo
if !t.FromBeginning {
seek.Whence = 2
seek.Offset = 0
}
var errS string
// Create a "tailer" for each file
for _, filepath := range t.Files {
g, err := globpath.Compile(filepath)
if err != nil {
log.Printf("E! Error Glob %s failed to compile, %s", filepath, err)
}
for file, _ := range g.Match() {
tailer, err := tail.TailFile(file,
tail.Config{
ReOpen: true,
Follow: true,
Location: &seek,
MustExist: true,
})
if err != nil {
errS += err.Error() + " "
continue
}
// create a goroutine for each "tailer"
t.wg.Add(1)
go t.receiver(tailer)
t.tailers = append(t.tailers, tailer)
}
}
if errS != "" {
return fmt.Errorf(errS)
}
return nil
}
func (v *FileTail) watcher() {
log.Printf("[INFO] [%s] File watcher started", v.tag)
for {
for file, _ := range v.files {
select {
case <-v.files[file].Dying():
log.Printf("[DEBUG] [%s] File \"%s\" closed", v.tag, file)
delete(v.files, file)
default:
continue
}
}
//Search for new files
for _, path := range v.paths {
f, _ := filepath.Glob(path)
for _, file := range f {
if _, ok := v.files[file]; !ok {
log.Printf("[DEBUG] [%s] Found file \"%s\"", v.tag, file)
tc := tail.Config{Follow: true, ReOpen: false, MustExist: true, Poll: true, Logger: tail.DiscardingLogger}
if s, err := ioutil.ReadFile(file + ".pos"); err == nil {
s := strings.Split(string(s), "\n")
if len(s) == 2 {
if ctime, err := strconv.Atoi(s[0]); err == nil && int64(ctime) == ctimeFile(file) {
if pos, err := strconv.Atoi(s[1]); err == nil {
tc.Location = &tail.SeekInfo{Whence: os.SEEK_CUR, Offset: int64(pos)}
log.Printf("[DEBUG] [%s] Restoring position %d in file \"%s\"", v.tag, pos, file)
}
}
}
}
t, err := tail.TailFile(file, tc)
if err == nil {
go v.worker(t)
v.files[file] = t
}
}
}
}
time.Sleep(time.Second)
}
}
func runStream(bus *pmb.PMB, conn *pmb.Connection, id string) error {
subConn, err := bus.ConnectSubClient(conn, streamCommand.Name)
if err != nil {
return err
}
var ident string
if len(streamCommand.Identifier) > 0 {
ident = streamCommand.Identifier
} else {
ident, err = localIP()
if err != nil {
return err
}
}
tailConfig := tail.Config{
Follow: true,
Location: &tail.SeekInfo{
Offset: 0,
Whence: os.SEEK_END,
},
}
fileTail, err := tail.TailFile(streamCommand.File, tailConfig)
if err != nil {
return err
}
for line := range fileTail.Lines {
subConn.Out <- pmb.Message{Contents: map[string]interface{}{
"type": "Stream",
"identifier": ident,
"data": line.Text,
}}
}
return nil
}
func main() {
flag.Parse()
if *version {
fmt.Println("version", Version)
os.Exit(1)
}
t, err := tail.TailFile(*filename, tail.Config{
Follow: true,
ReOpen: true,
Poll: true,
Location: &tail.SeekInfo{0, 2}})
if nil != err {
fmt.Println(err)
os.Exit(1)
}
num := make(chan int)
if *open_falcon {
go send2falcon(num)
}
lineText := make(chan string, 1)
for line := range t.Lines {
matched, err := regexp.MatchString(*regexpstring, line.Text)
if nil != err {
fmt.Println(err)
os.Exit(1)
}
if matched == true {
if *open_falcon {
num <- 1
}
lineText <- line.Text
go send2zabbix(lineText) //send2zabbix
}
}
}
func ReadLogFile(state *State) {
t, err := tail.TailFile(AuthLog, tail.Config{Follow: true, ReOpen: true})
if err != nil {
log("There was a problem opening the file.")
}
// Tail the auth.log file and inspect lines as they come in.
for line := range t.Lines {
// Check for the failed attempts line pattern
r, _ := regexp.Compile("Failed password|Invalid user")
m := r.MatchString(line.Text)
// If we found it, lets gather up the ip and hand it off to be saved.
if m == true {
r, _ := regexp.Compile("[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}")
ip := r.FindString(line.Text)
if len(ip) < 3 {
log(fmt.Sprintf("The ip was to short to be valid: %v", ip))
} else {
// Does this ip need to be blocked
scum, block := state.CheckIP(ip)
log(fmt.Sprintf("%v to block IP: %v", block, ip))
if block == true {
log(fmt.Sprintf("Blocking ip: %v", ip))
// Block this ip
BlockIP(ip)
// Mark this scum as being blocked
scum.Blocked = true
} else {
log(fmt.Sprintf("ReadLogFile: Not blocking ip: %v", ip))
}
}
}
// Else, we will just move on since this log line has no relevance
}
}