func (tail *Tail) seek() { fi, err := tail.Stat() if err != nil { logs.Error("Can't stat file: %s", err) return } off := tail.Offset() if tail.maxBackfill >= 0 { if off < fi.Size()-tail.maxBackfill { off = fi.Size() - tail.maxBackfill } } _, err = tail.handle.Seek(off, 0) if err != nil { logs.Error("Can't seek file: %s", err) return } }
func (tail *Tail) StartWatching() { go func() { fi, err := tail.Stat() if err != nil { logs.Error("Can't stat file: %s", err) return } c := tail.Watcher.ChangeEvents(&tomb.Tomb{}, fi) go tail.pollChannel(c.Modified) go tail.pollChannel(c.Truncated) go tail.pollChannel(c.Deleted) }() }
func (parser *RegexpParser) Consume(bytes []byte, counter *int64) { parser.buffer = append(parser.buffer, bytes...) logs.Debug("consuming %d bytes of %s", len(bytes), parser.file) l := len(parser.buffer) if l > parser.maxLineSize { off := l - parser.maxLineSize logs.Debug("chopping %d bytes off buffer (was: %d, max: %d)", off, l, parser.maxLineSize) atomic.AddInt64(counter, int64(off)) parser.buffer = parser.buffer[off:] } for { m := parser.compiled.FindSubmatchIndex(parser.buffer) if m == nil { return } hasher := sha1.New() out := make(map[string]Column) out["_offset"] = Column{Integer, Simple, atomic.LoadInt64(counter)} out["_file"] = Column{String, Simple, parser.file} out["_time"] = Column{Timestamp, Simple, StandardTimeProvider.Now().Unix()} out["_group"] = Column{String, Simple, parser.group} out["_hostname"] = Column{String, Simple, parser.hostname} for _, spec := range parser.fields { g := spec.Group if g < 0 || g > len(m)/2 { logs.Error("spec group out of range: alias: %s, name: %s, g: %d", spec.Alias, spec.Name, g) panic(-1) } if m[g*2] == -1 { continue } s := string(parser.buffer[m[g*2]:m[g*2+1]]) switch spec.Type { case Timestamp: t, err := time.Parse(spec.Format, s) if err != nil { logs.Warn("date parse error: %s", err) } else { if t.Year() == 0 { now := StandardTimeProvider.Now() adjusted := time.Date(now.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location()) if adjusted.After(now) { adjusted = time.Date(now.Year()-1, t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location()) } t = adjusted } out[spec.Alias] = Column{Timestamp, spec.Treatment, t.Unix()} } case String: if spec.Treatment == Tokens { out[spec.Alias] = Column{Tokens, spec.Treatment, spec.Pattern.FindAllString(s, -1)} } else if spec.Treatment == Hash { hasher.Reset() hasher.Write([]byte(spec.Salt)) hasher.Write([]byte(s)) sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil)) out[spec.Alias] = Column{Tokens, spec.Treatment, sha} } else { out[spec.Alias] = Column{String, spec.Treatment, s} } case Integer: n, err := strconv.ParseInt(s, 10, 64) if err == nil { out[spec.Alias] = Column{spec.Type, spec.Treatment, n} } case Double: n, err := strconv.ParseFloat(s, 64) if err == nil { out[spec.Alias] = Column{spec.Type, spec.Treatment, n} } default: panic(nil) } } parser.output <- out atomic.AddInt64(counter, int64(m[1])) parser.buffer = parser.buffer[m[1]:] } logs.Debug("done with %s", parser.file) }
func main() { flag.Parse() runtime.GOMAXPROCS(*cpus) // set the logger path handle, err := os.OpenFile(*logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { logs.Warn("Unable to open log file %s, using stderr: %s", *logFile, err) } else { logs.Logger = log.New(handle, "", log.LstdFlags|log.Lshortfile) } // Check whether we're in debug mode if *debug { logs.SetLevel(logs.DEBUG) logs.Debug("logging at DEBUG") } else { logs.SetLevel(logs.INFO) } if *name == "unknown" { *name, err = os.Hostname() if err != nil { logs.Warn("Unable to determine hostname: %s", err) } } // Read the config files config, err := dendrite.NewConfig(*configFile, *name) if err != nil { logs.Fatal("Can't read configuration: %s", err) } // Link up all of the objects ch := make(chan dendrite.Record, 100) logs.Debug("original %s", ch) dests := config.CreateDestinations() groups := config.CreateAllTailGroups(ch) // If any of our destinations talk back, log it. go func() { reader := bufio.NewReader(dests.Reader()) for { str, err := reader.ReadString('\n') if err == io.EOF { logs.Debug("eof") time.Sleep(1 * time.Second) } else if err != nil { logs.Error("error reading: %s", err) } else { logs.Info("received: %s", str) } } }() // Do the event loop finished := make(chan bool, 0) go dests.Consume(ch, finished) if *quitAfter >= 0 { start := time.Now() logs.Debug("starting the poll") i := 0 for { i++ if i%10 == 0 { groups.Refresh() } groups.Poll() if time.Now().Sub(start) >= time.Duration((*quitAfter)*float64(time.Second)) { break } } } else { logs.Debug("starting the loop") groups.Loop() } logs.Info("Closing...") close(ch) <-finished logs.Info("Goodbye!") }