func (tail *Tail) seek() {
fi, err := tail.Stat()
if err != nil {
logs.Error("Can't stat file: %s", err)
return
}
off := tail.Offset()
if tail.maxBackfill >= 0 {
if off < fi.Size()-tail.maxBackfill {
off = fi.Size() - tail.maxBackfill
}
}
_, err = tail.handle.Seek(off, 0)
if err != nil {
logs.Error("Can't seek file: %s", err)
return
}
}
func (tail *Tail) StartWatching() {
go func() {
fi, err := tail.Stat()
if err != nil {
logs.Error("Can't stat file: %s", err)
return
}
c := tail.Watcher.ChangeEvents(&tomb.Tomb{}, fi)
go tail.pollChannel(c.Modified)
go tail.pollChannel(c.Truncated)
go tail.pollChannel(c.Deleted)
}()
}
func (parser *RegexpParser) Consume(bytes []byte, counter *int64) {
parser.buffer = append(parser.buffer, bytes...)
logs.Debug("consuming %d bytes of %s", len(bytes), parser.file)
l := len(parser.buffer)
if l > parser.maxLineSize {
off := l - parser.maxLineSize
logs.Debug("chopping %d bytes off buffer (was: %d, max: %d)", off, l, parser.maxLineSize)
atomic.AddInt64(counter, int64(off))
parser.buffer = parser.buffer[off:]
}
for {
m := parser.compiled.FindSubmatchIndex(parser.buffer)
if m == nil {
return
}
hasher := sha1.New()
out := make(map[string]Column)
out["_offset"] = Column{Integer, Simple, atomic.LoadInt64(counter)}
out["_file"] = Column{String, Simple, parser.file}
out["_time"] = Column{Timestamp, Simple, StandardTimeProvider.Now().Unix()}
out["_group"] = Column{String, Simple, parser.group}
out["_hostname"] = Column{String, Simple, parser.hostname}
for _, spec := range parser.fields {
g := spec.Group
if g < 0 || g > len(m)/2 {
logs.Error("spec group out of range: alias: %s, name: %s, g: %d", spec.Alias, spec.Name, g)
panic(-1)
}
if m[g*2] == -1 {
continue
}
s := string(parser.buffer[m[g*2]:m[g*2+1]])
switch spec.Type {
case Timestamp:
t, err := time.Parse(spec.Format, s)
if err != nil {
logs.Warn("date parse error: %s", err)
} else {
if t.Year() == 0 {
now := StandardTimeProvider.Now()
adjusted := time.Date(now.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location())
if adjusted.After(now) {
adjusted = time.Date(now.Year()-1, t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location())
}
t = adjusted
}
out[spec.Alias] = Column{Timestamp, spec.Treatment, t.Unix()}
}
case String:
if spec.Treatment == Tokens {
out[spec.Alias] = Column{Tokens, spec.Treatment, spec.Pattern.FindAllString(s, -1)}
} else if spec.Treatment == Hash {
hasher.Reset()
hasher.Write([]byte(spec.Salt))
hasher.Write([]byte(s))
sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
out[spec.Alias] = Column{Tokens, spec.Treatment, sha}
} else {
out[spec.Alias] = Column{String, spec.Treatment, s}
}
case Integer:
n, err := strconv.ParseInt(s, 10, 64)
if err == nil {
out[spec.Alias] = Column{spec.Type, spec.Treatment, n}
}
case Double:
n, err := strconv.ParseFloat(s, 64)
if err == nil {
out[spec.Alias] = Column{spec.Type, spec.Treatment, n}
}
default:
panic(nil)
}
}
parser.output <- out
atomic.AddInt64(counter, int64(m[1]))
parser.buffer = parser.buffer[m[1]:]
}
logs.Debug("done with %s", parser.file)
}
func main() {
flag.Parse()
runtime.GOMAXPROCS(*cpus)
// set the logger path
handle, err := os.OpenFile(*logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
logs.Warn("Unable to open log file %s, using stderr: %s", *logFile, err)
} else {
logs.Logger = log.New(handle, "", log.LstdFlags|log.Lshortfile)
}
// Check whether we're in debug mode
if *debug {
logs.SetLevel(logs.DEBUG)
logs.Debug("logging at DEBUG")
} else {
logs.SetLevel(logs.INFO)
}
if *name == "unknown" {
*name, err = os.Hostname()
if err != nil {
logs.Warn("Unable to determine hostname: %s", err)
}
}
// Read the config files
config, err := dendrite.NewConfig(*configFile, *name)
if err != nil {
logs.Fatal("Can't read configuration: %s", err)
}
// Link up all of the objects
ch := make(chan dendrite.Record, 100)
logs.Debug("original %s", ch)
dests := config.CreateDestinations()
groups := config.CreateAllTailGroups(ch)
// If any of our destinations talk back, log it.
go func() {
reader := bufio.NewReader(dests.Reader())
for {
str, err := reader.ReadString('\n')
if err == io.EOF {
logs.Debug("eof")
time.Sleep(1 * time.Second)
} else if err != nil {
logs.Error("error reading: %s", err)
} else {
logs.Info("received: %s", str)
}
}
}()
// Do the event loop
finished := make(chan bool, 0)
go dests.Consume(ch, finished)
if *quitAfter >= 0 {
start := time.Now()
logs.Debug("starting the poll")
i := 0
for {
i++
if i%10 == 0 {
groups.Refresh()
}
groups.Poll()
if time.Now().Sub(start) >= time.Duration((*quitAfter)*float64(time.Second)) {
break
}
}
} else {
logs.Debug("starting the loop")
groups.Loop()
}
logs.Info("Closing...")
close(ch)
<-finished
logs.Info("Goodbye!")
}
