func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
go func() {
time.Sleep(60 * time.Second)
pprof.StopCPUProfile()
panic("done")
}()
}
// Turn 'host' and 'host:port' into 'tcp://host:port'
if *servers == "" {
log.Printf("No servers specified, please provide the -servers setting\n")
return
}
server_list := strings.Split(*servers, ",")
for i, server := range server_list {
if !strings.Contains(server, ":") {
server_list[i] = "tcp://" + server + ":5005"
} else {
server_list[i] = "tcp://" + server
}
}
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
// TODO(sissel): support flags for setting... stuff
event_chan := make(chan *lumberjack.FileEvent, 16)
publisher_chan := make(chan []*lumberjack.FileEvent, 1)
// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to resume a harvester.
// Prospect the globs/paths given on the command line and launch harvesters
go lumberjack.Prospect(flag.Args(), event_chan)
// Harvesters dump events into the spooler.
go lumberjack.Spool(event_chan, publisher_chan, *spool_size, *idle_timeout)
lumberjack.Publish(publisher_chan, server_list, *server_timeout)
// TODO(sissel): publisher should send state to the registrar
// TODO(sissel): registrar records last acknowledged positions in all files.
} /* main */
func main() {
// TODO(sissel): support flags for setting... stuff
event_chan := make(chan *lumberjack.FileEvent, 32)
publisher_chan := make(chan *lumberjack.EventEnvelope, 1)
// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to resume a harvester.
// TODO(sissel): need a prospector scan for files and launch harvesters
// Prospect the globs/paths given on the command line and launch harvesters
go lumberjack.Prospect(os.Args[1:], event_chan)
var window_size uint64 = 1024 // Make this a flag
var idle_timeout time.Duration = 1 * time.Second // Make this a flag
// Harvesters dump events into the spooler.
go lumberjack.Spooler(event_chan, publisher_chan, window_size, idle_timeout)
// Spooler flushes periodically to the publisher
for x := range publisher_chan {
// got a bunch of events, ship them out.
fmt.Printf("Spooler gave me %d events\n", len(x.Events))
for _, event := range x.Events {
fmt.Println(event)
}
}
// TODO(sissel): publisher should send state to the registrar
// TODO(sissel): registrar records last acknowledged positions in all files.
} /* main */