func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
go func() {
time.Sleep(60 * time.Second)
pprof.StopCPUProfile()
panic("done")
}()
}
// Turn 'host' and 'host:port' into 'tcp://host:port'
if *servers == "" {
log.Printf("No servers specified, please provide the -servers setting\n")
return
}
server_list := strings.Split(*servers, ",")
for i, server := range server_list {
if !strings.Contains(server, ":") {
server_list[i] = "tcp://" + server + ":5005"
} else {
server_list[i] = "tcp://" + server
}
}
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
// TODO(sissel): support flags for setting... stuff
event_chan := make(chan *lumberjack.FileEvent, 16)
publisher_chan := make(chan []*lumberjack.FileEvent, 1)
// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to resume a harvester.
// Prospect the globs/paths given on the command line and launch harvesters
go lumberjack.Prospect(flag.Args(), event_chan)
// Harvesters dump events into the spooler.
go lumberjack.Spool(event_chan, publisher_chan, *spool_size, *idle_timeout)
lumberjack.Publish(publisher_chan, server_list, *server_timeout)
// TODO(sissel): publisher should send state to the registrar
// TODO(sissel): registrar records last acknowledged positions in all files.
} /* main */
func main() {
//f, err := os.OpenFile("log.out", os.O_WRONLY | os.O_CREATE, 0644)
//log.SetOutput(f)
event_chan := make(chan *lumberjack.FileEvent, 16)
publisher_chan := make(chan []*lumberjack.FileEvent, 5)
registrar_chan := make(chan []*lumberjack.FileEvent, 5)
endpoint := "tcp://127.0.0.1:47342"
public, secret := sodium.CryptoBoxKeypair()
go generator(event_chan)
go lumberjack.Spool(event_chan, publisher_chan, SPOOLSIZE, 5*time.Second)
go lumberjack.Publish(publisher_chan, registrar_chan, []string{endpoint},
public, secret, 5*time.Second)
session := sodium.NewSession(public, secret)
context, _ := zmq.NewContext()
socket, _ := context.NewSocket(zmq.REP)
socket.SetSockOptInt(zmq.LINGER, 0)
err := socket.Bind(endpoint)
if err != nil {
log.Fatalf("Failed to bind to %s.\n", endpoint)
}
var buffer bytes.Buffer
var decompressed bytes.Buffer
tmp := make([]byte, 2048)
count := 0
start := time.Now()
for count < 800000 {
nonce, err := socket.Recv(0)
if err != nil {
panic(fmt.Sprintf("socket.Recv: %s\n", err))
}
ciphertext, err := socket.Recv(0)
if err != nil {
panic(fmt.Sprintf("socket.Recv2: %s\n", err))
}
count += int(SPOOLSIZE)
socket.Send([]byte(""), 0)
continue
// Decrypt it
plaintext := session.Open(nonce, ciphertext)
buffer.Truncate(0)
buffer.Write(plaintext)
zr, _ := zlib.NewReader(&buffer)
decompressed.Truncate(0)
for {
n, err := zr.Read(tmp)
if n > 0 {
decompressed.Write(tmp[0:n])
}
if err == io.EOF {
break
}
}
zr.Close()
var events []lumberjack.FileEvent
err = json.Unmarshal(decompressed.Bytes(), &events)
if err != nil {
panic("JSON Unmarshal failed")
}
count += len(events)
}
log.Printf("%d @ %f/sec\n", count, float64(count)/time.Since(start).Seconds())
}
func main() {
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
go func() {
time.Sleep(60 * time.Second)
pprof.StopCPUProfile()
panic("done")
}()
}
if *their_public_key_path == "" {
log.Fatalf("No -their-public-key flag given")
}
// Turn 'host' and 'host:port' into 'tcp://host:port'
if *servers == "" {
log.Fatalf("No servers specified, please provide the -servers setting\n")
}
server_list := strings.Split(*servers, ",")
for i, server := range server_list {
if !strings.Contains(server, ":") {
server_list[i] = "tcp://" + server + ":5005"
} else {
server_list[i] = "tcp://" + server
}
}
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
// TODO(sissel): support flags for setting... stuff
event_chan := make(chan *lumberjack.FileEvent, 16)
publisher_chan := make(chan []*lumberjack.FileEvent, 1)
registrar_chan := make(chan []*lumberjack.FileEvent, 1)
paths := flag.Args()
if len(paths) == 0 {
log.Fatalf("No paths given. What files do you want me to watch?\n")
}
var public_key [sodium.PUBLICKEYBYTES]byte
err := read_key(*their_public_key_path, public_key[:])
if err != nil {
log.Fatalf("Unable to read public key path (%s): %s\n",
*their_public_key_path, err)
}
var secret_key [sodium.SECRETKEYBYTES]byte
if *our_secret_key_path == "" {
log.Printf("No secret key given; generating one.")
_, secret_key = sodium.CryptoBoxKeypair()
} else {
err := read_key(*our_secret_key_path, secret_key[:])
if err != nil {
log.Printf("Unable to read secret key (%s): %s\n",
*our_secret_key_path, err)
log.Printf("Generating a key pair now.\n")
_, sk := sodium.CryptoBoxKeypair()
copy(secret_key[:], sk[:])
}
}
// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to resume a harvester.
// Prospect the globs/paths given on the command line and launch harvesters
go lumberjack.Prospect(paths, event_chan)
// Harvesters dump events into the spooler.
go lumberjack.Spool(event_chan, publisher_chan, *spool_size, *idle_timeout)
lumberjack.Publish(publisher_chan, registrar_chan, server_list,
public_key, secret_key, *server_timeout)
// TODO(sissel): registrar db path
// TODO(sissel): registrar records last acknowledged positions in all files.
//lumberjack.Registrar(registrar_chan)
} /* main */