Example #1
0
func main() {
	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		go func() {
			time.Sleep(60 * time.Second)
			pprof.StopCPUProfile()
			panic("done")
		}()
	}

	// Turn 'host' and 'host:port' into 'tcp://host:port'
	if *servers == "" {
		log.Printf("No servers specified, please provide the -servers setting\n")
		return
	}
	server_list := strings.Split(*servers, ",")
	for i, server := range server_list {
		if !strings.Contains(server, ":") {
			server_list[i] = "tcp://" + server + ":5005"
		} else {
			server_list[i] = "tcp://" + server
		}
	}

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// TODO(sissel): support flags for setting... stuff
	event_chan := make(chan *lumberjack.FileEvent, 16)
	publisher_chan := make(chan []*lumberjack.FileEvent, 1)

	// The basic model of execution:
	// - prospector: finds files in paths/globs to harvest, starts harvesters
	// - harvester: reads a file, sends events to the spooler
	// - spooler: buffers events until ready to flush to the publisher
	// - publisher: writes to the network, notifies registrar
	// - registrar: records positions of files read
	// Finally, prospector uses the registrar information, on restart, to
	// determine where in each file to resume a harvester.

	// Prospect the globs/paths given on the command line and launch harvesters
	go lumberjack.Prospect(flag.Args(), event_chan)

	// Harvesters dump events into the spooler.
	go lumberjack.Spool(event_chan, publisher_chan, *spool_size, *idle_timeout)

	lumberjack.Publish(publisher_chan, server_list, *server_timeout)

	// TODO(sissel): publisher should send state to the registrar
	// TODO(sissel): registrar records last acknowledged positions in all files.
} /* main */
Example #2
0
func main() {
	//f, err := os.OpenFile("log.out", os.O_WRONLY | os.O_CREATE, 0644)
	//log.SetOutput(f)
	event_chan := make(chan *lumberjack.FileEvent, 16)
	publisher_chan := make(chan []*lumberjack.FileEvent, 5)
	registrar_chan := make(chan []*lumberjack.FileEvent, 5)

	endpoint := "tcp://127.0.0.1:47342"
	public, secret := sodium.CryptoBoxKeypair()

	go generator(event_chan)
	go lumberjack.Spool(event_chan, publisher_chan, SPOOLSIZE, 5*time.Second)
	go lumberjack.Publish(publisher_chan, registrar_chan, []string{endpoint},
		public, secret, 5*time.Second)

	session := sodium.NewSession(public, secret)
	context, _ := zmq.NewContext()
	socket, _ := context.NewSocket(zmq.REP)
	socket.SetSockOptInt(zmq.LINGER, 0)
	err := socket.Bind(endpoint)
	if err != nil {
		log.Fatalf("Failed to bind to %s.\n", endpoint)
	}

	var buffer bytes.Buffer
	var decompressed bytes.Buffer
	tmp := make([]byte, 2048)
	count := 0
	start := time.Now()

	for count < 800000 {
		nonce, err := socket.Recv(0)
		if err != nil {
			panic(fmt.Sprintf("socket.Recv: %s\n", err))
		}
		ciphertext, err := socket.Recv(0)
		if err != nil {
			panic(fmt.Sprintf("socket.Recv2: %s\n", err))
		}

		count += int(SPOOLSIZE)
		socket.Send([]byte(""), 0)
		continue

		// Decrypt it
		plaintext := session.Open(nonce, ciphertext)

		buffer.Truncate(0)
		buffer.Write(plaintext)
		zr, _ := zlib.NewReader(&buffer)
		decompressed.Truncate(0)
		for {
			n, err := zr.Read(tmp)
			if n > 0 {
				decompressed.Write(tmp[0:n])
			}
			if err == io.EOF {
				break
			}
		}
		zr.Close()

		var events []lumberjack.FileEvent
		err = json.Unmarshal(decompressed.Bytes(), &events)
		if err != nil {
			panic("JSON Unmarshal failed")
		}
		count += len(events)
	}

	log.Printf("%d @ %f/sec\n", count, float64(count)/time.Since(start).Seconds())
}
Example #3
0
func main() {
	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		go func() {
			time.Sleep(60 * time.Second)
			pprof.StopCPUProfile()
			panic("done")
		}()
	}

	if *their_public_key_path == "" {
		log.Fatalf("No -their-public-key flag given")
	}

	// Turn 'host' and 'host:port' into 'tcp://host:port'
	if *servers == "" {
		log.Fatalf("No servers specified, please provide the -servers setting\n")
	}

	server_list := strings.Split(*servers, ",")
	for i, server := range server_list {
		if !strings.Contains(server, ":") {
			server_list[i] = "tcp://" + server + ":5005"
		} else {
			server_list[i] = "tcp://" + server
		}
	}

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// TODO(sissel): support flags for setting... stuff
	event_chan := make(chan *lumberjack.FileEvent, 16)
	publisher_chan := make(chan []*lumberjack.FileEvent, 1)
	registrar_chan := make(chan []*lumberjack.FileEvent, 1)

	paths := flag.Args()

	if len(paths) == 0 {
		log.Fatalf("No paths given. What files do you want me to watch?\n")
	}

	var public_key [sodium.PUBLICKEYBYTES]byte

	err := read_key(*their_public_key_path, public_key[:])
	if err != nil {
		log.Fatalf("Unable to read public key path (%s): %s\n",
			*their_public_key_path, err)
	}

	var secret_key [sodium.SECRETKEYBYTES]byte
	if *our_secret_key_path == "" {
		log.Printf("No secret key given; generating one.")
		_, secret_key = sodium.CryptoBoxKeypair()
	} else {
		err := read_key(*our_secret_key_path, secret_key[:])
		if err != nil {
			log.Printf("Unable to read secret key (%s): %s\n",
				*our_secret_key_path, err)
			log.Printf("Generating a key pair now.\n")
			_, sk := sodium.CryptoBoxKeypair()
			copy(secret_key[:], sk[:])
		}
	}

	// The basic model of execution:
	// - prospector: finds files in paths/globs to harvest, starts harvesters
	// - harvester: reads a file, sends events to the spooler
	// - spooler: buffers events until ready to flush to the publisher
	// - publisher: writes to the network, notifies registrar
	// - registrar: records positions of files read
	// Finally, prospector uses the registrar information, on restart, to
	// determine where in each file to resume a harvester.

	// Prospect the globs/paths given on the command line and launch harvesters
	go lumberjack.Prospect(paths, event_chan)

	// Harvesters dump events into the spooler.
	go lumberjack.Spool(event_chan, publisher_chan, *spool_size, *idle_timeout)

	lumberjack.Publish(publisher_chan, registrar_chan, server_list,
		public_key, secret_key, *server_timeout)

	// TODO(sissel): registrar db path
	// TODO(sissel): registrar records last acknowledged positions in all files.
	//lumberjack.Registrar(registrar_chan)
} /* main */