func Publish(input chan []*FileEvent,
registrar chan []*FileEvent,
server_list []string,
public_key [sodium.PUBLICKEYBYTES]byte,
secret_key [sodium.SECRETKEYBYTES]byte,
server_timeout time.Duration) {
var buffer bytes.Buffer
session := sodium.NewSession(public_key, secret_key)
socket := FFS{
Endpoints: server_list,
SocketType: zmq.REQ,
RecvTimeout: server_timeout,
SendTimeout: server_timeout,
}
//defer socket.Close()
for events := range input {
// got a bunch of events, ship them out.
log.Printf("Publisher received %d events\n", len(events))
data, _ := json.Marshal(events)
// TODO(sissel): check error
// Compress it
// A new zlib writer is used for every payload of events so that any
// individual payload can be decompressed alone.
// TODO(sissel): Make compression level tunable
compressor, _ := zlib.NewWriterLevel(&buffer, 3)
buffer.Truncate(0)
_, err := compressor.Write(data)
err = compressor.Flush()
compressor.Close()
//log.Printf("compressed %d bytes\n", buffer.Len())
// TODO(sissel): check err
// TODO(sissel): implement security/encryption/etc
// Send full payload over zeromq REQ/REP
// TODO(sissel): check error
//buffer.Write(data)
ciphertext, nonce := session.Box(buffer.Bytes())
//log.Printf("plaintext: %d\n", len(data))
//log.Printf("compressed: %d\n", buffer.Len())
//log.Printf("ciphertext: %d %v\n", len(ciphertext), ciphertext[:20])
//log.Printf("nonce: %d\n", len(nonce))
// TODO(sissel): figure out encoding for ciphertext + nonce
// TODO(sissel): figure out encoding for ciphertext + nonce
// Loop forever trying to send.
// This will cause reconnects/etc on failures automatically
for {
err = socket.Send(nonce[:], zmq.SNDMORE)
if err != nil {
continue // send failed, retry!
}
err = socket.Send(ciphertext, 0)
if err != nil {
continue // send failed, retry!
}
data, err = socket.Recv(0)
// TODO(sissel): Figure out acknowledgement protocol? If any?
if err == nil {
break // success!
}
}
// Tell the registrar that we've successfully sent these events
//registrar <- events
} /* for each event payload */
} // Publish
func main() {
//f, err := os.OpenFile("log.out", os.O_WRONLY | os.O_CREATE, 0644)
//log.SetOutput(f)
event_chan := make(chan *lumberjack.FileEvent, 16)
publisher_chan := make(chan []*lumberjack.FileEvent, 5)
registrar_chan := make(chan []*lumberjack.FileEvent, 5)
endpoint := "tcp://127.0.0.1:47342"
public, secret := sodium.CryptoBoxKeypair()
go generator(event_chan)
go lumberjack.Spool(event_chan, publisher_chan, SPOOLSIZE, 5*time.Second)
go lumberjack.Publish(publisher_chan, registrar_chan, []string{endpoint},
public, secret, 5*time.Second)
session := sodium.NewSession(public, secret)
context, _ := zmq.NewContext()
socket, _ := context.NewSocket(zmq.REP)
socket.SetSockOptInt(zmq.LINGER, 0)
err := socket.Bind(endpoint)
if err != nil {
log.Fatalf("Failed to bind to %s.\n", endpoint)
}
var buffer bytes.Buffer
var decompressed bytes.Buffer
tmp := make([]byte, 2048)
count := 0
start := time.Now()
for count < 800000 {
nonce, err := socket.Recv(0)
if err != nil {
panic(fmt.Sprintf("socket.Recv: %s\n", err))
}
ciphertext, err := socket.Recv(0)
if err != nil {
panic(fmt.Sprintf("socket.Recv2: %s\n", err))
}
count += int(SPOOLSIZE)
socket.Send([]byte(""), 0)
continue
// Decrypt it
plaintext := session.Open(nonce, ciphertext)
buffer.Truncate(0)
buffer.Write(plaintext)
zr, _ := zlib.NewReader(&buffer)
decompressed.Truncate(0)
for {
n, err := zr.Read(tmp)
if n > 0 {
decompressed.Write(tmp[0:n])
}
if err == io.EOF {
break
}
}
zr.Close()
var events []lumberjack.FileEvent
err = json.Unmarshal(decompressed.Bytes(), &events)
if err != nil {
panic("JSON Unmarshal failed")
}
count += len(events)
}
log.Printf("%d @ %f/sec\n", count, float64(count)/time.Since(start).Seconds())
}