func (s *S) TestReconnect(c *C) { l, err := net.Listen("tcp", ":0") c.Assert(err, IsNil) err = l.Close() c.Assert(err, IsNil) msgc := make(chan *rfc5424.Message) handler := func(msg *rfc5424.Message) { msgc <- msg } addr := l.Addr().String() _, err = s.discd.AddServiceAndRegister("test-reconnect", addr) c.Assert(err, IsNil) donec := make(chan struct{}) defer close(donec) sc, err := connect(s.discd, "test-reconnect", donec) c.Assert(err, IsNil) l, err = net.Listen("tcp", addr) c.Assert(err, IsNil) go runServer(l, handler) want := rfc5424.NewMessage(&rfc5424.Header{Version: 1}, []byte("test message")) _, err = sc.Write(rfc6587.Bytes(want)) c.Assert(err, IsNil) c.Assert(<-msgc, DeepEquals, want) }
func (s *ServerTestSuite) TestServerDurability(c *C) { srv := testServer(c) c.Assert(srv.Start(), IsNil) defer srv.Shutdown() cl := testClient(c, srv) conn, err := net.Dial("tcp", srv.SyslogAddr().String()) c.Assert(err, IsNil) defer conn.Close() zero := 0 rc, err := cl.GetLog("app-A", &logagg.LogOpts{Follow: true, Lines: &zero}) c.Assert(err, IsNil) for _, msg := range appAMessages { conn.Write(rfc6587.Bytes(msg)) } var got client.Message dec := json.NewDecoder(rc) for _, want := range appAMessages { c.Assert(dec.Decode(&got), IsNil) c.Assert(got.HostID, Equals, string(want.Hostname)) c.Assert(got.Stream, Equals, utils.StreamType(want)) c.Assert(got.Timestamp.Equal(want.Timestamp), Equals, true) procType, jobID := splitProcID(want.ProcID) c.Assert(got.ProcessType, Equals, string(procType)) c.Assert(got.JobID, Equals, string(jobID)) } }
func (m *LogMux) drainTo(w io.Writer) { defer close(m.donec) g := grohl.NewContext(grohl.Data{"at": "logmux_drain"}) for { msg, ok := <-m.logc if !ok { return // shutdown } _, err := w.Write(rfc6587.Bytes(msg)) if err != nil { g.Log(grohl.Data{"status": "error", "err": err.Error()}) // write logs to local logger when the writer fails g.Log(grohl.Data{"msg": msg.String()}) for msg := range m.logc { g.Log(grohl.Data{"msg": msg.String()}) } return // shutdown } } }
func (s *ServerTestSuite) TestHostCursors(c *C) { srv := testServer(c) srv.testMessageHook = make(chan struct{}, 1) c.Assert(srv.Start(), IsNil) defer srv.Shutdown() cl := testClient(c, srv) conn, err := net.Dial("tcp", srv.SyslogAddr().String()) c.Assert(err, IsNil) defer conn.Close() assertCursors := func(expected map[string]utils.HostCursor) { cursors, err := cl.GetCursors() c.Assert(err, IsNil) c.Assert(cursors, DeepEquals, expected) } write := func(msg *rfc5424.Message) { conn.Write(rfc6587.Bytes(msg)) <-srv.testMessageHook } // write some messages msg1, msg2 := newSeqMessage("host1", 1, 0), newSeqMessage("host2", 1, 0) write(msg1) write(msg2) assertCursors(map[string]utils.HostCursor{ "host1": {msg1.Timestamp, 1}, "host2": {msg2.Timestamp, 1}, }) // test new timestamp with seq rolled over msg3 := newSeqMessage("host1", 1, 1) write(msg3) assertCursors(map[string]utils.HostCursor{ "host1": {msg3.Timestamp, 1}, "host2": {msg2.Timestamp, 1}, }) // test same timestamp with new seq msg4 := newSeqMessage("host1", 2, 1) write(msg4) assertCursors(map[string]utils.HostCursor{ "host1": {msg4.Timestamp, 2}, "host2": {msg2.Timestamp, 1}, }) }
func (s *S) TestDurableWrite(c *C) { l, err := net.Listen("tcp", ":0") c.Assert(err, IsNil) msgc := make(chan *rfc5424.Message) handler := func(msg *rfc5424.Message) { msgc <- msg } go runServer(l, handler) donec := make(chan struct{}) defer close(donec) sc, err := connect(s.discd, "test-broken", donec) c.Assert(err, IsNil) addr := l.Addr().String() _, err = s.discd.AddServiceAndRegister("test-broken", addr) c.Assert(err, IsNil) hdr := &rfc5424.Header{} for i := 0; i < 10; i++ { want := rfc5424.NewMessage(hdr, []byte(fmt.Sprintf("line %d", i+1))) _, err = sc.Write(rfc6587.Bytes(want)) c.Assert(err, IsNil) if i%2 == 0 { // break the underlying connection before the next Write c.Assert(sc.Conn.Close(), IsNil) } got := <-msgc c.Assert(want, DeepEquals, got) } }
func (l *appLog) Write(msg message) { l.l.Write(append(rfc6587.Bytes(msg.Message), '\n')) l.m.broadcast(l.appID, msg) }
func (m *Mux) addAggregator(addr string) { l := m.logger.New("fn", "addAggregator", "addr", addr) // TODO(titanous): add dial timeout conn, err := net.Dial("tcp", addr) if err != nil { l.Error("failed to connect to aggregator", "error", err) return } l.Info("connected to aggregator") host, _, _ := net.SplitHostPort(addr) c, _ := client.New("http://" + host) cursors, err := c.GetCursors() if err != nil { // TODO(titanous): retry l.Error("failed to get cursors from aggregator", "error", err) conn.Close() return } var aggCursor *utils.HostCursor if c, ok := cursors[m.hostID]; ok { aggCursor = &c } if aggCursor != nil { l.Info("got cursor", "cursor.timestamp", aggCursor.Time, "cursor.seq", aggCursor.Seq) } else { l.Info("no cursor for host") } appLogs, err := m.logFiles("") if err != nil { l.Error("failed to get local log files", "error", err) conn.Close() return } bufferedMessages := make(chan message) firehose := make(chan message) done := make(chan struct{}) // subscribe to all messages unsubscribe := m.subscribe(firehoseApp, firehose) bufferCursors := make(map[string]utils.HostCursor) var bufferCursorsMtx sync.Mutex go func() { l := m.logger.New("fn", "sendToAggregator", "addr", addr) defer unsubscribe() defer conn.Close() defer close(done) bm := bufferedMessages // make a copy so we can nil it later for { var m message var ok bool select { case m, ok = <-bm: if !ok { bm = nil continue } case m, ok = <-firehose: if !ok { return } // if app in list of app logs and cursor from reading files, skip appID := string(m.Message.AppName) if _, ok := appLogs[appID]; ok { bufferCursorsMtx.Lock() c, ok := bufferCursors[appID] bufferCursorsMtx.Unlock() if !ok || c.After(*m.HostCursor) { continue } } } if _, err := conn.Write(rfc6587.Bytes(m.Message)); err != nil { l.Error("failed to write message", "error", err) return } } }() for appID, logs := range appLogs { for i, name := range logs { func() { l := l.New("log", name) f, err := os.Open(name) if err != nil { l.Error("failed to open log file", "error", err) return } defer f.Close() sc := bufio.NewScanner(f) sc.Split(rfc6587.SplitWithNewlines) var cursor *utils.HostCursor cursorSaved := false scan: for sc.Scan() { msgBytes := sc.Bytes() // slice in msgBytes could get modified on next Scan(), need to copy it msgCopy := make([]byte, len(msgBytes)-1) copy(msgCopy, msgBytes) var msg *rfc5424.Message msg, cursor, err = utils.ParseMessage(msgCopy) if err != nil { l.Error("failed to parse message", "msg", string(msgCopy), "error", err) continue } if aggCursor != nil && !cursor.After(*aggCursor) { continue } select { case bufferedMessages <- message{cursor, msg}: case <-done: return } } if err := sc.Err(); err != nil { l.Error("failed to scan message", "error", err) return } if !cursorSaved && i == len(appLogs[appID])-1 { // last file, send cursor to processing goroutine bufferCursorsMtx.Lock() bufferCursors[appID] = *cursor bufferCursorsMtx.Unlock() cursorSaved = true // read to end of file again goto scan } }() } } close(bufferedMessages) }