Exemplo n.º 1
1
func (s *S) TestReconnect(c *C) {
	l, err := net.Listen("tcp", ":0")
	c.Assert(err, IsNil)
	err = l.Close()
	c.Assert(err, IsNil)

	msgc := make(chan *rfc5424.Message)
	handler := func(msg *rfc5424.Message) {
		msgc <- msg
	}

	addr := l.Addr().String()
	_, err = s.discd.AddServiceAndRegister("test-reconnect", addr)
	c.Assert(err, IsNil)

	donec := make(chan struct{})
	defer close(donec)

	sc, err := connect(s.discd, "test-reconnect", donec)
	c.Assert(err, IsNil)

	l, err = net.Listen("tcp", addr)
	c.Assert(err, IsNil)
	go runServer(l, handler)

	want := rfc5424.NewMessage(&rfc5424.Header{Version: 1}, []byte("test message"))
	_, err = sc.Write(rfc6587.Bytes(want))
	c.Assert(err, IsNil)

	c.Assert(<-msgc, DeepEquals, want)
}
Exemplo n.º 2
0
func (s *ServerTestSuite) TestServerDurability(c *C) {
	srv := testServer(c)
	c.Assert(srv.Start(), IsNil)
	defer srv.Shutdown()

	cl := testClient(c, srv)

	conn, err := net.Dial("tcp", srv.SyslogAddr().String())
	c.Assert(err, IsNil)
	defer conn.Close()

	zero := 0
	rc, err := cl.GetLog("app-A", &logagg.LogOpts{Follow: true, Lines: &zero})
	c.Assert(err, IsNil)

	for _, msg := range appAMessages {
		conn.Write(rfc6587.Bytes(msg))
	}

	var got client.Message
	dec := json.NewDecoder(rc)
	for _, want := range appAMessages {
		c.Assert(dec.Decode(&got), IsNil)
		c.Assert(got.HostID, Equals, string(want.Hostname))
		c.Assert(got.Stream, Equals, utils.StreamType(want))
		c.Assert(got.Timestamp.Equal(want.Timestamp), Equals, true)

		procType, jobID := splitProcID(want.ProcID)
		c.Assert(got.ProcessType, Equals, string(procType))
		c.Assert(got.JobID, Equals, string(jobID))
	}
}
Exemplo n.º 3
0
func (m *LogMux) drainTo(w io.Writer) {
	defer close(m.donec)

	g := grohl.NewContext(grohl.Data{"at": "logmux_drain"})

	for {
		msg, ok := <-m.logc
		if !ok {
			return // shutdown
		}

		_, err := w.Write(rfc6587.Bytes(msg))
		if err != nil {
			g.Log(grohl.Data{"status": "error", "err": err.Error()})

			// write logs to local logger when the writer fails
			g.Log(grohl.Data{"msg": msg.String()})
			for msg := range m.logc {
				g.Log(grohl.Data{"msg": msg.String()})
			}

			return // shutdown
		}
	}
}
Exemplo n.º 4
0
func (s *ServerTestSuite) TestHostCursors(c *C) {
	srv := testServer(c)
	srv.testMessageHook = make(chan struct{}, 1)
	c.Assert(srv.Start(), IsNil)
	defer srv.Shutdown()
	cl := testClient(c, srv)

	conn, err := net.Dial("tcp", srv.SyslogAddr().String())
	c.Assert(err, IsNil)
	defer conn.Close()

	assertCursors := func(expected map[string]utils.HostCursor) {
		cursors, err := cl.GetCursors()
		c.Assert(err, IsNil)
		c.Assert(cursors, DeepEquals, expected)
	}
	write := func(msg *rfc5424.Message) {
		conn.Write(rfc6587.Bytes(msg))
		<-srv.testMessageHook
	}

	// write some messages
	msg1, msg2 := newSeqMessage("host1", 1, 0), newSeqMessage("host2", 1, 0)
	write(msg1)
	write(msg2)

	assertCursors(map[string]utils.HostCursor{
		"host1": {msg1.Timestamp, 1},
		"host2": {msg2.Timestamp, 1},
	})

	// test new timestamp with seq rolled over
	msg3 := newSeqMessage("host1", 1, 1)
	write(msg3)
	assertCursors(map[string]utils.HostCursor{
		"host1": {msg3.Timestamp, 1},
		"host2": {msg2.Timestamp, 1},
	})

	// test same timestamp with new seq
	msg4 := newSeqMessage("host1", 2, 1)
	write(msg4)
	assertCursors(map[string]utils.HostCursor{
		"host1": {msg4.Timestamp, 2},
		"host2": {msg2.Timestamp, 1},
	})
}
Exemplo n.º 5
0
func (s *S) TestDurableWrite(c *C) {
	l, err := net.Listen("tcp", ":0")
	c.Assert(err, IsNil)

	msgc := make(chan *rfc5424.Message)
	handler := func(msg *rfc5424.Message) {
		msgc <- msg
	}

	go runServer(l, handler)

	donec := make(chan struct{})
	defer close(donec)

	sc, err := connect(s.discd, "test-broken", donec)
	c.Assert(err, IsNil)

	addr := l.Addr().String()
	_, err = s.discd.AddServiceAndRegister("test-broken", addr)
	c.Assert(err, IsNil)

	hdr := &rfc5424.Header{}
	for i := 0; i < 10; i++ {
		want := rfc5424.NewMessage(hdr, []byte(fmt.Sprintf("line %d", i+1)))

		_, err = sc.Write(rfc6587.Bytes(want))
		c.Assert(err, IsNil)

		if i%2 == 0 {
			// break the underlying connection before the next Write
			c.Assert(sc.Conn.Close(), IsNil)
		}

		got := <-msgc
		c.Assert(want, DeepEquals, got)
	}
}
Exemplo n.º 6
0
func (l *appLog) Write(msg message) {
	l.l.Write(append(rfc6587.Bytes(msg.Message), '\n'))
	l.m.broadcast(l.appID, msg)
}
Exemplo n.º 7
0
func (m *Mux) addAggregator(addr string) {
	l := m.logger.New("fn", "addAggregator", "addr", addr)
	// TODO(titanous): add dial timeout
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		l.Error("failed to connect to aggregator", "error", err)
		return
	}
	l.Info("connected to aggregator")

	host, _, _ := net.SplitHostPort(addr)
	c, _ := client.New("http://" + host)
	cursors, err := c.GetCursors()
	if err != nil {
		// TODO(titanous): retry
		l.Error("failed to get cursors from aggregator", "error", err)
		conn.Close()
		return
	}

	var aggCursor *utils.HostCursor
	if c, ok := cursors[m.hostID]; ok {
		aggCursor = &c
	}
	if aggCursor != nil {
		l.Info("got cursor", "cursor.timestamp", aggCursor.Time, "cursor.seq", aggCursor.Seq)
	} else {
		l.Info("no cursor for host")
	}

	appLogs, err := m.logFiles("")
	if err != nil {
		l.Error("failed to get local log files", "error", err)
		conn.Close()
		return
	}

	bufferedMessages := make(chan message)
	firehose := make(chan message)
	done := make(chan struct{})

	// subscribe to all messages
	unsubscribe := m.subscribe(firehoseApp, firehose)

	bufferCursors := make(map[string]utils.HostCursor)
	var bufferCursorsMtx sync.Mutex
	go func() {
		l := m.logger.New("fn", "sendToAggregator", "addr", addr)
		defer unsubscribe()
		defer conn.Close()
		defer close(done)
		bm := bufferedMessages // make a copy so we can nil it later
		for {
			var m message
			var ok bool
			select {
			case m, ok = <-bm:
				if !ok {
					bm = nil
					continue
				}
			case m, ok = <-firehose:
				if !ok {
					return
				}

				// if app in list of app logs and cursor from reading files, skip
				appID := string(m.Message.AppName)
				if _, ok := appLogs[appID]; ok {
					bufferCursorsMtx.Lock()
					c, ok := bufferCursors[appID]
					bufferCursorsMtx.Unlock()
					if !ok || c.After(*m.HostCursor) {
						continue
					}
				}
			}
			if _, err := conn.Write(rfc6587.Bytes(m.Message)); err != nil {
				l.Error("failed to write message", "error", err)
				return
			}
		}
	}()

	for appID, logs := range appLogs {
		for i, name := range logs {
			func() {
				l := l.New("log", name)
				f, err := os.Open(name)
				if err != nil {
					l.Error("failed to open log file", "error", err)
					return
				}
				defer f.Close()
				sc := bufio.NewScanner(f)
				sc.Split(rfc6587.SplitWithNewlines)
				var cursor *utils.HostCursor
				cursorSaved := false
			scan:
				for sc.Scan() {
					msgBytes := sc.Bytes()
					// slice in msgBytes could get modified on next Scan(), need to copy it
					msgCopy := make([]byte, len(msgBytes)-1)
					copy(msgCopy, msgBytes)
					var msg *rfc5424.Message
					msg, cursor, err = utils.ParseMessage(msgCopy)
					if err != nil {
						l.Error("failed to parse message", "msg", string(msgCopy), "error", err)
						continue
					}
					if aggCursor != nil && !cursor.After(*aggCursor) {
						continue
					}
					select {
					case bufferedMessages <- message{cursor, msg}:
					case <-done:
						return
					}
				}
				if err := sc.Err(); err != nil {
					l.Error("failed to scan message", "error", err)
					return
				}
				if !cursorSaved && i == len(appLogs[appID])-1 {
					// last file, send cursor to processing goroutine
					bufferCursorsMtx.Lock()
					bufferCursors[appID] = *cursor
					bufferCursorsMtx.Unlock()
					cursorSaved = true
					// read to end of file again
					goto scan
				}
			}()
		}
	}
	close(bufferedMessages)
}