예제 #1
0
func (s *SenderTestSuite) TestConnectErrors(t *C) {
	spool := mock.NewSpooler(nil)

	spool.FilesOut = []string{"slow001.json"}
	spool.DataOut = map[string][]byte{"slow001.json": []byte("...")}

	sender := data.NewSender(s.logger, s.client)

	err := sender.Start(spool, s.tickerChan, 60, false)
	t.Assert(err, IsNil)

	// Any connect error will do.
	s.client.ConnectError = io.EOF
	defer func() { s.client.ConnectError = nil }()

	// Tick causes send to connect and send all files.
	s.tickerChan <- time.Now()
	t0 := time.Now()

	// Wait for sender to start trying to connect...
	if !test.WaitStatus(5, sender, "data-sender", "Connecting") {
		t.Fatal("Timeout waiting for data-sender status=Connecting")
	}
	// ...then wait for it to finsih and return.
	if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") {
		t.Fatal("Timeout waiting for data-sender status=Idle")
	}
	d := time.Now().Sub(t0).Seconds()

	// It should wait between reconnects, but not too long.
	if d < float64((data.MAX_SEND_ERRORS-1)*data.CONNECT_ERROR_WAIT) {
		t.Error("Waits between reconnects")
	}
	if d > float64(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT) {
		t.Error("Waited too long between reconnects")
	}

	err = sender.Stop()
	t.Assert(err, IsNil)

	// Couldn't connect, so it doesn't send or reject anything.
	t.Check(len(spool.DataOut), Equals, 1)
	t.Check(len(spool.RejectedFiles), Equals, 0)

	// It should have called ConnectOnce() serveral times, else it didn't
	// really try to reconnect.
	trace := test.DrainTraceChan(s.client.TraceChan)
	t.Check(trace, DeepEquals, []string{
		"ConnectOnce",
		"ConnectOnce",
		"ConnectOnce",
		"DisconnectOnce",
	})
}
예제 #2
0
func (s *SenderTestSuite) Test500Error(t *C) {
	spool := mock.NewSpooler(nil)
	spool.FilesOut = []string{"file1", "file2", "file3"}
	spool.DataOut = map[string][]byte{
		"file1": []byte("file1"),
		"file2": []byte("file2"),
		"file3": []byte("file3"),
	}

	sender := data.NewSender(s.logger, s.client)
	err := sender.Start(spool, s.tickerChan, 5, false)
	t.Assert(err, IsNil)

	s.tickerChan <- time.Now()

	got := test.WaitBytes(s.dataChan)
	if same, diff := test.IsDeeply(got[0], []byte("file1")); !same {
		t.Error(diff)
	}

	// 3 files before API error.
	t.Check(len(spool.DataOut), Equals, 3)

	// Simulate API error.
	select {
	case s.respChan <- &proto.Response{Code: 503}:
	case <-time.After(500 * time.Millisecond):
		t.Error("Sender receives prot.Response after sending data")
	}

	// Wait for it to finsih and return.
	if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") {
		t.Fatal("Timeout waiting for data-sender status=Idle")
	}

	// Still 3 files after API error.
	t.Check(len(spool.DataOut), Equals, 3)
	t.Check(len(spool.RejectedFiles), Equals, 0)

	// There's only 1 call to SendBytes because after an API error
	// the send stops immediately.
	trace := test.DrainTraceChan(s.client.TraceChan)
	t.Check(trace, DeepEquals, []string{
		"ConnectOnce",
		"SendBytes",
		"Recv",
		"DisconnectOnce",
	})

	err = sender.Stop()
	t.Assert(err, IsNil)
}
예제 #3
0
func (s *SenderTestSuite) TestBadFiles(t *C) {
	spool := mock.NewSpooler(nil)
	spool.FilesOut = []string{"file1", "file2", "file3"}
	spool.DataOut = map[string][]byte{
		"file1": []byte("file1"),
		"file2": []byte("file2"),
		"file3": []byte("file3"),
	}

	sender := data.NewSender(s.logger, s.client)
	err := sender.Start(spool, s.tickerChan, 5, false)
	t.Assert(err, IsNil)

	doneChan := make(chan bool, 1)
	go func() {
		resp := []uint{400, 400, 200}
		for i := 0; i < 3; i++ {
			// Wait for sender to send data.
			select {
			case <-s.dataChan:
			case <-doneChan:
				return
			}

			// Simulate API returns 400.
			select {
			case s.respChan <- &proto.Response{Code: resp[i]}:
			case <-doneChan:
				return
			}
		}
	}()

	s.tickerChan <- time.Now()

	// Wait for sender to finish.
	if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") {
		t.Fatal("Timeout waiting for data-sender status=Idle")
	}

	doneChan <- true
	err = sender.Stop()
	t.Assert(err, IsNil)

	// Bad files are removed, so all files should have been sent.
	t.Check(len(spool.DataOut), Equals, 0)
	t.Check(len(spool.RejectedFiles), Equals, 0)
}
예제 #4
0
func (s *TestSuite) TestStartCollectStop(t *C) {
	// Create the monitor.
	config := &mysql.Config{
		Config: sysconfig.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
		},
	}
	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	// Start the monitor.
	err := m.Start(s.tickChan, s.reportChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	// monitor=Ready once it has successfully connected to MySQL.  This may
	// take a few seconds (hopefully < 5) on a slow test machine.
	if ok := test.WaitStatusPrefix(5, m, s.name, "Idle"); !ok {
		t.Fatal("Monitor is ready")
	}

	// Send tick to make the monitor collect.
	now := time.Now().UTC()
	s.tickChan <- now
	got := test.WaitSystemConfig(s.reportChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a sysconfig after tick")
	}
	c := got[0]

	if c.Ts != now.Unix() {
		t.Error("Report.Ts set to %s; got %s", now.Unix(), c.Ts)
	}

	if len(c.Settings) < 100 {
		t.Fatal("Collect > 100 vars; got %+v", c.Settings)
	}

	haveWaitTimeout := false
	val := ""
	for _, s := range c.Settings {
		if s[0] == "wait_timeout" {
			haveWaitTimeout = true
			val = s[1]
		}
	}
	if !haveWaitTimeout {
		t.Logf("%+v\n", c)
		t.Error("Got wait_timeout")
	}
	if val == "" {
		t.Error("wait_timeout has value")
	}

	/**
	 * Stop the monitor.
	 */

	m.Stop()

	if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok {
		t.Fatal("Monitor has stopped")
	}
}
예제 #5
0
func (s *ManagerTestSuite) TestStartCollectStop(t *C) {
	files := []string{"stat", "meminfo", "vmstat", "loadavg", "diskstats"}
	for _, file := range files {
		if !pct.FileExists("/proc/" + file) {
			t.Fatal("/proc/" + file + " does not exist")
		}
	}

	// Create the monitor.
	m := system.NewMonitor(s.name, &system.Config{}, s.logger)
	if m == nil {
		t.Fatal("Make new system.Monitor")
	}

	// Start the monitor.
	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	// system-monitor=Ready once it has started its internals,
	// should be very fast.
	if ok := test.WaitStatusPrefix(3, m, s.name, "Idle"); !ok {
		t.Fatal("Monitor is ready")
	}

	// The monitor should only collect and send metrics on ticks; we haven't ticked yet.
	got := test.WaitCollection(s.collectionChan, 0)
	if len(got) > 0 {
		t.Fatal("No tick, no collection; got %+v", got)
	}

	// Now tick.  This should make monitor collect.
	now := time.Now()
	s.tickChan <- now

	got = test.WaitCollection(s.collectionChan, 1)
	t.Assert(got, Not(HasLen), 0)
	t.Check(got, HasLen, 1)

	c := got[0]
	t.Check(c.Ts, Equals, now.Unix())

	t.Assert(c.Metrics, Not(HasLen), 0)

	// /proc/stat values are relative (current - prev) so there shouldn't be any
	// after one tick.
	haveCPU, _ := haveMetric("cpu/user", c.Metrics)
	t.Check(haveCPU, Equals, false)

	// But other metrics are not relative, so we should have them.
	metrics := []string{"memory/MemTotal", "vmstat/numa_local", "loadavg/running", "disk/sda/reads"}
	for _, metric := range metrics {
		ok, val := haveMetric(metric, c.Metrics)
		t.Check(ok, Equals, true)
		t.Check(val, Not(Equals), 0)
	}

	// Tick a 2nd time and now we should get CPU metrics.
	time.Sleep(200 * time.Millisecond)
	now = time.Now()
	s.tickChan <- now

	got = test.WaitCollection(s.collectionChan, 1)
	t.Assert(got, Not(HasLen), 0)
	t.Check(got, HasLen, 1)
	c = got[0]
	t.Check(c.Ts, Equals, now.Unix())
	t.Assert(c.Metrics, Not(HasLen), 0)

	metrics = []string{"cpu/user", "cpu/nice", "cpu/system", "cpu/idle"}
	for _, metric := range metrics {
		ok, val := haveMetric(metric, c.Metrics)
		t.Check(ok, Equals, true)

		// Running this test requires some CPU so user and idle shouldn't be zero.
		if metric == "cpu/user" || metric == "cpu/idle" {
			t.Check(val, Not(Equals), 0)
		}
	}

	/**
	 * Stop the monitor.
	 */

	m.Stop()

	if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok {
		t.Fatal("Monitor has stopped")
	}
}
예제 #6
0
func (s *SenderTestSuite) TestRecvErrors(t *C) {
	spool := mock.NewSpooler(nil)
	spool.FilesOut = []string{"slow001.json"}
	spool.DataOut = map[string][]byte{"slow001.json": []byte("...")}

	sender := data.NewSender(s.logger, s.client)

	err := sender.Start(spool, s.tickerChan, 60, false)
	t.Assert(err, IsNil)

	// Any recv error will do.
	doneChan := make(chan bool)
	go func() {
		for {
			select {
			case s.client.RecvError <- io.EOF:
			case <-doneChan:
				return
			}
		}
	}()
	defer func() { doneChan <- true }()

	// Tick causes send to connect and send all files.
	s.tickerChan <- time.Now()
	t0 := time.Now()

	// Wait for sender to finsih and return.
	if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") {
		t.Fatal("Timeout waiting for data-sender status=Idle")
	}
	d := time.Now().Sub(t0).Seconds()

	// It should wait between reconnects, but not too long.
	if d < float64((data.MAX_SEND_ERRORS-1)*data.CONNECT_ERROR_WAIT) {
		t.Error("Waits between reconnects")
	}
	if d > float64(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT) {
		t.Error("Waited too long between reconnects")
	}
	err = sender.Stop()
	t.Assert(err, IsNil)

	// Didn't receive proper ack, so it doesn't remove any data.
	t.Check(len(spool.DataOut), Equals, 1)
	t.Check(len(spool.RejectedFiles), Equals, 0)

	// It should have called ConnectOnce() serveral times, else it didn't
	// really try to reconnect.
	trace := test.DrainTraceChan(s.client.TraceChan)
	t.Check(trace, DeepEquals, []string{
		"ConnectOnce",
		"SendBytes",
		"Recv",
		"DisconnectOnce",
		"ConnectOnce",
		"SendBytes",
		"Recv",
		"DisconnectOnce",
		"ConnectOnce",
		"SendBytes",
		"Recv",
		"DisconnectOnce",
		"DisconnectOnce",
	})
}