func (s *SenderTestSuite) TestConnectErrors(t *C) { spool := mock.NewSpooler(nil) spool.FilesOut = []string{"slow001.json"} spool.DataOut = map[string][]byte{"slow001.json": []byte("...")} sender := data.NewSender(s.logger, s.client) err := sender.Start(spool, s.tickerChan, 60, false) t.Assert(err, IsNil) // Any connect error will do. s.client.ConnectError = io.EOF defer func() { s.client.ConnectError = nil }() // Tick causes send to connect and send all files. s.tickerChan <- time.Now() t0 := time.Now() // Wait for sender to start trying to connect... if !test.WaitStatus(5, sender, "data-sender", "Connecting") { t.Fatal("Timeout waiting for data-sender status=Connecting") } // ...then wait for it to finsih and return. if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") { t.Fatal("Timeout waiting for data-sender status=Idle") } d := time.Now().Sub(t0).Seconds() // It should wait between reconnects, but not too long. if d < float64((data.MAX_SEND_ERRORS-1)*data.CONNECT_ERROR_WAIT) { t.Error("Waits between reconnects") } if d > float64(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT) { t.Error("Waited too long between reconnects") } err = sender.Stop() t.Assert(err, IsNil) // Couldn't connect, so it doesn't send or reject anything. t.Check(len(spool.DataOut), Equals, 1) t.Check(len(spool.RejectedFiles), Equals, 0) // It should have called ConnectOnce() serveral times, else it didn't // really try to reconnect. trace := test.DrainTraceChan(s.client.TraceChan) t.Check(trace, DeepEquals, []string{ "ConnectOnce", "ConnectOnce", "ConnectOnce", "DisconnectOnce", }) }
func (s *SenderTestSuite) Test500Error(t *C) { spool := mock.NewSpooler(nil) spool.FilesOut = []string{"file1", "file2", "file3"} spool.DataOut = map[string][]byte{ "file1": []byte("file1"), "file2": []byte("file2"), "file3": []byte("file3"), } sender := data.NewSender(s.logger, s.client) err := sender.Start(spool, s.tickerChan, 5, false) t.Assert(err, IsNil) s.tickerChan <- time.Now() got := test.WaitBytes(s.dataChan) if same, diff := test.IsDeeply(got[0], []byte("file1")); !same { t.Error(diff) } // 3 files before API error. t.Check(len(spool.DataOut), Equals, 3) // Simulate API error. select { case s.respChan <- &proto.Response{Code: 503}: case <-time.After(500 * time.Millisecond): t.Error("Sender receives prot.Response after sending data") } // Wait for it to finsih and return. if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") { t.Fatal("Timeout waiting for data-sender status=Idle") } // Still 3 files after API error. t.Check(len(spool.DataOut), Equals, 3) t.Check(len(spool.RejectedFiles), Equals, 0) // There's only 1 call to SendBytes because after an API error // the send stops immediately. trace := test.DrainTraceChan(s.client.TraceChan) t.Check(trace, DeepEquals, []string{ "ConnectOnce", "SendBytes", "Recv", "DisconnectOnce", }) err = sender.Stop() t.Assert(err, IsNil) }
func (s *SenderTestSuite) TestBadFiles(t *C) { spool := mock.NewSpooler(nil) spool.FilesOut = []string{"file1", "file2", "file3"} spool.DataOut = map[string][]byte{ "file1": []byte("file1"), "file2": []byte("file2"), "file3": []byte("file3"), } sender := data.NewSender(s.logger, s.client) err := sender.Start(spool, s.tickerChan, 5, false) t.Assert(err, IsNil) doneChan := make(chan bool, 1) go func() { resp := []uint{400, 400, 200} for i := 0; i < 3; i++ { // Wait for sender to send data. select { case <-s.dataChan: case <-doneChan: return } // Simulate API returns 400. select { case s.respChan <- &proto.Response{Code: resp[i]}: case <-doneChan: return } } }() s.tickerChan <- time.Now() // Wait for sender to finish. if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") { t.Fatal("Timeout waiting for data-sender status=Idle") } doneChan <- true err = sender.Stop() t.Assert(err, IsNil) // Bad files are removed, so all files should have been sent. t.Check(len(spool.DataOut), Equals, 0) t.Check(len(spool.RejectedFiles), Equals, 0) }
func (s *TestSuite) TestStartCollectStop(t *C) { // Create the monitor. config := &mysql.Config{ Config: sysconfig.Config{ ServiceInstance: proto.ServiceInstance{ Service: "mysql", InstanceId: 1, }, }, } m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn)) if m == nil { t.Fatal("Make new mysql.Monitor") } // Start the monitor. err := m.Start(s.tickChan, s.reportChan) if err != nil { t.Fatalf("Start monitor without error, got %s", err) } // monitor=Ready once it has successfully connected to MySQL. This may // take a few seconds (hopefully < 5) on a slow test machine. if ok := test.WaitStatusPrefix(5, m, s.name, "Idle"); !ok { t.Fatal("Monitor is ready") } // Send tick to make the monitor collect. now := time.Now().UTC() s.tickChan <- now got := test.WaitSystemConfig(s.reportChan, 1) if len(got) == 0 { t.Fatal("Got a sysconfig after tick") } c := got[0] if c.Ts != now.Unix() { t.Error("Report.Ts set to %s; got %s", now.Unix(), c.Ts) } if len(c.Settings) < 100 { t.Fatal("Collect > 100 vars; got %+v", c.Settings) } haveWaitTimeout := false val := "" for _, s := range c.Settings { if s[0] == "wait_timeout" { haveWaitTimeout = true val = s[1] } } if !haveWaitTimeout { t.Logf("%+v\n", c) t.Error("Got wait_timeout") } if val == "" { t.Error("wait_timeout has value") } /** * Stop the monitor. */ m.Stop() if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok { t.Fatal("Monitor has stopped") } }
func (s *ManagerTestSuite) TestStartCollectStop(t *C) { files := []string{"stat", "meminfo", "vmstat", "loadavg", "diskstats"} for _, file := range files { if !pct.FileExists("/proc/" + file) { t.Fatal("/proc/" + file + " does not exist") } } // Create the monitor. m := system.NewMonitor(s.name, &system.Config{}, s.logger) if m == nil { t.Fatal("Make new system.Monitor") } // Start the monitor. err := m.Start(s.tickChan, s.collectionChan) if err != nil { t.Fatalf("Start monitor without error, got %s", err) } // system-monitor=Ready once it has started its internals, // should be very fast. if ok := test.WaitStatusPrefix(3, m, s.name, "Idle"); !ok { t.Fatal("Monitor is ready") } // The monitor should only collect and send metrics on ticks; we haven't ticked yet. got := test.WaitCollection(s.collectionChan, 0) if len(got) > 0 { t.Fatal("No tick, no collection; got %+v", got) } // Now tick. This should make monitor collect. now := time.Now() s.tickChan <- now got = test.WaitCollection(s.collectionChan, 1) t.Assert(got, Not(HasLen), 0) t.Check(got, HasLen, 1) c := got[0] t.Check(c.Ts, Equals, now.Unix()) t.Assert(c.Metrics, Not(HasLen), 0) // /proc/stat values are relative (current - prev) so there shouldn't be any // after one tick. haveCPU, _ := haveMetric("cpu/user", c.Metrics) t.Check(haveCPU, Equals, false) // But other metrics are not relative, so we should have them. metrics := []string{"memory/MemTotal", "vmstat/numa_local", "loadavg/running", "disk/sda/reads"} for _, metric := range metrics { ok, val := haveMetric(metric, c.Metrics) t.Check(ok, Equals, true) t.Check(val, Not(Equals), 0) } // Tick a 2nd time and now we should get CPU metrics. time.Sleep(200 * time.Millisecond) now = time.Now() s.tickChan <- now got = test.WaitCollection(s.collectionChan, 1) t.Assert(got, Not(HasLen), 0) t.Check(got, HasLen, 1) c = got[0] t.Check(c.Ts, Equals, now.Unix()) t.Assert(c.Metrics, Not(HasLen), 0) metrics = []string{"cpu/user", "cpu/nice", "cpu/system", "cpu/idle"} for _, metric := range metrics { ok, val := haveMetric(metric, c.Metrics) t.Check(ok, Equals, true) // Running this test requires some CPU so user and idle shouldn't be zero. if metric == "cpu/user" || metric == "cpu/idle" { t.Check(val, Not(Equals), 0) } } /** * Stop the monitor. */ m.Stop() if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok { t.Fatal("Monitor has stopped") } }
func (s *SenderTestSuite) TestRecvErrors(t *C) { spool := mock.NewSpooler(nil) spool.FilesOut = []string{"slow001.json"} spool.DataOut = map[string][]byte{"slow001.json": []byte("...")} sender := data.NewSender(s.logger, s.client) err := sender.Start(spool, s.tickerChan, 60, false) t.Assert(err, IsNil) // Any recv error will do. doneChan := make(chan bool) go func() { for { select { case s.client.RecvError <- io.EOF: case <-doneChan: return } } }() defer func() { doneChan <- true }() // Tick causes send to connect and send all files. s.tickerChan <- time.Now() t0 := time.Now() // Wait for sender to finsih and return. if !test.WaitStatusPrefix(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT, sender, "data-sender", "Idle") { t.Fatal("Timeout waiting for data-sender status=Idle") } d := time.Now().Sub(t0).Seconds() // It should wait between reconnects, but not too long. if d < float64((data.MAX_SEND_ERRORS-1)*data.CONNECT_ERROR_WAIT) { t.Error("Waits between reconnects") } if d > float64(data.MAX_SEND_ERRORS*data.CONNECT_ERROR_WAIT) { t.Error("Waited too long between reconnects") } err = sender.Stop() t.Assert(err, IsNil) // Didn't receive proper ack, so it doesn't remove any data. t.Check(len(spool.DataOut), Equals, 1) t.Check(len(spool.RejectedFiles), Equals, 0) // It should have called ConnectOnce() serveral times, else it didn't // really try to reconnect. trace := test.DrainTraceChan(s.client.TraceChan) t.Check(trace, DeepEquals, []string{ "ConnectOnce", "SendBytes", "Recv", "DisconnectOnce", "ConnectOnce", "SendBytes", "Recv", "DisconnectOnce", "ConnectOnce", "SendBytes", "Recv", "DisconnectOnce", "DisconnectOnce", }) }