func (s *RepoTestSuite) TestInit(t *C) {
im := instance.NewRepo(s.logger, s.configDir, s.api)
t.Assert(im, NotNil)
err := im.Init()
t.Check(err, IsNil)
err = test.CopyFile(test.RootDir+"/mm/config/mysql-1.conf", s.configDir)
t.Assert(err, IsNil)
err = im.Init()
t.Assert(err, IsNil)
mysqlIt := &proto.MySQLInstance{}
err = im.Get("mysql", 1, mysqlIt)
t.Assert(err, IsNil)
expect := &proto.MySQLInstance{
Id: 1,
Hostname: "db1",
DSN: "user:host@tcp:(127.0.0.1:3306)",
Distro: "Percona Server",
Version: "5.6.16",
}
if same, diff := test.IsDeeply(mysqlIt, expect); !same {
test.Dump(mysqlIt)
test.Dump(expect)
t.Error(diff)
}
}
func (s *ProcMeminfoTestSuite) TestProcMeminfo001(t *C) {
m := system.NewMonitor("", &system.Config{}, s.logger)
content, err := ioutil.ReadFile(sample + "/proc/meminfo001.txt")
if err != nil {
t.Fatal(err)
}
got, err := m.ProcMeminfo(content)
if err != nil {
t.Fatal(err)
}
// Remember: the order of this array must match order in which each
// stat appears in the input file:
expect := []mm.Metric{
{Name: "memory/MemTotal", Type: "gauge", Number: 8046892}, // ok
{Name: "memory/MemFree", Type: "gauge", Number: 5273644}, // ok
{Name: "memory/Buffers", Type: "gauge", Number: 300684}, // ok
{Name: "memory/Cached", Type: "gauge", Number: 946852}, // ok
{Name: "memory/SwapCached", Type: "gauge", Number: 0}, // ok
{Name: "memory/Active", Type: "gauge", Number: 1936436}, // ok
{Name: "memory/Inactive", Type: "gauge", Number: 598916}, // ok
{Name: "memory/SwapTotal", Type: "gauge", Number: 8253436}, // ok
{Name: "memory/SwapFree", Type: "gauge", Number: 8253436}, // ok
{Name: "memory/Dirty", Type: "gauge", Number: 0}, // ok
}
if same, diff := test.IsDeeply(got, expect); !same {
test.Dump(got)
t.Error(diff)
}
}
func (s *ProcVmstatTestSuite) TestProcVmstat001(t *C) {
m := system.NewMonitor("", &system.Config{}, s.logger)
content, err := ioutil.ReadFile(sample + "/proc/vmstat001.txt")
if err != nil {
t.Fatal(err)
}
got, err := m.ProcVmstat(content)
if err != nil {
t.Fatal(err)
}
// Remember: the order of this array must match order in which each
// stat appears in the input file:
expect := []mm.Metric{
{Name: "vmstat/numa_hit", Type: "counter", Number: 42594095}, // ok
{Name: "vmstat/numa_miss", Type: "counter", Number: 0}, // ok
{Name: "vmstat/numa_foreign", Type: "counter", Number: 0}, // ok
{Name: "vmstat/numa_interleave", Type: "counter", Number: 7297}, // ok
{Name: "vmstat/numa_local", Type: "counter", Number: 42594095}, // ok
{Name: "vmstat/numa_other", Type: "counter", Number: 0}, // ok
{Name: "vmstat/pgpgin", Type: "counter", Number: 646645}, // ok
{Name: "vmstat/pgpgout", Type: "counter", Number: 5401659}, // ok
{Name: "vmstat/pswpin", Type: "counter", Number: 0}, // ok
{Name: "vmstat/pswpout", Type: "counter", Number: 0}, // ok
}
if same, diff := test.IsDeeply(got, expect); !same {
test.Dump(got)
t.Error(diff)
}
}
func (s *AggregatorTestSuite) TestBadMetric(t *C) {
/**
* Bad metrics should not exist and certainly not aggregated because they
* can go undetected for a long time because they'll result in zero values
* which are valid in normal cases. The metric is bad in the input because
* its type is "guage" instead of "gauge", and it's the only metric so the
* result should be zero metrics.
*/
a := mm.NewAggregator(s.logger, 60, s.collectionChan, s.spool)
go a.Start()
defer a.Stop()
file := fmt.Sprintf("%s/bad_metric.json", sample)
if err := sendCollection(file, s.collectionChan); err != nil {
t.Fatal(file, err)
}
file = fmt.Sprintf("%s/bad_metric-n.json", sample)
if err := sendCollection(file, s.collectionChan); err != nil {
t.Fatal(file, err)
}
got := test.WaitMmReport(s.dataChan)
if got != nil {
test.Dump(got)
t.Error("Got a bad metric")
}
}
func (s *AgentTestSuite) TestStartServiceSlow(t *C) {
// This test is like TestStartService but simulates a slow starting service.
qanConfig := &qan.Config{
Interval: 60, // seconds
MaxSlowLogSize: 1073741824, // 1 GiB
RemoveOldSlowLogs: true,
ExampleQueries: true,
MaxWorkers: 2,
WorkerRunTime: 120, // seconds
}
qanConfigData, _ := json.Marshal(qanConfig)
serviceCmd := &proto.ServiceData{
Name: "qan",
Config: qanConfigData,
}
serviceData, _ := json.Marshal(serviceCmd)
now := time.Now()
cmd := &proto.Cmd{
Ts: now,
User: "******",
Service: "agent",
Cmd: "StartService",
Data: serviceData,
}
// Send the cmd to the client, tell the agent to stop, then wait for it.
s.sendChan <- cmd
// No replies yet.
gotReplies := test.WaitReply(s.recvChan)
if len(gotReplies) != 0 {
t.Fatal("No reply before StartService")
}
// Agent should be able to reply on status chan, indicating that it's
// still starting the service.
gotStatus := test.GetStatus(s.sendChan, s.recvChan)
if !t.Check(gotStatus["agent"], Equals, "Idle") {
test.Dump(gotStatus)
}
// Make it seem like service has started now.
s.readyChan <- true
// Agent sends reply: no error.
gotReplies = test.WaitReply(s.recvChan)
if len(gotReplies) == 0 {
t.Fatal("Get reply")
}
if len(gotReplies) > 1 {
t.Errorf("One reply, got %+v", gotReplies)
}
reply := &proto.Reply{}
_ = json.Unmarshal(gotReplies[0].Data, reply)
t.Check(reply.Error, Equals, "")
}
func (s *ManagerTestSuite) TestStartStopManager(t *C) {
/**
* mm is a proxy manager for monitors, so it's always running.
* It should implement the service manager interface anyway,
* but it doesn't actually start or stop. Its main work is done
* in Handle, starting and stopping monitors (tested later).
*/
mrm := mock.NewMrmsMonitor()
m := mm.NewManager(s.logger, s.factory, s.clock, s.spool, s.im, mrm)
if m == nil {
t.Fatal("Make new mm.Manager")
}
// It shouldn't have added a tickChan yet.
if len(s.clock.Added) != 0 {
t.Error("tickChan not added yet")
}
// First the API marshals an mm.Config.
config := &mm.Config{
ServiceInstance: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Collect: 1,
Report: 60,
// No monitor-specific config
}
err := pct.Basedir.WriteConfig("mm-mysql-1", config)
t.Assert(err, IsNil)
// The agent calls mm.Start().
err = m.Start()
t.Assert(err, IsNil)
// There is a monitor so there should be tickers.
if ok, diff := test.IsDeeply(s.clock.Added, []uint{1}); !ok {
test.Dump(s.clock.Added)
t.Errorf("Does not add tickChan, got %#v", diff)
}
// Its status should be "Running".
status := m.Status()
t.Check(status["mm"], Equals, "Running")
// Can't start mm twice.
err = m.Start()
t.Check(err, Not(Equals), "")
// Stopping should be idempotent.
err = m.Stop()
t.Check(err, IsNil)
err = m.Stop()
t.Check(err, IsNil)
status = m.Status()
t.Check(status["mm"], Equals, "Stopped")
}
func (s *AgentTestSuite) TestLoadConfig(t *C) {
// Load a partial config to make sure LoadConfig() works in general but also
// when the config has missing options (which is normal).
os.Remove(s.configFile)
test.CopyFile(sample+"/config001.json", s.configFile)
bytes, err := agent.LoadConfig()
t.Assert(err, IsNil)
got := &agent.Config{}
if err := json.Unmarshal(bytes, got); err != nil {
t.Fatal(err)
}
expect := &agent.Config{
AgentUuid: "abc-123-def",
ApiHostname: agent.DEFAULT_API_HOSTNAME,
ApiKey: "123",
Keepalive: agent.DEFAULT_KEEPALIVE,
PidFile: agent.DEFAULT_PIDFILE,
}
if same, diff := test.IsDeeply(got, expect); !same {
// @todo: if expect is not ptr, IsDeeply dies with "got ptr, expected struct"
test.Dump(got)
t.Error(diff)
}
// Load a config with all options to make sure LoadConfig() hasn't missed any.
os.Remove(s.configFile)
test.CopyFile(sample+"/full_config.json", s.configFile)
bytes, err = agent.LoadConfig()
t.Assert(err, IsNil)
got = &agent.Config{}
if err := json.Unmarshal(bytes, got); err != nil {
t.Fatal(err)
}
expect = &agent.Config{
ApiHostname: "agent hostname",
ApiKey: "api key",
AgentUuid: "agent uuid",
Keepalive: agent.DEFAULT_KEEPALIVE,
PidFile: "pid file",
}
if same, diff := test.IsDeeply(got, expect); !same {
test.Dump(got)
t.Error(diff)
}
}
func (s *AggregatorTestSuite) TestC003Lost(t *C) {
interval := int64(5)
a := mm.NewAggregator(s.logger, interval, s.collectionChan, s.spool)
go a.Start()
defer a.Stop()
// Ts in c003 is 2009-11-10 23:00:00.
t1, _ := time.Parse("2006-01-02 15:04:05", "2009-11-10 23:00:00")
// The full sequence is files 1-5, but we send only 1 and 5,
// simulating monitor failure during 2-4. More below...
file := fmt.Sprintf("%s/c003-1.json", sample)
if err := sendCollection(file, s.collectionChan); err != nil {
t.Fatal(file, err)
}
file = fmt.Sprintf("%s/c003-5.json", sample)
if err := sendCollection(file, s.collectionChan); err != nil {
t.Fatal(file, err)
}
// Next interval causes 1st to be reported.
file = fmt.Sprintf("%s/c003-n.json", sample)
if err := sendCollection(file, s.collectionChan); err != nil {
t.Fatal(file, err)
}
/**
* Values we did get are 100 and 1600 and ts 00 to 04. So that looks like
* 1500 bytes / 4s = 375. And since there was only 1 interval, we expect
* 375 for all stat values.
*/
got := test.WaitMmReport(s.dataChan)
t.Assert(got, NotNil)
t.Check(got.Ts, Equals, t1)
t.Check(uint64(got.Duration), Equals, uint64(interval))
expect := &mm.Report{}
if err := test.LoadMmReport(sample+"/c003rlost.json", expect); err != nil {
t.Fatal("c003r.json ", err)
}
if ok, diff := test.IsDeeply(got.Stats, expect.Stats); !ok {
test.Dump(got.Stats)
test.Dump(expect.Stats)
t.Fatal(diff)
}
}
func (s *AggregatorTestSuite) TestC001(t *C) {
interval := int64(300)
a := mm.NewAggregator(s.logger, interval, s.collectionChan, s.spool)
go a.Start()
defer a.Stop()
// Load collection from file and send to aggregator.
if err := sendCollection(sample+"/c001-1.json", s.collectionChan); err != nil {
t.Fatal(err)
}
// Ts in c001 is 2009-11-10 23:00:00.
t1, _ := time.Parse("2006-01-02 15:04:05", "2009-11-10 23:00:00")
got := test.WaitMmReport(s.dataChan)
if got != nil {
t.Error("No report before 2nd interval, got: %+v", got)
}
// Ts in c001 is 2009-11-10 23:05:01, 1s into the next interval.
if err := sendCollection(sample+"/c001-2.json", s.collectionChan); err != nil {
t.Fatal(err)
}
got = test.WaitMmReport(s.dataChan)
t.Assert(got, NotNil)
t.Check(got.Ts, Equals, t1)
t.Check(uint64(got.Duration), Equals, uint64(interval))
expect := &mm.Report{}
if err := test.LoadMmReport(sample+"/c001r.json", expect); err != nil {
t.Fatal(err)
}
t.Check(got.Ts, Equals, t1)
if ok, diff := test.IsDeeply(got.Stats, expect.Stats); !ok {
test.Dump(got.Stats)
test.Dump(expect.Stats)
t.Fatal(diff)
}
}
func (s *AgentTestSuite) TestGetAllConfigs(t *C) {
cmd := &proto.Cmd{
Ts: time.Now(),
User: "******",
Cmd: "GetAllConfigs",
Service: "agent",
}
s.sendChan <- cmd
got := test.WaitReply(s.recvChan)
t.Assert(len(got), Equals, 1)
reply := got[0]
t.Check(reply.Error, Equals, "")
t.Assert(reply.Data, Not(HasLen), 0)
gotConfigs := []proto.AgentConfig{}
err := json.Unmarshal(reply.Data, &gotConfigs)
t.Assert(err, IsNil)
bytes, _ := json.Marshal(s.config)
sort.Sort(test.ByInternalService(gotConfigs))
expectConfigs := []proto.AgentConfig{
{
InternalService: "agent",
Config: string(bytes),
Running: true,
},
{
InternalService: "mm",
Config: `{"Foo":"bar"}`,
Running: false,
},
{
InternalService: "qan",
Config: `{"Foo":"bar"}`,
Running: false,
},
}
if ok, diff := test.IsDeeply(gotConfigs, expectConfigs); !ok {
test.Dump(gotConfigs)
t.Error(diff)
}
}
func (s *ProcLoadavgTestSuite) TestProcLoadavg001(t *C) {
m := system.NewMonitor("", &system.Config{}, s.logger)
content, err := ioutil.ReadFile(sample + "/proc/loadavg001.txt")
if err != nil {
t.Fatal(err)
}
got, err := m.ProcLoadavg(content)
if err != nil {
t.Fatal(err)
}
// Remember: the order of this array must match order in which each
// stat appears in the input file:
expect := []mm.Metric{
{Name: "loadavg/1min", Type: "gauge", Number: 0.45}, // ok
{Name: "loadavg/5min", Type: "gauge", Number: 0.56}, // ok
{Name: "loadavg/15min", Type: "gauge", Number: 0.58}, // ok
{Name: "loadavg/running", Type: "gauge", Number: 1}, // ok
{Name: "loadavg/processes", Type: "gauge", Number: 598}, // ok
}
if same, diff := test.IsDeeply(got, expect); !same {
test.Dump(got)
t.Error(diff)
}
}
func (s *ProcStatTestSuite) TestProcStat001(t *C) {
files, err := filepath.Glob(sample + "/proc/stat001-*.txt")
if err != nil {
t.Fatal(err)
}
m := system.NewMonitor("", &system.Config{}, s.logger)
metrics := []mm.Metric{}
for _, file := range files {
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
got, err := m.ProcStat(content)
if err != nil {
t.Fatal(err)
}
metrics = append(metrics, got...)
}
/*
Totals Diff
stat001-1
cpu 390817611
cpu0 97641434
cpu1 97717127
stat001-2
cpu 391386603 568992
cpu0 97783608 142174 These don't add up because the real input has 4 CPU.
cpu1 97859411 142284 This does not affect the tests.
stat001-3
cpu 391759882 373279
cpu0 97876875 93267
cpu1 97952757 93346
1 2 3 4 5 6 7 8 9 10
user nice system idle iowait irq softirq steal guest guestlow
*/
expect := []mm.Metric{
// First input, no CPU because that requires previous values, so only cpu-ext:
{Name: "cpu-ext/intr", Type: "counter", Number: 39222211}, // ok
{Name: "cpu-ext/ctxt", Type: "counter", Number: 122462971}, // ok
{Name: "cpu-ext/processes", Type: "counter", Number: 227223}, // ok
{Name: "cpu-ext/procs_running", Type: "gauge", Number: 1}, // ok
{Name: "cpu-ext/procs_blocked", Type: "gauge", Number: 0}, // ok
// Second input, now we have CPU values, plus more cpu-ext values.
{Name: "cpu/user", Type: "gauge", Number: 0.041477}, // ok 5
{Name: "cpu/nice", Type: "gauge", Number: 0}, // ok
{Name: "cpu/system", Type: "gauge", Number: 0.017751}, // ok 7
{Name: "cpu/idle", Type: "gauge", Number: 99.938488}, // ok
{Name: "cpu/iowait", Type: "gauge", Number: 0.002285}, // ok 9
{Name: "cpu/irq", Type: "gauge", Number: 0}, // ok
{Name: "cpu/softirq", Type: "gauge", Number: 0}, // ok 11
{Name: "cpu/steal", Type: "gauge", Number: 0}, // ok
{Name: "cpu/guest", Type: "gauge", Number: 0}, // ok 13
{Name: "cpu0/user", Type: "gauge", Number: 0.131529}, // ok
{Name: "cpu0/nice", Type: "gauge", Number: 0}, // ok 15
{Name: "cpu0/system", Type: "gauge", Number: 0.039388}, // ok
{Name: "cpu0/idle", Type: "gauge", Number: 99.819939}, // ok 17
{Name: "cpu0/iowait", Type: "gauge", Number: 0.009144},
{Name: "cpu0/irq", Type: "gauge", Number: 0}, // 19
{Name: "cpu0/softirq", Type: "gauge", Number: 0},
{Name: "cpu0/steal", Type: "gauge", Number: 0}, // 21
{Name: "cpu0/guest", Type: "gauge", Number: 0},
{Name: "cpu1/user", Type: "gauge", Number: 0.026707}, // 23
{Name: "cpu1/nice", Type: "gauge", Number: 0},
{Name: "cpu1/system", Type: "gauge", Number: 0.023193}, // 25
{Name: "cpu1/idle", Type: "gauge", Number: 99.950100},
{Name: "cpu1/iowait", Type: "gauge", Number: 0}, // 27
{Name: "cpu1/irq", Type: "gauge", Number: 0},
{Name: "cpu1/softirq", Type: "gauge", Number: 0}, // 29
{Name: "cpu1/steal", Type: "gauge", Number: 0},
{Name: "cpu1/guest", Type: "gauge", Number: 0}, // ok 31
{Name: "cpu-ext/intr", Type: "counter", Number: 39276666}, // ok
{Name: "cpu-ext/ctxt", Type: "counter", Number: 122631533}, // ok 33
{Name: "cpu-ext/processes", Type: "counter", Number: 227521}, // ok
{Name: "cpu-ext/procs_running", Type: "gauge", Number: 2}, // ok 35
{Name: "cpu-ext/procs_blocked", Type: "gauge", Number: 0}, // ok
// Third input.
{Name: "cpu/user", Type: "gauge", Number: 0.038309}, // ok 37
{Name: "cpu/nice", Type: "gauge", Number: 0}, // ok
{Name: "cpu/system", Type: "gauge", Number: 0.017681}, // ok 39
{Name: "cpu/idle", Type: "gauge", Number: 99.941063},
{Name: "cpu/iowait", Type: "gauge", Number: 0.002947}, // 41
{Name: "cpu/irq", Type: "gauge", Number: 0},
{Name: "cpu/softirq", Type: "gauge", Number: 0}, // 43
{Name: "cpu/steal", Type: "gauge", Number: 0},
{Name: "cpu/guest", Type: "gauge", Number: 0}, // 45
{Name: "cpu0/user", Type: "gauge", Number: 0.122230},
{Name: "cpu0/nice", Type: "gauge", Number: 0}, // 47
{Name: "cpu0/system", Type: "gauge", Number: 0.041815},
{Name: "cpu0/idle", Type: "gauge", Number: 99.824161}, // 49
{Name: "cpu0/iowait", Type: "gauge", Number: 0.011794},
{Name: "cpu0/irq", Type: "gauge", Number: 0}, // 51
{Name: "cpu0/softirq", Type: "gauge", Number: 0},
{Name: "cpu0/steal", Type: "gauge", Number: 0}, // 53
{Name: "cpu0/guest", Type: "gauge", Number: 0},
{Name: "cpu1/user", Type: "gauge", Number: 0.021426}, // ok 55
{Name: "cpu1/nice", Type: "gauge", Number: 0}, // ok
{Name: "cpu1/system", Type: "gauge", Number: 0.024640}, // -- 57
{Name: "cpu1/idle", Type: "gauge", Number: 99.953935},
{Name: "cpu1/iowait", Type: "gauge", Number: 0}, // 59
{Name: "cpu1/irq", Type: "gauge", Number: 0},
{Name: "cpu1/softirq", Type: "gauge", Number: 0}, // 61
{Name: "cpu1/steal", Type: "gauge", Number: 0},
{Name: "cpu1/guest", Type: "gauge", Number: 0}, // 63
{Name: "cpu-ext/intr", Type: "counter", Number: 39312673}, // ok
{Name: "cpu-ext/ctxt", Type: "counter", Number: 122742465}, // 65 ok
{Name: "cpu-ext/processes", Type: "counter", Number: 227717}, // ok
{Name: "cpu-ext/procs_running", Type: "gauge", Number: 3}, // 67 ok
{Name: "cpu-ext/procs_blocked", Type: "gauge", Number: 0}, // ok
}
if same, diff := test.IsDeeply(metrics, expect); !same {
test.Dump(metrics)
t.Error(diff)
}
}
func (s *AgentTestSuite) TestStatus(t *C) {
// This is what the API would send:
statusCmd := &proto.Cmd{
Ts: time.Now(),
User: "******",
Cmd: "Status",
}
s.sendChan <- statusCmd
got := test.WaitStatusReply(s.recvChan)
t.Assert(got, NotNil)
expectStatus := map[string]string{
"agent": "Idle",
}
if ok, diff := test.IsDeeply(got, expectStatus); !ok {
test.Dump(got)
t.Error(diff)
}
// We asked for all status, so we should get mm too.
_, ok := got["mm"]
t.Check(ok, Equals, true)
/**
* Get only agent's status
*/
statusCmd = &proto.Cmd{
Ts: time.Now(),
User: "******",
Cmd: "Status",
Service: "agent",
}
s.sendChan <- statusCmd
got = test.WaitStatusReply(s.recvChan)
t.Assert(got, NotNil)
// Only asked for agent, so we shouldn't get mm.
_, ok = got["mm"]
t.Check(ok, Equals, false)
/**
* Get only sub-service status.
*/
statusCmd = &proto.Cmd{
Ts: time.Now(),
User: "******",
Cmd: "Status",
Service: "mm",
}
s.sendChan <- statusCmd
got = test.WaitStatusReply(s.recvChan)
t.Assert(got, NotNil)
// Asked for mm, so we get it.
_, ok = got["mm"]
t.Check(ok, Equals, true)
// Didn't ask for all or agent, so we don't get it.
_, ok = got["agent"]
t.Check(ok, Equals, false)
}
func (s *ManagerTestSuite) TestGetConfig(t *C) {
m := data.NewManager(s.logger, s.dataDir, s.trashDir, "localhost", s.client)
t.Assert(m, NotNil)
config := &data.Config{
Encoding: "",
SendInterval: 1,
Limits: proto.DataSpoolLimits{
MaxAge: data.DEFAULT_DATA_MAX_AGE,
MaxSize: data.DEFAULT_DATA_MAX_SIZE,
MaxFiles: data.DEFAULT_DATA_MAX_FILES,
},
}
bytes, _ := json.Marshal(config)
// Write config to disk because manager reads it on start,
// else it uses default config.
pct.Basedir.WriteConfig("data", config)
err := m.Start()
t.Assert(err, IsNil)
sender := m.Sender()
t.Check(sender, NotNil)
/**
* GetConfig
*/
cmd := &proto.Cmd{
User: "******",
Service: "data",
Cmd: "GetConfig",
}
reply := m.Handle(cmd)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
gotConfig := []proto.AgentConfig{}
if err := json.Unmarshal(reply.Data, &gotConfig); err != nil {
t.Fatal(err)
}
expectConfig := []proto.AgentConfig{
{
InternalService: "data",
Config: string(bytes),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfig, expectConfig); !same {
test.Dump(gotConfig)
t.Error(diff)
}
err = m.Stop()
t.Assert(err, IsNil)
if !test.WaitStatus(5, m, "data", "Stopped") {
t.Fatal("test.WaitStatus() timeout")
}
status := m.Status()
t.Check(status["data-spooler"], Equals, "Stopped")
t.Check(status["data-sender"], Equals, "Stopped")
// Config should report Running: false.
reply = m.Handle(cmd)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
if err := json.Unmarshal(reply.Data, &gotConfig); err != nil {
t.Fatal(err)
}
expectConfig[0].Running = false
if same, diff := test.IsDeeply(gotConfig, expectConfig); !same {
test.Dump(gotConfig)
t.Error(diff)
}
}
func (s *ManagerTestSuite) TestGetConfig(t *C) {
m := mm.NewManager(s.logger, s.factory, s.clock, s.spool, s.im)
t.Assert(m, NotNil)
err := m.Start()
t.Assert(err, IsNil)
/**
* Start a mock MySQL monitor.
*/
mysqlMonitorConfig := &mysql.Config{
Config: mm.Config{
ServiceInstance: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Collect: 1,
Report: 60,
},
Status: map[string]string{
"threads_connected": "gauge",
"threads_running": "gauge",
},
}
mysqlData, err := json.Marshal(mysqlMonitorConfig)
t.Assert(err, IsNil)
cmd := &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "StartService",
Data: mysqlData,
}
s.mysqlMonitor.SetConfig(mysqlMonitorConfig)
reply := m.Handle(cmd)
t.Assert(reply, NotNil)
t.Assert(reply.Error, Equals, "")
/**
* Start a mock system monitor.
*/
systemMonitorConfig := &system.Config{
Config: mm.Config{
ServiceInstance: proto.ServiceInstance{
Service: "server",
InstanceId: 1,
},
Collect: 10,
Report: 60,
},
}
systemData, err := json.Marshal(systemMonitorConfig)
t.Assert(err, IsNil)
cmd = &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "StartService",
Data: systemData,
}
s.systemMonitor.SetConfig(systemMonitorConfig)
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Assert(reply.Error, Equals, "")
/**
* GetConfig from mm which should return all monitors' configs.
*/
cmd = &proto.Cmd{
Cmd: "GetConfig",
Service: "mm",
}
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
gotConfig := []proto.AgentConfig{}
if err := json.Unmarshal(reply.Data, &gotConfig); err != nil {
t.Fatal(err)
}
expectConfig := []proto.AgentConfig{
{
InternalService: "mm",
ExternalService: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Config: string(mysqlData),
Running: true,
},
{
InternalService: "mm",
ExternalService: proto.ServiceInstance{
Service: "server",
InstanceId: 1,
},
Config: string(systemData),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfig, expectConfig); !same {
test.Dump(gotConfig)
t.Error(diff)
}
}
/**
* Tests:
* - starting monitor
* - stopping monitor
* - starting monitor again (restarting monitor)
* - sneaked in:) unknown cmd test
*/
func (s *ManagerTestSuite) TestRestartMonitor(t *C) {
// Create and start mm, no monitors yet.
m := mm.NewManager(s.logger, s.factory, s.clock, s.spool, s.im)
t.Assert(m, NotNil)
err := m.Start()
t.Assert(err, IsNil)
// Start a monitor by sending StartService + monitor config.
// This is the config in test/mm/config/mm-mysql-1.conf.
mmConfig := &mysql.Config{
Config: mm.Config{
ServiceInstance: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Collect: 1,
Report: 60,
},
Status: map[string]string{
"threads_connected": "gauge",
"threads_running": "gauge",
},
}
mmConfigData, err := json.Marshal(mmConfig)
t.Assert(err, IsNil)
// If this were a real monitor, it would decode and set its own config.
// The mock monitor doesn't have any real config type, so we set it manually.
s.mysqlMonitor.SetConfig(mmConfig)
// The agent calls mm.Handle() with the cmd (for logging and status) and the config data.
cmd := &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "StartService",
Data: mmConfigData,
}
reply := m.Handle(cmd)
t.Assert(reply, NotNil)
t.Check(reply.Error, Equals, "")
// The monitor should be running. The mock monitor returns "Running" if
// Start() has been called; else it returns "Stopped".
status := m.Status()
t.Check(status["monitor"], Equals, "Running")
// There should be a 1s collect ticker for the monitor.
if ok, diff := test.IsDeeply(s.clock.Added, []uint{1}); !ok {
t.Errorf("Make 1s ticker for collect interval\n%s", diff)
}
// After starting a monitor, mm should write its config to the dir
// it learned when mm.LoadConfig() was called. Next time agent starts,
// it will have mm start the monitor with this config.
data, err := ioutil.ReadFile(s.configDir + "/mm-mysql-1.conf")
t.Check(err, IsNil)
gotConfig := &mysql.Config{}
err = json.Unmarshal(data, gotConfig)
t.Check(err, IsNil)
if same, diff := test.IsDeeply(gotConfig, mmConfig); !same {
test.Dump(gotConfig)
t.Error(diff)
}
/**
* Stop the monitor.
*/
cmd = &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "StopService",
Data: mmConfigData,
}
// Handles StopService without error.
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Check(reply.Error, Equals, "")
// Stop a monitor removes it from the managers list of monitors.
// So it's no longer present in a status request.
status = m.Status()
t.Check(status["monitor"], Equals, "")
// After stopping the monitor, the manager should remove its tickChan.
if len(s.clock.Removed) != 1 {
t.Error("Remove's monitor's tickChan from clock")
}
// After stopping a monitor, mm should remove its config file so agent
// doesn't start it on restart.
file := s.configDir + "/mm-mysql-1.conf"
if pct.FileExists(file) {
t.Error("Stopping monitor removes its config; ", file, " exists")
}
/**
* Start the monitor again (restarting monitor).
*/
cmd = &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "StartService",
Data: mmConfigData,
}
// If this were a real monitor, it would decode and set its own config.
// The mock monitor doesn't have any real config type, so we set it manually.
s.mysqlMonitor.SetConfig(mmConfig)
// The agent calls mm.Handle() with the cmd (for logging and status) and the config data.
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Check(reply.Error, Equals, "")
// The monitor should be running. The mock monitor returns "Running" if
// Start() has been called; else it returns "Stopped".
status = m.Status()
t.Check(status["monitor"], Equals, "Running")
// There should be a 1s collect ticker for the monitor.
// (Actually two in s.clock.Added, as this is mock and we started monitor twice)
if ok, diff := test.IsDeeply(s.clock.Added, []uint{1, 1}); !ok {
t.Errorf("Make 1s ticker for collect interval\n%s", diff)
}
// After starting a monitor, mm should write its config to the dir
// it learned when mm.LoadConfig() was called. Next time agent starts,
// it will have mm start the monitor with this config.
data, err = ioutil.ReadFile(s.configDir + "/mm-mysql-1.conf")
t.Check(err, IsNil)
gotConfig = &mysql.Config{}
err = json.Unmarshal(data, gotConfig)
t.Check(err, IsNil)
if same, diff := test.IsDeeply(gotConfig, mmConfig); !same {
t.Logf("%+v", gotConfig)
t.Error(diff)
}
/**
* While we're all setup and working, let's sneak in an unknown cmd test.
*/
cmd = &proto.Cmd{
User: "******",
Service: "mm",
Cmd: "Pontificate",
Data: mmConfigData,
}
// Unknown cmd causes error.
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Check(reply.Error, Not(Equals), "")
}
func (s *ManagerTestSuite) TestSetConfig(t *C) {
m := data.NewManager(s.logger, s.dataDir, s.trashDir, "localhost", s.client)
t.Assert(m, NotNil)
config := &data.Config{
Encoding: "",
SendInterval: 1,
Limits: proto.DataSpoolLimits{
MaxAge: data.DEFAULT_DATA_MAX_AGE,
MaxSize: data.DEFAULT_DATA_MAX_SIZE,
MaxFiles: data.DEFAULT_DATA_MAX_FILES,
},
}
pct.Basedir.WriteConfig("data", config)
err := m.Start()
t.Assert(err, IsNil)
sender := m.Sender()
t.Check(sender, NotNil)
/**
* Change SendInterval
*/
config.SendInterval = 5
configData, err := json.Marshal(config)
t.Assert(err, IsNil)
cmd := &proto.Cmd{
User: "******",
Service: "data",
Cmd: "SetConfig",
Data: configData,
}
gotReply := m.Handle(cmd)
t.Assert(gotReply.Error, Equals, "")
cmd = &proto.Cmd{
User: "******",
Service: "data",
Cmd: "GetConfig",
}
reply := m.Handle(cmd)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
gotConfigRes := []proto.AgentConfig{}
if err := json.Unmarshal(reply.Data, &gotConfigRes); err != nil {
t.Fatal(err)
}
expectConfigRes := []proto.AgentConfig{
{
InternalService: "data",
Config: string(configData),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfigRes, expectConfigRes); !same {
test.Dump(gotConfigRes)
t.Error(diff)
}
// Verify new config on disk.
content, err := ioutil.ReadFile(pct.Basedir.ConfigFile("data"))
t.Assert(err, IsNil)
gotConfig := &data.Config{}
if err := json.Unmarshal(content, gotConfig); err != nil {
t.Fatal(err)
}
if same, diff := test.IsDeeply(gotConfig, config); !same {
test.Dump(gotConfig)
t.Error(diff)
}
/**
* Change Encoding
*/
config.Encoding = "gzip"
configData, err = json.Marshal(config)
t.Assert(err, IsNil)
cmd = &proto.Cmd{
User: "******",
Service: "data",
Cmd: "SetConfig",
Data: configData,
}
gotReply = m.Handle(cmd)
t.Assert(gotReply.Error, Equals, "")
cmd = &proto.Cmd{
User: "******",
Service: "data",
Cmd: "GetConfig",
}
reply = m.Handle(cmd)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
if err := json.Unmarshal(reply.Data, &gotConfigRes); err != nil {
t.Fatal(err)
}
expectConfigRes = []proto.AgentConfig{
{
InternalService: "data",
Config: string(configData),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfigRes, expectConfigRes); !same {
test.Dump(gotConfigRes)
t.Error(diff)
}
// Verify new config on disk.
content, err = ioutil.ReadFile(pct.Basedir.ConfigFile("data"))
t.Assert(err, IsNil)
gotConfig = &data.Config{}
if err := json.Unmarshal(content, gotConfig); err != nil {
t.Fatal(err)
}
if same, diff := test.IsDeeply(gotConfig, config); !same {
test.Dump(gotConfig)
t.Error(diff)
}
}
func (s *ManagerTestSuite) TestGetConfig(t *C) {
m := sysconfig.NewManager(s.logger, s.factory, s.clock, s.spool, s.im)
t.Assert(m, NotNil)
err := m.Start()
t.Assert(err, IsNil)
// Start a sysconfig monitor.
sysconfigConfig := &mysql.Config{
Config: sysconfig.Config{
ServiceInstance: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Report: 3600,
},
}
sysconfigConfigData, err := json.Marshal(sysconfigConfig)
t.Assert(err, IsNil)
cmd := &proto.Cmd{
User: "******",
Service: "sysconfig",
Cmd: "StartService",
Data: sysconfigConfigData,
}
reply := m.Handle(cmd)
t.Assert(reply, NotNil)
t.Check(reply.Error, Equals, "")
s.mockMonitor.SetConfig(sysconfigConfig)
/**
* GetConfig from sysconfig which should return all monitors' configs.
*/
cmd = &proto.Cmd{
Cmd: "GetConfig",
Service: "sysconfig",
}
reply = m.Handle(cmd)
t.Assert(reply, NotNil)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
gotConfig := []proto.AgentConfig{}
if err := json.Unmarshal(reply.Data, &gotConfig); err != nil {
t.Fatal(err)
}
expectConfig := []proto.AgentConfig{
{
InternalService: "sysconfig",
ExternalService: proto.ServiceInstance{
Service: "mysql",
InstanceId: 1,
},
Config: string(sysconfigConfigData),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfig, expectConfig); !same {
test.Dump(gotConfig)
t.Error(diff)
}
}
func (s *ManagerTestSuite) TestLogService(t *C) {
config := &log.Config{
File: s.logFile,
Level: "info",
}
pct.Basedir.WriteConfig("log", config)
m := log.NewManager(s.client, s.logChan)
err := m.Start()
t.Assert(err, IsNil)
relay := m.Relay()
t.Assert(relay, NotNil)
logger := pct.NewLogger(relay.LogChan(), "log-svc-test")
logger.Info("i'm a log entry")
// Log entry should be sent to API.
got := test.WaitLog(s.recvChan, 3)
if len(got) == 0 {
t.Fatal("No log entries")
}
var gotLog proto.LogEntry
for _, l := range got {
if l.Service == "log-svc-test" {
gotLog = l
break
}
}
t.Assert(gotLog, NotNil)
expectLog := proto.LogEntry{Ts: test.Ts, Level: proto.LOG_INFO, Service: "log-svc-test", Msg: "i'm a log entry"}
if same, diff := test.IsDeeply(gotLog, expectLog); !same {
t.Logf("%+v", got)
t.Error(diff)
}
// Since there's a log file, entry should be written to it too.
size, _ := test.FileSize(s.logFile)
test.WaitFileSize(s.logFile, size)
var content []byte
content, err = ioutil.ReadFile(s.logFile)
t.Assert(err, IsNil)
if !strings.Contains(string(content), "i'm a log entry") {
t.Error("Writes log entry to log file, got\n", string(content))
}
// Can't stop log service, but Stop() should work and not return error.
err = m.Stop()
t.Assert(err, IsNil)
/**
* Change log level and file
*/
newLogFile := s.logFile + "-2"
defer os.Remove(newLogFile)
config = &log.Config{
File: newLogFile,
Level: "warning",
}
configData, err := json.Marshal(config)
t.Assert(err, IsNil)
cmd := &proto.Cmd{
User: "******",
Service: "log",
Cmd: "SetConfig",
Data: configData,
}
gotReply := m.Handle(cmd)
expectReply := cmd.Reply(config)
if same, diff := test.IsDeeply(gotReply, expectReply); !same {
t.Logf("%+v", gotReply)
t.Error(diff)
}
// Log entry should NOT be sent to API if log level was really changed.
logger.Info("i'm lost")
got = test.WaitLog(s.recvChan, 3)
if len(got) != 0 {
t.Logf("%+v", got)
t.Error("Log level changed dynamically")
}
logger.Warn("blah")
got = test.WaitLog(s.recvChan, 3)
gotLog = proto.LogEntry{}
for _, l := range got {
if l.Service == "log-svc-test" {
gotLog = l
break
}
}
expectLog = proto.LogEntry{Ts: test.Ts, Level: proto.LOG_WARNING, Service: "log-svc-test", Msg: "blah"}
if same, diff := test.IsDeeply(gotLog, expectLog); !same {
t.Logf("%+v", got)
t.Error(diff)
}
// Entry should be written to new log file if it was really changed.
size, _ = test.FileSize(newLogFile)
test.WaitFileSize(newLogFile, size)
content, err = ioutil.ReadFile(newLogFile)
t.Assert(err, IsNil)
if !strings.Contains(string(content), "blah") {
t.Error("Log file changed dynamically, got\n", string(content))
}
// Verify new log config on disk.
data, err := ioutil.ReadFile(pct.Basedir.ConfigFile("log"))
t.Assert(err, IsNil)
gotConfig := &log.Config{}
if err := json.Unmarshal(data, gotConfig); err != nil {
t.Fatal(err)
}
if same, diff := test.IsDeeply(gotConfig, config); !same {
test.Dump(gotConfig)
t.Error(diff)
}
/**
* GetConfig
*/
cmd = &proto.Cmd{
User: "******",
Service: "log",
Cmd: "GetConfig",
}
reply := m.Handle(cmd)
t.Assert(reply.Error, Equals, "")
t.Assert(reply.Data, NotNil)
gotConfigRes := []proto.AgentConfig{}
if err := json.Unmarshal(reply.Data, &gotConfigRes); err != nil {
t.Fatal(err)
}
expectConfigRes := []proto.AgentConfig{
{
InternalService: "log",
Config: string(configData),
Running: true,
},
}
if same, diff := test.IsDeeply(gotConfigRes, expectConfigRes); !same {
test.Dump(gotConfigRes)
t.Error(diff)
}
/**
* Status (internal status of log and relay)
*/
status := m.Status()
t.Check(status["ws"], Equals, "Connected")
t.Check(status["log-file"], Equals, newLogFile)
t.Check(status["log-level"], Equals, "warning")
}
