func (s *AgentTestSuite) TestKeepalive(t *C) {
	// Agent should be sending a Pong every 1s now which is sent as a
	// reply to no cmd (it's a platypus).
	<-time.After(2 * time.Second)
	reply := test.WaitReply(s.recvChan)
	if len(reply) < 1 {
		t.Fatal("No Pong recieved")
	}
	t.Check(reply[0].Cmd, Equals, "Pong")

	// Disconnect and keepalives should stop.
	connectChan := make(chan bool)
	s.client.SetConnectChan(connectChan)
	defer s.client.SetConnectChan(nil)
	s.client.Disconnect()
	<-connectChan

	<-time.After(2 * time.Second)
	reply = test.WaitReply(s.recvChan)
	t.Check(reply, HasLen, 0)

	// Let agent reconnect and keepalives should resume.
	connectChan <- true
	<-time.After(2 * time.Second)
	reply = test.WaitReply(s.recvChan)
	if len(reply) < 1 {
		t.Fatal("No Pong recieved after reconnect")
	}
	t.Check(reply[0].Cmd, Equals, "Pong")
}
func (s *AgentTestSuite) TestStartStopUnknownService(t *C) {
	// Starting an unknown service should return an error.
	serviceCmd := &proto.ServiceData{
		Name: "foo",
	}
	serviceData, _ := json.Marshal(serviceCmd)
	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Service: "agent",
		Cmd:     "StartService",
		Data:    serviceData,
	}

	s.sendChan <- cmd
	gotReplies := test.WaitReply(s.recvChan)
	t.Assert(len(gotReplies), Equals, 1)
	t.Check(gotReplies[0].Cmd, Equals, "StartService")
	t.Check(gotReplies[0].Error, Not(Equals), "")

	// Stopp an unknown service should return an error.
	cmd = &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Service: "agent",
		Cmd:     "StopService",
		Data:    serviceData,
	}

	s.sendChan <- cmd
	gotReplies = test.WaitReply(s.recvChan)
	t.Assert(len(gotReplies), Equals, 1)
	t.Check(gotReplies[0].Cmd, Equals, "StopService")
	t.Check(gotReplies[0].Error, Not(Equals), "")
}
func (s *AgentTestSuite) TestStartServiceSlow(t *C) {
	// This test is like TestStartService but simulates a slow starting service.

	qanConfig := &qan.Config{
		Interval:          60,         // seconds
		MaxSlowLogSize:    1073741824, // 1 GiB
		RemoveOldSlowLogs: true,
		ExampleQueries:    true,
		MaxWorkers:        2,
		WorkerRunTime:     120, // seconds
	}
	qanConfigData, _ := json.Marshal(qanConfig)
	serviceCmd := &proto.ServiceData{
		Name:   "qan",
		Config: qanConfigData,
	}
	serviceData, _ := json.Marshal(serviceCmd)
	now := time.Now()
	cmd := &proto.Cmd{
		Ts:      now,
		User:    "******",
		Service: "agent",
		Cmd:     "StartService",
		Data:    serviceData,
	}

	// Send the cmd to the client, tell the agent to stop, then wait for it.
	s.sendChan <- cmd

	// No replies yet.
	gotReplies := test.WaitReply(s.recvChan)
	if len(gotReplies) != 0 {
		t.Fatal("No reply before StartService")
	}

	// Agent should be able to reply on status chan, indicating that it's
	// still starting the service.
	gotStatus := test.GetStatus(s.sendChan, s.recvChan)
	if !t.Check(gotStatus["agent"], Equals, "Idle") {
		test.Dump(gotStatus)
	}

	// Make it seem like service has started now.
	s.readyChan <- true

	// Agent sends reply: no error.
	gotReplies = test.WaitReply(s.recvChan)
	if len(gotReplies) == 0 {
		t.Fatal("Get reply")
	}
	if len(gotReplies) > 1 {
		t.Errorf("One reply, got %+v", gotReplies)
	}

	reply := &proto.Reply{}
	_ = json.Unmarshal(gotReplies[0].Data, reply)
	t.Check(reply.Error, Equals, "")
}
func (s *AgentTestSuite) TestGetConfig(t *C) {
	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "GetConfig",
		Service: "agent",
	}
	s.sendChan <- cmd

	got := test.WaitReply(s.recvChan)
	t.Assert(len(got), Equals, 1)
	gotConfig := []proto.AgentConfig{}
	if err := json.Unmarshal(got[0].Data, &gotConfig); err != nil {
		t.Fatal(err)
	}

	config := *s.config
	config.Links = nil
	bytes, _ := json.Marshal(config)
	expect := []proto.AgentConfig{
		{
			InternalService: "agent",
			Config:          string(bytes),
			Running:         true,
		},
	}

	if ok, diff := test.IsDeeply(gotConfig, expect); !ok {
		t.Logf("%+v", gotConfig)
		t.Error(diff)
	}
}
func (s *AgentTestSuite) TestGetVersion(t *C) {
	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "Version",
		Service: "agent",
	}
	s.sendChan <- cmd

	got := test.WaitReply(s.recvChan)
	t.Assert(len(got), Equals, 1)
	version := &proto.Version{}
	json.Unmarshal(got[0].Data, &version)
	t.Check(version.Running, Equals, agent.VERSION)
}
func (s *AgentTestSuite) TestCmdToService(t *C) {
	cmd := &proto.Cmd{
		Service: "mm",
		Cmd:     "Hello",
	}
	s.sendChan <- cmd

	reply := test.WaitReply(s.recvChan)
	t.Assert(reply, HasLen, 1)
	t.Check(reply[0].Error, Equals, "")
	t.Check(reply[0].Cmd, Equals, "Hello")

	t.Assert(s.services["mm"].Cmds, HasLen, 1)
	t.Check(s.services["mm"].Cmds[0].Cmd, Equals, "Hello")
}
func (s *AgentTestSuite) TestRestart(t *C) {
	// Stop the default agnet.  We need our own to check its return value.
	s.TearDownTest(t)

	cmdFactory := &mock.CmdFactory{}
	pctCmd.Factory = cmdFactory

	defer func() {
		os.Remove(pct.Basedir.File("start-lock"))
		os.Remove(pct.Basedir.File("start-script"))
	}()

	newAgent := agent.NewAgent(s.config, s.logger, s.api, s.client, s.servicesMap)
	doneChan := make(chan error, 1)
	go func() {
		doneChan <- newAgent.Run()
	}()

	cmd := &proto.Cmd{
		Service: "agent",
		Cmd:     "Restart",
	}
	s.sendChan <- cmd

	replies := test.WaitReply(s.recvChan)
	t.Assert(replies, HasLen, 1)
	t.Check(replies[0].Error, Equals, "")

	var err error
	select {
	case err = <-doneChan:
	case <-time.After(2 * time.Second):
		t.Fatal("Agent did not restart")
	}

	// Agent should return without an error.
	t.Check(err, IsNil)

	// Agent should create the start-lock file and start-script.
	t.Check(pct.FileExists(pct.Basedir.File("start-lock")), Equals, true)
	t.Check(pct.FileExists(pct.Basedir.File("start-script")), Equals, true)

	// Agent should make a command to run the start-script.
	t.Assert(cmdFactory.Cmds, HasLen, 1)
	t.Check(cmdFactory.Cmds[0].Name, Equals, pct.Basedir.File("start-script"))
	t.Check(cmdFactory.Cmds[0].Args, IsNil)
}
func (s *AgentTestSuite) TestGetAllConfigs(t *C) {
	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "GetAllConfigs",
		Service: "agent",
	}
	s.sendChan <- cmd

	got := test.WaitReply(s.recvChan)
	t.Assert(len(got), Equals, 1)
	reply := got[0]
	t.Check(reply.Error, Equals, "")
	t.Assert(reply.Data, Not(HasLen), 0)

	gotConfigs := []proto.AgentConfig{}
	err := json.Unmarshal(reply.Data, &gotConfigs)
	t.Assert(err, IsNil)

	bytes, _ := json.Marshal(s.config)

	sort.Sort(test.ByInternalService(gotConfigs))
	expectConfigs := []proto.AgentConfig{
		{
			InternalService: "agent",
			Config:          string(bytes),
			Running:         true,
		},
		{
			InternalService: "mm",
			Config:          `{"Foo":"bar"}`,
			Running:         false,
		},
		{
			InternalService: "qan",
			Config:          `{"Foo":"bar"}`,
			Running:         false,
		},
	}
	if ok, diff := test.IsDeeply(gotConfigs, expectConfigs); !ok {
		test.Dump(gotConfigs)
		t.Error(diff)
	}
}
func (s *AgentTestSuite) TestSetConfigApiHostname(t *C) {
	newConfig := *s.config
	newConfig.ApiHostname = "http://localhost"
	data, err := json.Marshal(newConfig)
	t.Assert(err, IsNil)

	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "SetConfig",
		Service: "agent",
		Data:    data,
	}
	s.sendChan <- cmd

	got := test.WaitReply(s.recvChan)
	t.Assert(len(got), Equals, 1)
	gotConfig := &agent.Config{}
	if err := json.Unmarshal(got[0].Data, gotConfig); err != nil {
		t.Fatal(err)
	}

	/**
	 * Verify new agent config in memory.
	 */
	expect := *s.config
	expect.ApiHostname = "http://localhost"
	expect.Links = nil
	if ok, diff := test.IsDeeply(gotConfig, &expect); !ok {
		t.Logf("%+v", gotConfig)
		t.Error(diff)
	}

	/**
	 * Verify new agent config in API connector.
	 */
	t.Check(s.api.Hostname(), Equals, "http://localhost")
	t.Check(s.api.ApiKey(), Equals, "789")

	/**
	 * Verify new agent config on disk.
	 */
	data, err = ioutil.ReadFile(s.configFile)
	t.Assert(err, IsNil)
	gotConfig = &agent.Config{}
	if err := json.Unmarshal(data, gotConfig); err != nil {
		t.Fatal(err)
	}
	if same, diff := test.IsDeeply(gotConfig, &expect); !same {
		// @todo: if expect is not ptr, IsDeeply dies with "got ptr, expected struct"
		t.Logf("%+v", gotConfig)
		t.Error(diff)
	}

	// After changing the API host, the agent's ws should NOT reconnect yet,
	// but status should show that its link has changed, so sending a Reconnect
	// cmd will cause agent to reconnect its ws.
	gotCalled := test.WaitTrace(s.client.TraceChan)
	expectCalled := []string{"Start", "Connect"}
	t.Check(gotCalled, DeepEquals, expectCalled)

	/**
	 * Test Reconnect here since it's usually done after changing ApiHostname/
	 */

	// There is NO reply after reconnect because we can't recv cmd on one connection
	// and reply on another.  Instead, we should see agent try to reconnect:
	connectChan := make(chan bool)
	s.client.SetConnectChan(connectChan)
	defer s.client.SetConnectChan(nil)

	cmd = &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "Reconnect",
		Service: "agent",
	}
	s.sendChan <- cmd

	// Wait for agent to reconnect.
	<-connectChan
	connectChan <- true

	gotCalled = test.WaitTrace(s.client.TraceChan)
	expectCalled = []string{"Disconnect", "Connect"}
	t.Check(gotCalled, DeepEquals, expectCalled)
}
func (s *AgentTestSuite) TestSetConfigApiKey(t *C) {
	newConfig := *s.config
	newConfig.ApiKey = "101"
	data, err := json.Marshal(newConfig)
	t.Assert(err, IsNil)

	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Cmd:     "SetConfig",
		Service: "agent",
		Data:    data,
	}
	s.sendChan <- cmd

	got := test.WaitReply(s.recvChan)
	t.Assert(len(got), Equals, 1)
	gotConfig := &agent.Config{}
	if err := json.Unmarshal(got[0].Data, gotConfig); err != nil {
		t.Fatal(err)
	}

	/**
	 * Verify new agent config in memory.
	 */
	expect := *s.config
	expect.ApiKey = "101"
	expect.Links = nil
	if ok, diff := test.IsDeeply(gotConfig, &expect); !ok {
		t.Logf("%+v", gotConfig)
		t.Error(diff)
	}

	/**
	 * Verify new agent config in API connector.
	 */
	t.Check(s.api.ApiKey(), Equals, "101")
	t.Check(s.api.Hostname(), Equals, agent.DEFAULT_API_HOSTNAME)

	/**
	 * Verify new agent config on disk.
	 */
	data, err = ioutil.ReadFile(s.configFile)
	t.Assert(err, IsNil)
	gotConfig = &agent.Config{}
	if err := json.Unmarshal(data, gotConfig); err != nil {
		t.Fatal(err)
	}
	if same, diff := test.IsDeeply(gotConfig, &expect); !same {
		// @todo: if expect is not ptr, IsDeeply dies with "got ptr, expected struct"
		t.Logf("%+v", gotConfig)
		t.Error(diff)
	}

	// After changing the API key, the agent's ws should NOT reconnect yet,
	// but status should show that its link has changed, so sending a Reconnect
	// cmd will cause agent to reconnect its ws.
	gotCalled := test.WaitTrace(s.client.TraceChan)
	expectCalled := []string{"Start", "Connect"}
	t.Check(gotCalled, DeepEquals, expectCalled)
}
func (s *AgentTestSuite) TestStartStopService(t *C) {
	// To start a service, first we make a config for the service:
	qanConfig := &qan.Config{
		Interval:          60,         // seconds
		MaxSlowLogSize:    1073741824, // 1 GiB
		RemoveOldSlowLogs: true,
		ExampleQueries:    true,
		MaxWorkers:        2,
		WorkerRunTime:     120, // seconds
	}

	// Second, the service config is encoded and encapsulated in a ServiceData:
	qanConfigData, _ := json.Marshal(qanConfig)
	serviceCmd := &proto.ServiceData{
		Name:   "qan",
		Config: qanConfigData,
	}

	// Third and final, the service data is encoded and encapsulated in a Cmd:
	serviceData, _ := json.Marshal(serviceCmd)
	cmd := &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Service: "agent",
		Cmd:     "StartService",
		Data:    serviceData,
	}

	// The readyChan is used by mock.MockServiceManager.Start() and Stop()
	// to simulate slow starts and stops.  We're not testing that here, so
	// this lets the service start immediately.
	s.readyChan <- true

	// Send the StartService cmd to the client, then wait for the reply
	// which should not have an error, indicating success.
	s.sendChan <- cmd
	gotReplies := test.WaitReply(s.recvChan)
	if len(gotReplies) != 1 {
		t.Fatal("Got Reply to Cmd:StartService")
	}
	reply := &proto.Reply{}
	_ = json.Unmarshal(gotReplies[0].Data, reply)
	if reply.Error != "" {
		t.Error("No Reply.Error to Cmd:StartService; got ", reply.Error)
	}

	// To double-check that the agent started without error, get its status
	// which should show everything is "Ready" or "Idle".
	status := test.GetStatus(s.sendChan, s.recvChan)
	expectStatus := map[string]string{
		"agent": "Idle",
		"qan":   "Ready",
		"mm":    "",
	}
	if same, diff := test.IsDeeply(status, expectStatus); !same {
		t.Error(diff)
	}

	// Finally, since we're using mock objects, let's double check the
	// execution trace, i.e. what calls the agent made based on all
	// the previous ^.
	got := test.WaitTrace(s.traceChan)
	expect := []string{
		`Start qan`,
		`Status qan`,
		`Status mm`,
	}
	t.Check(got, DeepEquals, expect)

	/**
	 * Stop the service.
	 */

	serviceCmd = &proto.ServiceData{
		Name: "qan",
	}
	serviceData, _ = json.Marshal(serviceCmd)
	cmd = &proto.Cmd{
		Ts:      time.Now(),
		User:    "******",
		Service: "agent",
		Cmd:     "StopService",
		Data:    serviceData,
	}

	// Let fake qan service stop immediately.
	s.readyChan <- true

	s.sendChan <- cmd
	gotReplies = test.WaitReply(s.recvChan)
	if len(gotReplies) != 1 {
		t.Fatal("Got Reply to Cmd:StopService")
	}
	reply = &proto.Reply{}
	_ = json.Unmarshal(gotReplies[0].Data, reply)
	if reply.Error != "" {
		t.Error("No Reply.Error to Cmd:StopService; got ", reply.Error)
	}

	status = test.GetStatus(s.sendChan, s.recvChan)
	t.Check(status["agent"], Equals, "Idle")
	t.Check(status["qan"], Equals, "Stopped")
	t.Check(status["mm"], Equals, "")
}