func TestServer_JoinLAN(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, s2 := testServer(t)
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Check the members
testutil.WaitForResult(func() (bool, error) {
return len(s1.LANMembers()) == 2, nil
}, func(err error) {
t.Fatalf("bad len")
})
testutil.WaitForResult(func() (bool, error) {
return len(s2.LANMembers()) == 2, nil
}, func(err error) {
t.Fatalf("bad len")
})
}
func TestClient_JoinLAN(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Check the members
testutil.WaitForResult(func() (bool, error) {
server_check := len(s1.LANMembers()) == 2
client_check := len(c1.LANMembers()) == 2
return server_check && client_check, nil
}, func(err error) {
t.Fatalf("bad len")
})
// Check we have a new consul
testutil.WaitForResult(func() (bool, error) {
return len(c1.consuls) == 1, nil
}, func(err error) {
t.Fatalf("expected consul server")
})
}
func TestServer_JoinLAN_TLS(t *testing.T) {
dir1, conf1 := testServerConfig(t, "a.testco.internal")
conf1.VerifyIncoming = true
conf1.VerifyOutgoing = true
configureTLS(conf1)
s1, err := NewServer(conf1)
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, conf2 := testServerConfig(t, "b.testco.internal")
conf2.Bootstrap = false
conf2.VerifyIncoming = true
conf2.VerifyOutgoing = true
configureTLS(conf2)
s2, err := NewServer(conf2)
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Check the members
testutil.WaitForResult(func() (bool, error) {
return len(s1.LANMembers()) == 2, nil
}, func(err error) {
t.Fatalf("bad len")
})
testutil.WaitForResult(func() (bool, error) {
return len(s2.LANMembers()) == 2, nil
}, func(err error) {
t.Fatalf("bad len")
})
// Verify Raft has established a peer
testutil.WaitForResult(func() (bool, error) {
return s1.Stats()["raft"]["num_peers"] == "1", nil
}, func(err error) {
t.Fatalf("no peer established")
})
testutil.WaitForResult(func() (bool, error) {
return s2.Stats()["raft"]["num_peers"] == "1", nil
}, func(err error) {
t.Fatalf("no peer established")
})
}
func TestServer_Leave(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
// Second server not in bootstrap mode
dir2, s2 := testServerDCBootstrap(t, "dc1", false)
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
var p1 []string
var p2 []string
testutil.WaitForResult(func() (bool, error) {
p1, _ = s1.raftPeers.Peers()
return len(p1) == 2, errors.New(fmt.Sprintf("%v", p1))
}, func(err error) {
t.Fatalf("should have 2 peers: %v", err)
})
testutil.WaitForResult(func() (bool, error) {
p2, _ = s2.raftPeers.Peers()
return len(p2) == 2, errors.New(fmt.Sprintf("%v", p1))
}, func(err error) {
t.Fatalf("should have 2 peers: %v", err)
})
// Issue a leave to the non-leader
for _, s := range []*Server{s1, s2} {
if s.IsLeader() {
continue
}
if err := s.Leave(); err != nil {
t.Fatalf("err: %v", err)
}
}
// Should lose a peer
for _, s := range []*Server{s1, s2} {
testutil.WaitForResult(func() (bool, error) {
p1, _ = s.raftPeers.Peers()
return len(p1) == 1, nil
}, func(err error) {
t.Fatalf("should have 1 peer: %v", p1)
})
}
}
func TestLeader_LeftServer(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, s2 := testServerDCBootstrap(t, "dc1", false)
defer os.RemoveAll(dir2)
defer s2.Shutdown()
dir3, s3 := testServerDCBootstrap(t, "dc1", false)
defer os.RemoveAll(dir3)
defer s3.Shutdown()
servers := []*Server{s1, s2, s3}
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
if _, err := s3.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
for _, s := range servers {
testutil.WaitForResult(func() (bool, error) {
peers, _ := s.raftPeers.Peers()
return len(peers) == 3, nil
}, func(err error) {
t.Fatalf("should have 3 peers")
})
}
testutil.WaitForResult(func() (bool, error) {
// Kill any server
servers[0].Shutdown()
// Force remove the non-leader (transition to left state)
if err := servers[1].RemoveFailedNode(servers[0].config.NodeName); err != nil {
t.Fatalf("err: %v", err)
}
for _, s := range servers[1:] {
peers, _ := s.raftPeers.Peers()
return len(peers) == 2, errors.New(fmt.Sprintf("%v", peers))
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestLeader_FailedMember(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
client := rpcClient(t, s1)
testutil.WaitForLeader(t, client.Call, "dc1")
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Fail the member
c1.Shutdown()
// Should be registered
state := s1.fsm.State()
testutil.WaitForResult(func() (bool, error) {
_, found, _ := state.GetNode(c1.config.NodeName)
return found == true, nil
}, func(err error) {
t.Fatalf("client not registered")
})
// Should have a check
_, checks := state.NodeChecks(c1.config.NodeName)
if len(checks) != 1 {
t.Fatalf("client missing check")
}
if checks[0].CheckID != SerfCheckID {
t.Fatalf("bad check: %v", checks[0])
}
if checks[0].Name != SerfCheckName {
t.Fatalf("bad check: %v", checks[0])
}
testutil.WaitForResult(func() (bool, error) {
_, checks = state.NodeChecks(c1.config.NodeName)
return checks[0].Status == structs.HealthCritical, errors.New(checks[0].Status)
}, func(err error) {
t.Fatalf("check status is %v, should be critical", err)
})
}
func TestHTTPAgentJoin_WAN(t *testing.T) {
dir, srv := makeHTTPServer(t)
defer os.RemoveAll(dir)
defer srv.Shutdown()
defer srv.agent.Shutdown()
dir2, a2 := makeAgent(t, nextConfig())
defer os.RemoveAll(dir2)
defer a2.Shutdown()
addr := fmt.Sprintf("127.0.0.1:%d", a2.config.Ports.SerfWan)
req, err := http.NewRequest("GET", fmt.Sprintf("/v1/agent/join/%s?wan=true", addr), nil)
if err != nil {
t.Fatalf("err: %v", err)
}
obj, err := srv.AgentJoin(nil, req)
if err != nil {
t.Fatalf("Err: %v", err)
}
if obj != nil {
t.Fatalf("Err: %v", obj)
}
testutil.WaitForResult(func() (bool, error) {
return len(a2.WANMembers()) == 2, nil
}, func(err error) {
t.Fatalf("should have 2 members")
})
}
func TestHealthChecksInState(t *testing.T) {
httpTest(t, func(srv *HTTPServer) {
req, err := http.NewRequest("GET", "/v1/health/state/passing?dc=dc1", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
resp := httptest.NewRecorder()
obj, err := srv.HealthChecksInState(resp, req)
if err != nil {
return false, err
}
if err := checkIndex(resp); err != nil {
return false, err
}
// Should be 1 health check for the server
nodes := obj.(structs.HealthChecks)
if len(nodes) != 1 {
return false, fmt.Errorf("bad: %v", obj)
}
return true, nil
}, func(err error) { t.Fatalf("err: %v", err) })
})
}
func TestRPCClientForceLeave(t *testing.T) {
p1 := testRPCClient(t)
p2 := testRPCClient(t)
defer p1.Close()
defer p2.Close()
s2Addr := fmt.Sprintf("127.0.0.1:%d", p2.agent.config.Ports.SerfLan)
if _, err := p1.agent.JoinLAN([]string{s2Addr}); err != nil {
t.Fatalf("err: %s", err)
}
if err := p2.agent.Shutdown(); err != nil {
t.Fatalf("err: %s", err)
}
if err := p1.client.ForceLeave(p2.agent.config.NodeName); err != nil {
t.Fatalf("err: %s", err)
}
m := p1.agent.LANMembers()
if len(m) != 2 {
t.Fatalf("should have 2 members: %#v", m)
}
testutil.WaitForResult(func() (bool, error) {
m := p1.agent.LANMembers()
success := m[1].Status == serf.StatusLeft
return success, errors.New(m[1].Status.String())
}, func(err error) {
t.Fatalf("member status is %v, should be left", err)
})
}
func TestEventList(t *testing.T) {
httpTest(t, func(srv *HTTPServer) {
p := &UserEvent{Name: "test"}
if err := srv.agent.UserEvent("dc1", "root", p); err != nil {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
req, err := http.NewRequest("GET", "/v1/event/list", nil)
if err != nil {
return false, err
}
resp := httptest.NewRecorder()
obj, err := srv.EventList(resp, req)
if err != nil {
return false, err
}
list, ok := obj.([]*UserEvent)
if !ok {
return false, fmt.Errorf("bad: %#v", obj)
}
if len(list) != 1 || list[0].Name != "test" {
return false, fmt.Errorf("bad: %#v", list)
}
header := resp.Header().Get("X-Consul-Index")
if header == "" || header == "0" {
return false, fmt.Errorf("bad: %#v", header)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
})
}
func TestLeader_MultiBootstrap(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, s2 := testServer(t)
defer os.RemoveAll(dir2)
defer s2.Shutdown()
servers := []*Server{s1, s2}
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
for _, s := range servers {
testutil.WaitForResult(func() (bool, error) {
peers := s.serfLAN.Members()
return len(peers) == 2, nil
}, func(err error) {
t.Fatalf("should have 2 peers")
})
}
// Ensure we don't have multiple raft peers
for _, s := range servers {
peers, _ := s.raftPeers.Peers()
if len(peers) != 1 {
t.Fatalf("should only have 1 raft peer!")
}
}
}
func expectStatus(t *testing.T, script, status string) {
mock := &MockNotify{
state: make(map[string]string),
updates: make(map[string]int),
output: make(map[string]string),
}
check := &CheckMonitor{
Notify: mock,
CheckID: "foo",
Script: script,
Interval: 10 * time.Millisecond,
Logger: log.New(os.Stderr, "", log.LstdFlags),
}
check.Start()
defer check.Stop()
testutil.WaitForResult(func() (bool, error) {
// Should have at least 2 updates
if mock.updates["foo"] < 2 {
return false, fmt.Errorf("should have 2 updates %v", mock.updates)
}
if mock.state["foo"] != status {
return false, fmt.Errorf("should be %v %v", status, mock.state)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestCatalog_Node(t *testing.T) {
t.Parallel()
c, s := makeClient(t)
defer s.Stop()
catalog := c.Catalog()
name, _ := c.Agent().NodeName()
testutil.WaitForResult(func() (bool, error) {
info, meta, err := catalog.Node(name, nil)
if err != nil {
return false, err
}
if meta.LastIndex == 0 {
return false, fmt.Errorf("Bad: %v", meta)
}
if len(info.Services) == 0 {
return false, fmt.Errorf("Bad: %v", info)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestLeader_Reconcile(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Join before we have a leader, this should cause a reconcile!
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Should not be registered
state := s1.fsm.State()
_, found, _ := state.GetNode(c1.config.NodeName)
if found {
t.Fatalf("client registered")
}
// Should be registered
testutil.WaitForResult(func() (bool, error) {
_, found, _ = state.GetNode(c1.config.NodeName)
return found == true, nil
}, func(err error) {
t.Fatalf("client should be registered")
})
}
func TestHealth_Node(t *testing.T) {
t.Parallel()
c, s := makeClient(t)
defer s.Stop()
agent := c.Agent()
health := c.Health()
info, err := agent.Self()
if err != nil {
t.Fatalf("err: %v", err)
}
name := info["Config"]["NodeName"].(string)
testutil.WaitForResult(func() (bool, error) {
checks, meta, err := health.Node(name, nil)
if err != nil {
return false, err
}
if meta.LastIndex == 0 {
return false, fmt.Errorf("bad: %v", meta)
}
if len(checks) == 0 {
return false, fmt.Errorf("bad: %v", checks)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestCatalog_Service(t *testing.T) {
t.Parallel()
c, s := makeClient(t)
defer s.Stop()
catalog := c.Catalog()
testutil.WaitForResult(func() (bool, error) {
services, meta, err := catalog.Service("consul", "", nil)
if err != nil {
return false, err
}
if meta.LastIndex == 0 {
return false, fmt.Errorf("Bad: %v", meta)
}
if len(services) == 0 {
return false, fmt.Errorf("Bad: %v", services)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %s", err)
})
}
func TestCatalogRegister(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
client := rpcClient(t, s1)
defer client.Close()
arg := structs.RegisterRequest{
Datacenter: "dc1",
Node: "foo",
Address: "127.0.0.1",
Service: &structs.NodeService{
Service: "db",
Tags: []string{"master"},
Port: 8000,
},
}
var out struct{}
err := client.Call("Catalog.Register", &arg, &out)
if err == nil || err.Error() != "No cluster leader" {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
err := client.Call("Catalog.Register", &arg, &out)
return err == nil, err
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestLeader_ReapMember(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
var found bool
state := s1.fsm.State()
// Should be registered
testutil.WaitForResult(func() (bool, error) {
_, found, _ = state.GetNode(c1.config.NodeName)
return found == true, nil
}, func(err error) {
t.Fatalf("client should be registered")
})
// Simulate a node reaping
mems := s1.LANMembers()
var c1mem serf.Member
for _, m := range mems {
if m.Name == c1.config.NodeName {
c1mem = m
c1mem.Status = StatusReap
break
}
}
s1.reconcileCh <- c1mem
// Should be deregistered
testutil.WaitForResult(func() (bool, error) {
_, found, _ = state.GetNode(c1.config.NodeName)
return found == false, nil
}, func(err error) {
t.Fatalf("client should not be registered")
})
}
func TestACL_NonAuthority_Management(t *testing.T) {
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1" // Enable ACLs!
c.ACLMasterToken = "foobar"
c.ACLDefaultPolicy = "deny"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
client := rpcClient(t, s1)
defer client.Close()
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1" // Enable ACLs!
c.ACLDefaultPolicy = "deny"
c.Bootstrap = false // Disable bootstrap
})
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
p1, _ := s1.raftPeers.Peers()
return len(p1) == 2, errors.New(fmt.Sprintf("%v", p1))
}, func(err error) {
t.Fatalf("should have 2 peers: %v", err)
})
testutil.WaitForLeader(t, client.Call, "dc1")
// find the non-authoritative server
var nonAuth *Server
if !s1.IsLeader() {
nonAuth = s1
} else {
nonAuth = s2
}
// Resolve the token
acl, err := nonAuth.resolveToken("foobar")
if err != nil {
t.Fatalf("err: %v", err)
}
if acl == nil {
t.Fatalf("missing acl")
}
// Check the policy, should allow all
if !acl.KeyRead("foo/test") {
t.Fatalf("unexpected failed read")
}
}
func TestRPCClientInstallKey(t *testing.T) {
key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
p1 := testRPCClientWithConfig(t, func(c *Config) {
c.EncryptKey = key1
})
defer p1.Close()
// key2 is not installed yet
testutil.WaitForResult(func() (bool, error) {
keys := listKeys(t, p1.client)
if num, ok := keys["dc1"][key2]; ok || num != 0 {
return false, fmt.Errorf("bad: %#v", keys)
}
if num, ok := keys["WAN"][key2]; ok || num != 0 {
return false, fmt.Errorf("bad: %#v", keys)
}
return true, nil
}, func(err error) {
t.Fatal(err.Error())
})
// install key2
r, err := p1.client.InstallKey(key2)
if err != nil {
t.Fatalf("err: %s", err)
}
keyringSuccess(t, r)
// key2 should now be installed
testutil.WaitForResult(func() (bool, error) {
keys := listKeys(t, p1.client)
if num, ok := keys["dc1"][key2]; !ok || num != 1 {
return false, fmt.Errorf("bad: %#v", keys)
}
if num, ok := keys["WAN"][key2]; !ok || num != 1 {
return false, fmt.Errorf("bad: %#v", keys)
}
return true, nil
}, func(err error) {
t.Fatal(err.Error())
})
}
func TestLeader_RegisterMember(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
client := rpcClient(t, s1)
testutil.WaitForLeader(t, client.Call, "dc1")
// Client should be registered
state := s1.fsm.State()
testutil.WaitForResult(func() (bool, error) {
_, found, _ := state.GetNode(c1.config.NodeName)
return found == true, nil
}, func(err error) {
t.Fatalf("client not registered")
})
// Should have a check
_, checks := state.NodeChecks(c1.config.NodeName)
if len(checks) != 1 {
t.Fatalf("client missing check")
}
if checks[0].CheckID != SerfCheckID {
t.Fatalf("bad check: %v", checks[0])
}
if checks[0].Name != SerfCheckName {
t.Fatalf("bad check: %v", checks[0])
}
if checks[0].Status != structs.HealthPassing {
t.Fatalf("bad check: %v", checks[0])
}
// Server should be registered
_, found, _ := state.GetNode(s1.config.NodeName)
if !found {
t.Fatalf("server not registered")
}
// Service should be registered
_, services := state.NodeServices(s1.config.NodeName)
if _, ok := services.Services["consul"]; !ok {
t.Fatalf("consul service not registered: %v", services)
}
}
func TestExecCommand_Sessions_Foreign(t *testing.T) {
a1 := testAgent(t)
defer a1.Shutdown()
waitForLeader(t, a1.httpAddr)
client, err := HTTPClient(a1.httpAddr)
if err != nil {
t.Fatalf("err: %v", err)
}
ui := new(cli.MockUi)
c := &ExecCommand{
Ui: ui,
client: client,
}
c.conf.foreignDC = true
c.conf.localDC = "dc1"
c.conf.localNode = "foo"
var id string
testutil.WaitForResult(func() (bool, error) {
id, err = c.createSession()
if err != nil && strings.Contains(err.Error(), "Failed to find Consul server") {
err = nil
}
return id != "", err
}, func(err error) {
t.Fatalf("err: %v", err)
})
se, _, err := client.Session().Info(id, nil)
if err != nil {
t.Fatalf("err: %v", err)
}
if se == nil || se.Name != "Remote Exec via foo@dc1" {
t.Fatalf("bad: %v", se)
}
c.sessionID = id
err = c.destroySession()
if err != nil {
t.Fatalf("err: %v", err)
}
se, _, err = client.Session().Info(id, nil)
if err != nil {
t.Fatalf("err: %v", err)
}
if se != nil {
t.Fatalf("bad: %v", se)
}
}
func waitForLeader(t *testing.T, httpAddr string) {
client, err := HTTPClient(httpAddr)
if err != nil {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
_, qm, err := client.Catalog().Nodes(nil)
return err == nil && qm.KnownLeader && qm.LastIndex > 0, err
}, func(err error) {
t.Fatalf("failed to find leader: %v", err)
})
}
func TestClient_RPC_TLS(t *testing.T) {
dir1, conf1 := testServerConfig(t, "a.testco.internal")
conf1.VerifyIncoming = true
conf1.VerifyOutgoing = true
configureTLS(conf1)
s1, err := NewServer(conf1)
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, conf2 := testClientConfig(t, "b.testco.internal")
conf2.VerifyOutgoing = true
configureTLS(conf2)
c1, err := NewClient(conf2)
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Try an RPC
var out struct{}
if err := c1.RPC("Status.Ping", struct{}{}, &out); err != structs.ErrNoServers {
t.Fatalf("err: %v", err)
}
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
// Check the members
if len(s1.LANMembers()) != 2 {
t.Fatalf("bad len")
}
if len(c1.LANMembers()) != 2 {
t.Fatalf("bad len")
}
// RPC should succeed
testutil.WaitForResult(func() (bool, error) {
err := c1.RPC("Status.Ping", struct{}{}, &out)
return err == nil, err
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestLeader_LeftMember(t *testing.T) {
dir1, s1 := testServer(t)
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, c1 := testClient(t)
defer os.RemoveAll(dir2)
defer c1.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
var found bool
state := s1.fsm.State()
// Should be registered
testutil.WaitForResult(func() (bool, error) {
_, found, _ = state.GetNode(c1.config.NodeName)
return found == true, nil
}, func(err error) {
t.Fatalf("client should be registered")
})
// Node should leave
c1.Leave()
c1.Shutdown()
// Should be deregistered
testutil.WaitForResult(func() (bool, error) {
_, found, _ = state.GetNode(c1.config.NodeName)
return found == false, nil
}, func(err error) {
t.Fatalf("client should not be registered")
})
}
func TestRPCClientUseKey(t *testing.T) {
key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
key2 := "xAEZ3uVHRMZD9GcYMZaRQw=="
p1 := testRPCClientWithConfig(t, func(c *Config) {
c.EncryptKey = key1
})
defer p1.Close()
// add a second key to the ring
r, err := p1.client.InstallKey(key2)
if err != nil {
t.Fatalf("err: %s", err)
}
keyringSuccess(t, r)
// key2 is installed
testutil.WaitForResult(func() (bool, error) {
keys := listKeys(t, p1.client)
if num, ok := keys["dc1"][key2]; !ok || num != 1 {
return false, fmt.Errorf("bad: %#v", keys)
}
if num, ok := keys["WAN"][key2]; !ok || num != 1 {
return false, fmt.Errorf("bad: %#v", keys)
}
return true, nil
}, func(err error) {
t.Fatal(err.Error())
})
// can't remove key1 yet
r, err = p1.client.RemoveKey(key1)
if err != nil {
t.Fatalf("err: %s", err)
}
keyringError(t, r)
// change primary key
r, err = p1.client.UseKey(key2)
if err != nil {
t.Fatalf("err: %s", err)
}
keyringSuccess(t, r)
// can remove key1 now
r, err = p1.client.RemoveKey(key1)
if err != nil {
t.Fatalf("err: %s", err)
}
keyringSuccess(t, r)
}
func TestLeader_ReapTombstones(t *testing.T) {
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.TombstoneTTL = 50 * time.Millisecond
c.TombstoneTTLGranularity = 10 * time.Millisecond
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
client := rpcClient(t, s1)
testutil.WaitForLeader(t, client.Call, "dc1")
// Create a KV entry
arg := structs.KVSRequest{
Datacenter: "dc1",
Op: structs.KVSSet,
DirEnt: structs.DirEntry{
Key: "test",
Value: []byte("test"),
},
}
var out bool
if err := client.Call("KVS.Apply", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
// Delete the KV entry (tombstoned)
arg.Op = structs.KVSDelete
if err := client.Call("KVS.Apply", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
// Ensure we have a tombstone
_, res, err := s1.fsm.State().tombstoneTable.Get("id")
if err != nil {
t.Fatalf("err: %v", err)
}
if len(res) == 0 {
t.Fatalf("missing tombstones")
}
// Check that the new leader has a pending GC expiration
testutil.WaitForResult(func() (bool, error) {
_, res, err := s1.fsm.State().tombstoneTable.Get("id")
return len(res) == 0, err
}, func(err error) {
t.Fatalf("err: %v", err)
})
}
func TestACL_NonAuthority_NotFound(t *testing.T) {
dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1"
})
defer os.RemoveAll(dir1)
defer s1.Shutdown()
dir2, s2 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1" // Enable ACLs!
c.Bootstrap = false // Disable bootstrap
})
defer os.RemoveAll(dir2)
defer s2.Shutdown()
// Try to join
addr := fmt.Sprintf("127.0.0.1:%d",
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := s2.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err)
}
testutil.WaitForResult(func() (bool, error) {
p1, _ := s1.raftPeers.Peers()
return len(p1) == 2, errors.New(fmt.Sprintf("%v", p1))
}, func(err error) {
t.Fatalf("should have 2 peers: %v", err)
})
client := rpcClient(t, s1)
defer client.Close()
testutil.WaitForLeader(t, client.Call, "dc1")
// find the non-authoritative server
var nonAuth *Server
if !s1.IsLeader() {
nonAuth = s1
} else {
nonAuth = s2
}
acl, err := nonAuth.resolveToken("does not exist")
if err == nil || err.Error() != aclNotFound {
t.Fatalf("err: %v", err)
}
if acl != nil {
t.Fatalf("got acl")
}
}
func TestEventList_EventBufOrder(t *testing.T) {
httpTest(t, func(srv *HTTPServer) {
// Fire some events in a non-sequential order
expected := &UserEvent{Name: "foo"}
for _, e := range []*UserEvent{
&UserEvent{Name: "foo"},
&UserEvent{Name: "bar"},
&UserEvent{Name: "foo"},
expected,
&UserEvent{Name: "bar"},
} {
if err := srv.agent.UserEvent("dc1", "root", e); err != nil {
t.Fatalf("err: %v", err)
}
}
// Test that the event order is preserved when name
// filtering on a list of > 1 matching event.
testutil.WaitForResult(func() (bool, error) {
url := "/v1/event/list?name=foo"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return false, err
}
resp := httptest.NewRecorder()
obj, err := srv.EventList(resp, req)
if err != nil {
return false, err
}
list, ok := obj.([]*UserEvent)
if !ok {
return false, fmt.Errorf("bad: %#v", obj)
}
if len(list) != 3 || list[2].ID != expected.ID {
return false, fmt.Errorf("bad: %#v", list)
}
return true, nil
}, func(err error) {
t.Fatalf("err: %v", err)
})
})
}
func TestServer_globalRPCErrors(t *testing.T) {
dir1, s1 := testServerDC(t, "dc1")
defer os.RemoveAll(dir1)
defer s1.Shutdown()
testutil.WaitForResult(func() (bool, error) {
return len(s1.remoteConsuls) == 1, nil
}, func(err error) {
t.Fatalf("Server did not join LAN successfully")
})
// Check that an error from a remote DC is returned
err := s1.globalRPC("Bad.Method", nil, &fakeGlobalResp{})
if err == nil {
t.Fatalf("should have errored")
}
if !strings.Contains(err.Error(), "Bad.Method") {
t.Fatalf("unexpcted error: %s", err)
}
}