// generateRandomCoordinate creates a random coordinate. This mucks with the
// underlying structure directly, so it's not really useful for any particular
// position in the network, but it's a good payload to send through to make
// sure things come out the other side or get stored correctly.
func generateRandomCoordinate() *coordinate.Coordinate {
config := coordinate.DefaultConfig()
coord := coordinate.NewCoordinate(config)
for i := range coord.Vec {
coord.Vec[i] = rand.NormFloat64()
}
coord.Error = rand.NormFloat64()
coord.Adjustment = rand.NormFloat64()
return coord
}
// AckPayload is called to produce a payload to send back in response to a ping
// request. In this case we send back a legit ping response with a bad coordinate.
func (p *pingDimensionMetaDelegate) AckPayload() []byte {
var buf bytes.Buffer
// The first byte is the version number, forming a simple header.
version := []byte{PingVersion}
buf.Write(version)
// Make a bad coordinate with the wrong number of dimensions.
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec = make([]float64, 2*len(coord.Vec))
// The rest of the message is the serialized coordinate.
enc := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
if err := enc.Encode(coord); err != nil {
p.t.Fatalf("err: %v", err)
}
return buf.Bytes()
}
func TestRTTCommand_Run_LAN(t *testing.T) {
updatePeriod := 10 * time.Millisecond
a := testAgentWithConfig(t, func(c *agent.Config) {
c.ConsulConfig.CoordinateUpdatePeriod = updatePeriod
})
defer a.Shutdown()
waitForLeader(t, a.httpAddr)
// Inject some known coordinates.
c1 := coordinate.NewCoordinate(coordinate.DefaultConfig())
c2 := c1.Clone()
c2.Vec[0] = 0.123
dist_str := fmt.Sprintf("%.3f ms", c1.DistanceTo(c2).Seconds()*1000.0)
{
req := structs.CoordinateUpdateRequest{
Datacenter: a.config.Datacenter,
Node: a.config.NodeName,
Coord: c1,
}
var reply struct{}
if err := a.agent.RPC("Coordinate.Update", &req, &reply); err != nil {
t.Fatalf("err: %s", err)
}
}
{
req := structs.RegisterRequest{
Datacenter: a.config.Datacenter,
Node: "dogs",
Address: "127.0.0.2",
}
var reply struct{}
if err := a.agent.RPC("Catalog.Register", &req, &reply); err != nil {
t.Fatalf("err: %s", err)
}
}
{
var reply struct{}
req := structs.CoordinateUpdateRequest{
Datacenter: a.config.Datacenter,
Node: "dogs",
Coord: c2,
}
if err := a.agent.RPC("Coordinate.Update", &req, &reply); err != nil {
t.Fatalf("err: %s", err)
}
}
// Wait for the updates to get flushed to the data store.
time.Sleep(2 * updatePeriod)
// Try two known nodes.
{
ui := new(cli.MockUi)
c := &RTTCommand{Ui: ui}
args := []string{
"-http-addr=" + a.httpAddr,
a.config.NodeName,
"dogs",
}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d: %#v", code, ui.ErrorWriter.String())
}
// Make sure the proper RTT was reported in the output.
expected := fmt.Sprintf("rtt: %s", dist_str)
if !strings.Contains(ui.OutputWriter.String(), expected) {
t.Fatalf("bad: %#v", ui.OutputWriter.String())
}
}
// Default to the agent's node.
{
ui := new(cli.MockUi)
c := &RTTCommand{Ui: ui}
args := []string{
"-http-addr=" + a.httpAddr,
"dogs",
}
code := c.Run(args)
if code != 0 {
t.Fatalf("bad: %d: %#v", code, ui.ErrorWriter.String())
}
// Make sure the proper RTT was reported in the output.
expected := fmt.Sprintf("rtt: %s", dist_str)
if !strings.Contains(ui.OutputWriter.String(), expected) {
t.Fatalf("bad: %#v", ui.OutputWriter.String())
}
}
// Try an unknown node.
{
ui := new(cli.MockUi)
c := &RTTCommand{Ui: ui}
args := []string{
"-http-addr=" + a.httpAddr,
a.config.NodeName,
"nope",
}
code := c.Run(args)
if code != 1 {
t.Fatalf("bad: %d: %#v", code, ui.ErrorWriter.String())
}
}
}
func TestCatalogServiceNodes_DistanceSort(t *testing.T) {
dir, srv := makeHTTPServer(t)
defer os.RemoveAll(dir)
defer srv.Shutdown()
defer srv.agent.Shutdown()
testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
// Register nodes.
args := &structs.RegisterRequest{
Datacenter: "dc1",
Node: "bar",
Address: "127.0.0.1",
Service: &structs.NodeService{
Service: "api",
Tags: []string{"a"},
},
}
var out struct{}
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
req, err := http.NewRequest("GET", "/v1/catalog/service/api?tag=a", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
args = &structs.RegisterRequest{
Datacenter: "dc1",
Node: "foo",
Address: "127.0.0.2",
Service: &structs.NodeService{
Service: "api",
Tags: []string{"a"},
},
}
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
// Nobody has coordinates set so this will still return them in the
// order they are indexed.
req, err = http.NewRequest("GET", "/v1/catalog/service/api?tag=a&near=foo", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := httptest.NewRecorder()
obj, err := srv.CatalogServiceNodes(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
assertIndex(t, resp)
nodes := obj.(structs.ServiceNodes)
if len(nodes) != 2 {
t.Fatalf("bad: %v", obj)
}
if nodes[0].Node != "bar" {
t.Fatalf("bad: %v", nodes)
}
if nodes[1].Node != "foo" {
t.Fatalf("bad: %v", nodes)
}
// Send an update for the node and wait for it to get applied.
arg := structs.CoordinateUpdateRequest{
Datacenter: "dc1",
Node: "foo",
Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()),
}
if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
time.Sleep(200 * time.Millisecond)
// Query again and now foo should have moved to the front of the line.
req, err = http.NewRequest("GET", "/v1/catalog/service/api?tag=a&near=foo", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp = httptest.NewRecorder()
obj, err = srv.CatalogServiceNodes(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
assertIndex(t, resp)
nodes = obj.(structs.ServiceNodes)
if len(nodes) != 2 {
t.Fatalf("bad: %v", obj)
}
if nodes[0].Node != "foo" {
t.Fatalf("bad: %v", nodes)
}
if nodes[1].Node != "bar" {
t.Fatalf("bad: %v", nodes)
}
}
// generateCoordinate creates a new coordinate with the given distance from the
// origin.
func generateCoordinate(rtt time.Duration) *coordinate.Coordinate {
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec[0] = rtt.Seconds()
coord.Height = 0
return coord
}
func TestHealthChecksInState_DistanceSort(t *testing.T) {
dir, srv := makeHTTPServer(t)
defer os.RemoveAll(dir)
defer srv.Shutdown()
defer srv.agent.Shutdown()
testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
args := &structs.RegisterRequest{
Datacenter: "dc1",
Node: "bar",
Address: "127.0.0.1",
Check: &structs.HealthCheck{
Node: "bar",
Name: "node check",
Status: structs.HealthCritical,
},
}
var out struct{}
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
args.Node, args.Check.Node = "foo", "foo"
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
req, err := http.NewRequest("GET", "/v1/health/state/critical?dc=dc1&near=foo", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := httptest.NewRecorder()
obj, err := srv.HealthChecksInState(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
assertIndex(t, resp)
nodes := obj.(structs.HealthChecks)
if len(nodes) != 2 {
t.Fatalf("bad: %v", nodes)
}
if nodes[0].Node != "bar" {
t.Fatalf("bad: %v", nodes)
}
if nodes[1].Node != "foo" {
t.Fatalf("bad: %v", nodes)
}
// Send an update for the node and wait for it to get applied.
arg := structs.CoordinateUpdateRequest{
Datacenter: "dc1",
Node: "foo",
Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()),
}
if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
// Retry until foo moves to the front of the line.
testutil.WaitForResult(func() (bool, error) {
resp = httptest.NewRecorder()
obj, err = srv.HealthChecksInState(resp, req)
if err != nil {
return false, fmt.Errorf("err: %v", err)
}
assertIndex(t, resp)
nodes = obj.(structs.HealthChecks)
if len(nodes) != 2 {
return false, fmt.Errorf("bad: %v", nodes)
}
if nodes[0].Node != "foo" {
return false, fmt.Errorf("bad: %v", nodes)
}
if nodes[1].Node != "bar" {
return false, fmt.Errorf("bad: %v", nodes)
}
return true, nil
}, func(err error) {
t.Fatalf("failed to get sorted service nodes: %v", err)
})
}
func TestCoordinate_Nodes(t *testing.T) {
dir, srv := makeHTTPServer(t)
defer os.RemoveAll(dir)
defer srv.Shutdown()
defer srv.agent.Shutdown()
testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
// Make sure an empty list is non-nil.
req, err := http.NewRequest("GET", "/v1/coordinate/nodes?dc=dc1", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := httptest.NewRecorder()
obj, err := srv.CoordinateNodes(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
coordinates := obj.(structs.Coordinates)
if coordinates == nil || len(coordinates) != 0 {
t.Fatalf("bad: %v", coordinates)
}
// Register the nodes.
nodes := []string{"foo", "bar"}
for _, node := range nodes {
req := structs.RegisterRequest{
Datacenter: "dc1",
Node: node,
Address: "127.0.0.1",
}
var reply struct{}
if err := srv.agent.RPC("Catalog.Register", &req, &reply); err != nil {
t.Fatalf("err: %s", err)
}
}
// Send some coordinates for a few nodes, waiting a little while for the
// batch update to run.
arg1 := structs.CoordinateUpdateRequest{
Datacenter: "dc1",
Node: "foo",
Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()),
}
var out struct{}
if err := srv.agent.RPC("Coordinate.Update", &arg1, &out); err != nil {
t.Fatalf("err: %v", err)
}
arg2 := structs.CoordinateUpdateRequest{
Datacenter: "dc1",
Node: "bar",
Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()),
}
if err := srv.agent.RPC("Coordinate.Update", &arg2, &out); err != nil {
t.Fatalf("err: %v", err)
}
time.Sleep(200 * time.Millisecond)
// Query back and check the nodes are present and sorted correctly.
req, err = http.NewRequest("GET", "/v1/coordinate/nodes?dc=dc1", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp = httptest.NewRecorder()
obj, err = srv.CoordinateNodes(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
coordinates = obj.(structs.Coordinates)
if len(coordinates) != 2 ||
coordinates[0].Node != "bar" ||
coordinates[1].Node != "foo" {
t.Fatalf("bad: %v", coordinates)
}
}
func TestHealthChecksInState_DistanceSort(t *testing.T) {
dir, srv := makeHTTPServer(t)
defer os.RemoveAll(dir)
defer srv.Shutdown()
defer srv.agent.Shutdown()
testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
args := &structs.RegisterRequest{
Datacenter: "dc1",
Node: "bar",
Address: "127.0.0.1",
Check: &structs.HealthCheck{
Node: "bar",
Name: "node check",
Status: structs.HealthCritical,
},
}
var out struct{}
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
args.Node, args.Check.Node = "foo", "foo"
if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
t.Fatalf("err: %v", err)
}
req, err := http.NewRequest("GET", "/v1/health/state/critical?dc=dc1&near=foo", nil)
if err != nil {
t.Fatalf("err: %v", err)
}
resp := httptest.NewRecorder()
obj, err := srv.HealthChecksInState(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
assertIndex(t, resp)
nodes := obj.(structs.HealthChecks)
if len(nodes) != 2 {
t.Fatalf("bad: %v", nodes)
}
if nodes[0].Node != "bar" {
t.Fatalf("bad: %v", nodes)
}
if nodes[1].Node != "foo" {
t.Fatalf("bad: %v", nodes)
}
// Send an update for the node and wait for it to get applied.
arg := structs.CoordinateUpdateRequest{
Datacenter: "dc1",
Node: "foo",
Coord: coordinate.NewCoordinate(coordinate.DefaultConfig()),
}
if err := srv.agent.RPC("Coordinate.Update", &arg, &out); err != nil {
t.Fatalf("err: %v", err)
}
time.Sleep(300 * time.Millisecond)
// Query again and now foo should have moved to the front of the line.
resp = httptest.NewRecorder()
obj, err = srv.HealthChecksInState(resp, req)
if err != nil {
t.Fatalf("err: %v", err)
}
assertIndex(t, resp)
nodes = obj.(structs.HealthChecks)
if len(nodes) != 2 {
t.Fatalf("bad: %v", nodes)
}
if nodes[0].Node != "foo" {
t.Fatalf("bad: %v", nodes)
}
if nodes[1].Node != "bar" {
t.Fatalf("bad: %v", nodes)
}
}
func TestSnapshotter_forceCompact(t *testing.T) {
td, err := ioutil.TempDir("", "serf")
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(td)
// Set up a coordinate at a known location.
coordClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec[0] = 123.4
coordClient.SetCoordinate(coord)
clock := new(LamportClock)
stopCh := make(chan struct{})
logger := log.New(os.Stderr, "", log.LstdFlags)
// Create a very low limit
inCh, snap, err := NewSnapshotter(td+"snap", 1024, false,
logger, clock, coordClient, nil, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Write lots of user events
for i := 0; i < 1024; i++ {
ue := UserEvent{
LTime: LamportTime(i),
}
inCh <- ue
}
// Write lots of queries
for i := 0; i < 1024; i++ {
q := &Query{
LTime: LamportTime(i),
}
inCh <- q
}
// Wait for drain
for len(inCh) > 0 {
time.Sleep(20 * time.Millisecond)
}
// Close the snapshoter
close(stopCh)
snap.Wait()
// Make a new client back at the origin.
newClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
// Open the snapshoter
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, newClient, nil, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Check the values
if snap.LastEventClock() != 1023 {
t.Fatalf("bad clock %d", snap.LastEventClock())
}
if snap.LastQueryClock() != 1023 {
t.Fatalf("bad clock %d", snap.LastQueryClock())
}
coord = newClient.GetCoordinate()
if coord.Vec[0] != 123.4 {
t.Fatalf("bad coordinate: %v", coord)
}
close(stopCh)
snap.Wait()
}
func TestSnapshotter(t *testing.T) {
td, err := ioutil.TempDir("", "serf")
if err != nil {
t.Fatalf("err: %v", err)
}
defer os.RemoveAll(td)
// Set up a coordinate at a known location.
coordClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
coord := coordinate.NewCoordinate(coordinate.DefaultConfig())
coord.Vec[0] = 123.4
coordClient.SetCoordinate(coord)
clock := new(LamportClock)
outCh := make(chan Event, 64)
stopCh := make(chan struct{})
logger := log.New(os.Stderr, "", log.LstdFlags)
inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, coordClient, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Write some user events
ue := UserEvent{
LTime: 42,
Name: "bar",
}
inCh <- ue
// Write some queries
q := &Query{
LTime: 50,
Name: "uptime",
}
inCh <- q
// Write some member events
clock.Witness(100)
meJoin := MemberEvent{
Type: EventMemberJoin,
Members: []Member{
Member{
Name: "foo",
Addr: []byte{127, 0, 0, 1},
Port: 5000,
},
},
}
meFail := MemberEvent{
Type: EventMemberFailed,
Members: []Member{
Member{
Name: "foo",
Addr: []byte{127, 0, 0, 1},
Port: 5000,
},
},
}
inCh <- meJoin
inCh <- meFail
inCh <- meJoin
// Check these get passed through
select {
case e := <-outCh:
if !reflect.DeepEqual(e, ue) {
t.Fatalf("expected user event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, q) {
t.Fatalf("expected query event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meJoin) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meFail) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
select {
case e := <-outCh:
if !reflect.DeepEqual(e, meJoin) {
t.Fatalf("expected member event: %#v", e)
}
case <-time.After(200 * time.Millisecond):
t.Fatalf("timeout")
}
// Manually kick a coordinate update so the test doesn't have to wait
// for the long period.
snap.updateCoordinate()
// Close the snapshoter
close(stopCh)
snap.Wait()
// Make a new client back at the origin.
newClient, err := coordinate.NewClient(coordinate.DefaultConfig())
if err != nil {
t.Fatalf("err: %v", err)
}
// Open the snapshoter
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, newClient, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
// Check the values
if snap.LastClock() != 100 {
t.Fatalf("bad clock %d", snap.LastClock())
}
if snap.LastEventClock() != 42 {
t.Fatalf("bad clock %d", snap.LastEventClock())
}
if snap.LastQueryClock() != 50 {
t.Fatalf("bad clock %d", snap.LastQueryClock())
}
prev := snap.AliveNodes()
if len(prev) != 1 {
t.Fatalf("expected alive: %#v", prev)
}
if prev[0].Name != "foo" {
t.Fatalf("bad name: %#v", prev[0])
}
if prev[0].Addr != "127.0.0.1:5000" {
t.Fatalf("bad addr: %#v", prev[0])
}
coord = newClient.GetCoordinate()
if coord.Vec[0] != 123.4 {
t.Fatalf("bad coordinate: %v", coord)
}
// Close the snapshotter.
close(stopCh)
snap.Wait()
// Open the snapshotter, make sure nothing dies reading with coordinates
// disabled.
stopCh = make(chan struct{})
_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
logger, clock, nil, outCh, stopCh)
if err != nil {
t.Fatalf("err: %v", err)
}
close(stopCh)
snap.Wait()
}