func TestTemplate(t *testing.T) {
tablet := topo.NewTablet(0, "cell", "a")
ts := []*TabletStats{
{
Key: "a",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: false,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
TabletExternallyReparentedTimestamp: 0,
},
}
tcs := &TabletsCacheStatus{
Cell: "cell",
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
TabletsStats: ts,
}
templ := template.New("").Funcs(status.StatusFuncs)
templ, err := templ.Parse(HealthCheckTemplate)
if err != nil {
t.Fatalf("error parsing template: %v", err)
}
wr := &bytes.Buffer{}
if err := templ.Execute(wr, []*TabletsCacheStatus{tcs}); err != nil {
t.Fatalf("error executing template: %v", err)
}
}
// AddTestTablet inserts a fake entry into FakeHealthCheck.
// The Tablet can be talked to using the provided connection.
func (fhc *FakeHealthCheck) AddTestTablet(cell, host string, port int32, keyspace, shard string, tabletType topodatapb.TabletType, serving bool, reparentTS int64, err error) *sandboxconn.SandboxConn {
t := topo.NewTablet(0, cell, host)
t.Keyspace = keyspace
t.Shard = shard
t.Type = tabletType
t.PortMap["vt"] = port
key := TabletToMapKey(t)
fhc.mu.Lock()
defer fhc.mu.Unlock()
item := fhc.items[key]
if item == nil {
item = &fhcItem{
ts: &TabletStats{
Tablet: t,
},
}
fhc.items[key] = item
}
item.ts.Target = &querypb.Target{
Keyspace: keyspace,
Shard: shard,
TabletType: tabletType,
}
item.ts.Serving = serving
item.ts.TabletExternallyReparentedTimestamp = reparentTS
item.ts.Stats = &querypb.RealtimeStats{}
item.ts.LastError = err
conn := sandboxconn.NewSandboxConn(t)
item.conn = conn
return conn
}
func TestHealthCheckTimeout(t *testing.T) {
timeout := 500 * time.Millisecond
tablet := topo.NewTablet(0, "cell", "a")
tablet.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(tablet, input)
t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
l := newListener()
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, timeout).(*HealthCheckImpl)
hc.SetListener(l, false)
hc.AddTablet(tablet, "")
t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`)
// one tablet after receiving a StreamHealthResponse
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Serving: true,
TabletExternallyReparentedTimestamp: 10,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
want := &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res := <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// wait for timeout period
time.Sleep(2 * timeout)
t.Logf(`Sleep(2 * timeout)`)
res = <-l.output
if res.Serving {
t.Errorf(`<-l.output: %+v; want not serving`, res)
}
// send a healthcheck response, it should be serving again
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// close healthcheck
hc.Close()
}
func TestWaitForTablets(t *testing.T) {
shortCtx, shortCancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer shortCancel()
waitAvailableTabletInterval = 20 * time.Millisecond
tablet := topo.NewTablet(0, "cell", "a")
tablet.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(tablet, input)
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, 1*time.Hour)
tsc := NewTabletStatsCache(hc, "cell")
hc.AddTablet(tablet, "")
// this should time out
if err := tsc.WaitForTablets(shortCtx, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != context.DeadlineExceeded {
t.Errorf("got wrong error: %v", err)
}
// this should fail, but return a non-timeout error
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel()
if err := tsc.WaitForTablets(cancelledCtx, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err == nil || err == context.DeadlineExceeded {
t.Errorf("want: non-timeout error, got: %v", err)
}
// send the tablet in
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{
Keyspace: "keyspace",
Shard: "shard",
TabletType: topodatapb.TabletType_REPLICA,
},
Serving: true,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
input <- shr
// and ask again, with longer time outs so it's not flaky
longCtx, longCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer longCancel()
waitAvailableTabletInterval = 10 * time.Millisecond
if err := tsc.WaitForTablets(longCtx, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
t.Errorf("got error: %v", err)
}
}
func TestHealthCheck(t *testing.T) {
tablet := topo.NewTablet(0, "cell", "a")
tablet.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(tablet, input)
t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
l := newListener()
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, time.Hour).(*HealthCheckImpl)
hc.SetListener(l, true)
hc.AddTablet(tablet, "")
t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`)
// Immediately after AddTablet() there will be the first notification.
want := &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{},
Up: true,
Serving: false,
}
res := <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// one tablet after receiving a StreamHealthResponse
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Serving: true,
TabletExternallyReparentedTimestamp: 10,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
tcsl := hc.CacheStatus()
tcslWant := TabletsCacheStatusList{{
Cell: "cell",
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
TabletsStats: TabletStatsList{{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}},
}}
if !reflect.DeepEqual(tcsl, tcslWant) {
t.Errorf(`hc.CacheStatus() = %+v; want %+v`, tcsl, tcslWant)
}
// TabletType changed, should get both old and new event
shr = &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Serving: true,
TabletExternallyReparentedTimestamp: 0,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`)
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: false,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
TabletExternallyReparentedTimestamp: 0,
}
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// Serving & RealtimeStats changed
shr = &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Serving: false,
TabletExternallyReparentedTimestamp: 0,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
}
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: false,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
TabletExternallyReparentedTimestamp: 0,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// HealthError
shr = &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Serving: true,
TabletExternallyReparentedTimestamp: 0,
RealtimeStats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3},
}
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: false,
Stats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3},
TabletExternallyReparentedTimestamp: 0,
LastError: fmt.Errorf("vttablet error: some error"),
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// remove tablet
hc.deleteConn(tablet)
t.Logf(`hc.RemoveTablet({Host: "a", PortMap: {"vt": 1}})`)
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: false,
Serving: false,
Stats: &querypb.RealtimeStats{HealthError: "some error", SecondsBehindMaster: 1, CpuUsage: 0.3},
TabletExternallyReparentedTimestamp: 0,
LastError: context.Canceled,
}
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// close healthcheck
hc.Close()
}
// TestHealthCheckCloseWaitsForGoRoutines tests that Close() waits for all Go
// routines to finish and the listener won't be called anymore.
func TestHealthCheckCloseWaitsForGoRoutines(t *testing.T) {
tablet := topo.NewTablet(0, "cell", "a")
tablet.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse, 1)
createFakeConn(tablet, input)
t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
l := newListener()
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, time.Hour).(*HealthCheckImpl)
hc.SetListener(l, false)
hc.AddTablet(tablet, "")
t.Logf(`hc = HealthCheck(); hc.AddTablet({Host: "a", PortMap: {"vt": 1}}, "")`)
// Immediately after AddTablet() there will be the first notification.
want := &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{},
Up: true,
Serving: false,
}
res := <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// Verify that the listener works in general.
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Serving: true,
TabletExternallyReparentedTimestamp: 10,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
want = &TabletStats{
Key: "a,vt:1",
Tablet: tablet,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
input <- shr
t.Logf(`input <- %v`, shr)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// Change input to distinguish between stats sent before and after Close().
shr.TabletExternallyReparentedTimestamp = 11
// Close the healthcheck. Tablet connections are closed asynchronously and
// Close() will block until all Go routines (one per connection) are done.
hc.Close()
// Try to send more updates. They should be ignored and the listener should
// not be called from any Go routine anymore.
// Note that this code is racy by nature. If there is a regression, it should
// fail in some cases.
input <- shr
t.Logf(`input <- %v`, shr)
// After Close() we'll receive one or two notifications with Serving == false.
res = <-l.output
if res.Serving {
t.Errorf(`Received one more notification with Serving == true: %+v`, res)
}
select {
case res = <-l.output:
if res.TabletExternallyReparentedTimestamp == 10 && res.LastError == context.Canceled {
// HealthCheck repeats the previous stats if there is an error.
// This is expected.
break
}
t.Fatalf("healthCheck still running after Close(): listener received: %v but should not have been called", res)
case <-time.After(1 * time.Millisecond):
// No response after timeout. Close probably closed all Go routines
// properly and won't use the listener anymore.
}
// The last notification should have Up = false.
if res.Up || res.Serving {
t.Errorf(`Last notification doesn't have Up == false and Serving == false: %+v`, res)
}
// Check if there are more updates than the one emitted during Close().
select {
case res := <-l.output:
t.Fatalf("healthCheck still running after Close(): listener received: %v but should not have been called", res)
case <-time.After(1 * time.Millisecond):
// No response after timeout. Listener probably not called again. Success.
}
}
func TestFilterByReplicationLag(t *testing.T) {
// 0 tablet
got := FilterByReplicationLag([]*TabletStats{})
if len(got) != 0 {
t.Errorf("FilterByReplicationLag([]) = %+v, want []", got)
}
// 1 serving tablet
ts1 := &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{},
}
ts2 := &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: false,
Stats: &querypb.RealtimeStats{},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2})
if len(got) != 1 {
t.Errorf("len(FilterByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}])) = %v, want 1", len(got))
}
if len(got) > 0 && !reflect.DeepEqual(got[0], ts1) {
t.Errorf("FilterByReplicationLag([{Tablet: {Uid: 1}, Serving: true}, {Tablet: {Uid: 2}, Serving: false}]) = %+v, want %+v", got[0], ts1)
}
// lags of (1s, 1s, 1s, 30s)
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts3 := &TabletStats{
Tablet: topo.NewTablet(3, "cell", "host3"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts4 := &TabletStats{
Tablet: topo.NewTablet(4, "cell", "host4"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 30},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4})
if len(got) != 4 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) || !reflect.DeepEqual(got[2], ts3) || !reflect.DeepEqual(got[3], ts4) {
t.Errorf("FilterByReplicationLag([1s, 1s, 1s, 30s]) = %+v, want all", got)
}
// lags of (5s, 10s, 15s, 120s)
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 5},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10},
}
ts3 = &TabletStats{
Tablet: topo.NewTablet(3, "cell", "host3"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 15},
}
ts4 = &TabletStats{
Tablet: topo.NewTablet(4, "cell", "host4"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 120},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4})
if len(got) != 3 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) || !reflect.DeepEqual(got[2], ts3) {
t.Errorf("FilterByReplicationLag([5s, 10s, 15s, 120s]) = %+v, want [5s, 10s, 15s]", got)
}
// lags of (30m, 35m, 40m, 45m)
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 30 * 60},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 35 * 60},
}
ts3 = &TabletStats{
Tablet: topo.NewTablet(3, "cell", "host3"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 40 * 60},
}
ts4 = &TabletStats{
Tablet: topo.NewTablet(4, "cell", "host4"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 45 * 60},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4})
if len(got) != 4 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) || !reflect.DeepEqual(got[2], ts3) || !reflect.DeepEqual(got[3], ts4) {
t.Errorf("FilterByReplicationLag([30m, 35m, 40m, 45m]) = %+v, want all", got)
}
// lags of (1s, 1s, 1m, 40m, 40m) - not run filter the second time as first run removed two items.
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts3 = &TabletStats{
Tablet: topo.NewTablet(3, "cell", "host3"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60},
}
ts4 = &TabletStats{
Tablet: topo.NewTablet(4, "cell", "host4"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 40 * 60},
}
ts5 := &TabletStats{
Tablet: topo.NewTablet(5, "cell", "host5"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 40 * 60},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4, ts5})
if len(got) != 3 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) || !reflect.DeepEqual(got[2], ts3) {
t.Errorf("FilterByReplicationLag([1s, 1s, 1m, 40m, 40m]) = %+v, want [1s, 1s, 1m]", got)
}
// lags of (1s, 1s, 10m, 40m) - run filter twice to remove two items
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1},
}
ts3 = &TabletStats{
Tablet: topo.NewTablet(3, "cell", "host3"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10 * 60},
}
ts4 = &TabletStats{
Tablet: topo.NewTablet(4, "cell", "host4"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 40 * 60},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2, ts3, ts4})
if len(got) != 2 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) {
t.Errorf("FilterByReplicationLag([1s, 1s, 10m, 40m]) = %+v, want [1s, 1s]", got)
}
// lags of (1m, 100m) - return at least 2 items to avoid overloading if the 2nd one is not delayed too much
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 100 * 60},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2})
if len(got) != 2 || !reflect.DeepEqual(got[0], ts1) || !reflect.DeepEqual(got[1], ts2) {
t.Errorf("FilterByReplicationLag([1m, 100m]) = %+v, want all", got)
}
// lags of (1m, 3h) - return 1 if the 2nd one is delayed too much
ts1 = &TabletStats{
Tablet: topo.NewTablet(1, "cell", "host1"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1 * 60},
}
ts2 = &TabletStats{
Tablet: topo.NewTablet(2, "cell", "host2"),
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 3 * 60 * 60},
}
got = FilterByReplicationLag([]*TabletStats{ts1, ts2})
if len(got) != 1 || !reflect.DeepEqual(got[0], ts1) {
t.Errorf("FilterByReplicationLag([1m, 3h]) = %+v, want [1m]", got)
}
}
package worker
import (
"testing"
"github.com/golang/protobuf/proto"
"github.com/youtube/vitess/go/vt/discovery"
"github.com/youtube/vitess/go/vt/topo"
querypb "github.com/youtube/vitess/go/vt/proto/query"
topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)
var ts1 = discovery.TabletStats{
Tablet: topo.NewTablet(10, "cell", "host1"),
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
}
var ts2 = discovery.TabletStats{
Tablet: topo.NewTablet(20, "cell", "host1"),
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
}
var allTs = []discovery.TabletStats{ts1, ts2}
func TestTabletsInUse(t *testing.T) {
tt := NewTabletTracker()
tt.Track([]discovery.TabletStats{ts1})
if got, want := tt.TabletsInUse(), "cell-0000000010"; got != want {
t.Fatalf("TabletsInUse() = %v, want = %v", got, want)
}
// TestTabletStatsCache tests the functionality of the TabletStatsCache class.
func TestTabletStatsCache(t *testing.T) {
// We want to unit test TabletStatsCache without a full-blown
// HealthCheck object, so we can't call NewTabletStatsCache.
// So we just construct this object here.
tsc := &TabletStatsCache{
cell: "cell",
entries: make(map[string]map[string]map[topodatapb.TabletType]*tabletStatsCacheEntry),
}
// empty
a := tsc.GetTabletStats("k", "s", topodatapb.TabletType_MASTER)
if len(a) != 0 {
t.Errorf("wrong result, expected empty list: %v", a)
}
// add a tablet
tablet1 := topo.NewTablet(10, "cell", "host1")
ts1 := &TabletStats{
Key: "t1",
Tablet: tablet1,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
tsc.StatsUpdate(ts1)
// check it's there
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// update stats with a change that won't change health array
stillHealthyTs1 := &TabletStats{
Key: "t1",
Tablet: tablet1,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 2, CpuUsage: 0.2},
}
tsc.StatsUpdate(stillHealthyTs1)
// check the previous ts1 is still there, as the new one is ignored.
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// update stats with a change that will change arrays
notHealthyTs1 := &TabletStats{
Key: "t1",
Tablet: tablet1,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 35, CpuUsage: 0.2},
}
tsc.StatsUpdate(notHealthyTs1)
// check it's there
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*notHealthyTs1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*notHealthyTs1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// add a second tablet
tablet2 := topo.NewTablet(11, "cell", "host2")
ts2 := &TabletStats{
Key: "t2",
Tablet: tablet2,
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 10, CpuUsage: 0.2},
}
tsc.StatsUpdate(ts2)
// check it's there
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 2 {
t.Errorf("unexpected result: %v", a)
} else {
if a[0].Tablet.Alias.Uid == 11 {
a[0], a[1] = a[1], a[0]
}
if !reflect.DeepEqual(*ts1, a[0]) || !reflect.DeepEqual(*ts2, a[1]) {
t.Errorf("unexpected result: %v", a)
}
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 2 {
t.Errorf("unexpected result: %v", a)
} else {
if a[0].Tablet.Alias.Uid == 11 {
a[0], a[1] = a[1], a[0]
}
if !reflect.DeepEqual(*ts1, a[0]) || !reflect.DeepEqual(*ts2, a[1]) {
t.Errorf("unexpected result: %v", a)
}
}
// one tablet goes unhealthy
ts2.Serving = false
tsc.StatsUpdate(ts2)
// check we only have one left in healthy version
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 2 {
t.Errorf("unexpected result: %v", a)
} else {
if a[0].Tablet.Alias.Uid == 11 {
a[0], a[1] = a[1], a[0]
}
if !reflect.DeepEqual(*ts1, a[0]) || !reflect.DeepEqual(*ts2, a[1]) {
t.Errorf("unexpected result: %v", a)
}
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// second tablet turns into a master, we receive down + up
ts2.Serving = true
ts2.Up = false
tsc.StatsUpdate(ts2)
ts2.Up = true
ts2.Target.TabletType = topodatapb.TabletType_MASTER
ts2.TabletExternallyReparentedTimestamp = 10
tsc.StatsUpdate(ts2)
// check we only have one replica left
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// check we have a master now
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_MASTER)
if len(a) != 1 || !reflect.DeepEqual(*ts2, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// reparent: old replica goes into master
ts1.Up = false
tsc.StatsUpdate(ts1)
ts1.Up = true
ts1.Target.TabletType = topodatapb.TabletType_MASTER
ts1.TabletExternallyReparentedTimestamp = 20
tsc.StatsUpdate(ts1)
// check we lost all replicas, and master is new one
a = tsc.GetTabletStats("k", "s", topodatapb.TabletType_REPLICA)
if len(a) != 0 {
t.Errorf("unexpected result: %v", a)
}
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_MASTER)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
// old master sending an old ping should be ignored
tsc.StatsUpdate(ts2)
a = tsc.GetHealthyTabletStats("k", "s", topodatapb.TabletType_MASTER)
if len(a) != 1 || !reflect.DeepEqual(*ts1, a[0]) {
t.Errorf("unexpected result: %v", a)
}
}
// InitTablet will create or update a tablet. If not parent is
// specified, and the tablet created is a slave type, we will find the
// appropriate parent.
func (wr *Wrangler) InitTablet(tabletAlias topo.TabletAlias, hostname, mysqlPort, port, keyspace, shardId, tabletType string, parentAlias topo.TabletAlias, dbNameOverride string, force, update bool) error {
// if shardId contains a '-', we assume it's a range-based shard,
// so we try to extract the KeyRange.
var keyRange key.KeyRange
var err error
if strings.Contains(shardId, "-") {
parts := strings.Split(shardId, "-")
if len(parts) != 2 {
return fmt.Errorf("Invalid shardId, can only contains one '-': %v", shardId)
}
keyRange.Start, err = key.HexKeyspaceId(parts[0]).Unhex()
if err != nil {
return err
}
keyRange.End, err = key.HexKeyspaceId(parts[1]).Unhex()
if err != nil {
return err
}
shardId = strings.ToUpper(shardId)
}
if parentAlias == (topo.TabletAlias{}) && topo.TabletType(tabletType) != topo.TYPE_MASTER && topo.TabletType(tabletType) != topo.TYPE_IDLE {
parentAlias, err = wr.getMasterAlias(keyspace, shardId)
if err != nil {
return err
}
}
tablet, err := topo.NewTablet(tabletAlias.Cell, tabletAlias.Uid, parentAlias, fmt.Sprintf("%v:%v", hostname, port), fmt.Sprintf("%v:%v", hostname, mysqlPort), keyspace, shardId, topo.TabletType(tabletType))
if err != nil {
return err
}
tablet.DbNameOverride = dbNameOverride
tablet.KeyRange = keyRange
err = topo.CreateTablet(wr.ts, tablet)
if err != nil && err == topo.ErrNodeExists {
// Try to update nicely, but if it fails fall back to force behavior.
if update {
oldTablet, err := wr.ts.GetTablet(tabletAlias)
if err != nil {
relog.Warning("failed reading tablet %v: %v", tabletAlias, err)
} else {
if oldTablet.Keyspace == tablet.Keyspace && oldTablet.Shard == tablet.Shard {
*(oldTablet.Tablet) = *tablet
err := topo.UpdateTablet(wr.ts, oldTablet)
if err != nil {
relog.Warning("failed updating tablet %v: %v", tabletAlias, err)
} else {
return nil
}
}
}
}
if force {
if _, err = wr.Scrap(tabletAlias, force, false); err != nil {
relog.Error("failed scrapping tablet %v: %v", tabletAlias, err)
return err
}
if err = wr.ts.DeleteTablet(tabletAlias); err != nil {
// we ignore this
relog.Error("failed deleting tablet %v: %v", tabletAlias, err)
}
err = topo.CreateTablet(wr.ts, tablet)
}
}
return err
}