func (fhc *fakeHealthCheck) addTestEndPoint(cell, host string, port int32, keyspace, shard string, tabletType pbt.TabletType, reparentTS int64, err error, conn tabletconn.TabletConn) *pbt.EndPoint {
ep := topo.NewEndPoint(0, host)
ep.PortMap["vt"] = port
key := discovery.EndPointToMapKey(ep)
item := fhc.items[key]
if item == nil {
fhc.AddEndPoint(cell, ep)
item = fhc.items[key]
}
item.eps.Target = &pbq.Target{Keyspace: keyspace, Shard: shard, TabletType: tabletType}
item.eps.TabletExternallyReparentedTimestamp = reparentTS
item.eps.LastError = err
item.conn = conn
return ep
}
func TestWaitForEndPoints(t *testing.T) {
shortCtx, shortCancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer shortCancel()
waitAvailableEndPointInterval = 20 * time.Millisecond
ep := topo.NewEndPoint(0, "a")
ep.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(ep, input)
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, 1*time.Hour, "" /* statsSuffix */)
hc.AddEndPoint("cell", "", ep)
// this should time out
if err := WaitForEndPoints(shortCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != ErrWaitForEndPointsTimeout {
t.Errorf("got wrong error: %v", err)
}
// this should fail, but return a non-timeout error
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel()
if err := WaitForEndPoints(cancelledCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err == nil || err == ErrWaitForEndPointsTimeout {
t.Errorf("want: non-timeout error, got: %v", err)
}
// send the endpoint in
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{
Keyspace: "keyspace",
Shard: "shard",
TabletType: topodatapb.TabletType_REPLICA,
},
Serving: true,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
input <- shr
// and ask again, with longer time outs so it's not flaky
longCtx, longCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer longCancel()
waitAvailableEndPointInterval = 10 * time.Millisecond
if err := WaitForEndPoints(longCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
t.Errorf("got error: %v", err)
}
}
func TestWaitForEndPoints(t *testing.T) {
waitAvailableEndPointPeriod = 10 * time.Millisecond
waitAvailableEndPointInterval = 20 * time.Millisecond
ep := topo.NewEndPoint(0, "a")
ep.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(ep, input)
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, 1*time.Hour)
hc.AddEndPoint("cell", "", ep)
// this should time out
if err := WaitForEndPoints(hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != ErrWaitForEndPointsTimeout {
t.Errorf("got wrong error: %v", err)
}
// send the endpoint in
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{
Keyspace: "keyspace",
Shard: "shard",
TabletType: topodatapb.TabletType_REPLICA,
},
Serving: true,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
input <- shr
// and ask again, with longer time outs so it's not flaky
waitAvailableEndPointPeriod = 10 * time.Second
waitAvailableEndPointInterval = 10 * time.Millisecond
if err := WaitForEndPoints(hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
t.Errorf("got error: %v", err)
}
}
func TestHealthCheck(t *testing.T) {
ep := topo.NewEndPoint(0, "a")
ep.PortMap["vt"] = 1
input := make(chan *pbq.StreamHealthResponse)
createFakeConn(ep, input)
t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
l := newListener()
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond).(*HealthCheckImpl)
hc.SetListener(l)
hc.AddEndPoint("cell", "", ep)
t.Logf(`hc = HealthCheck(); hc.AddEndPoint("cell", "", {Host: "a", PortMap: {"vt": 1}})`)
// no endpoint before getting first StreamHealthResponse
epsList := hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 0 {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want empty`, epsList)
}
// one endpoint after receiving a StreamHealthResponse
shr := &pbq.StreamHealthResponse{
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
Serving: true,
TabletExternallyReparentedTimestamp: 10,
RealtimeStats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
want := &EndPointStats{
EndPoint: ep,
Cell: "cell",
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
Serving: true,
Stats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res := <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
}
epcsl := hc.CacheStatus()
epcslWant := EndPointsCacheStatusList{{
Cell: "cell",
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
EndPointsStats: EndPointStatsList{{
EndPoint: ep,
Cell: "cell",
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
Serving: true,
Stats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}},
}}
if !reflect.DeepEqual(epcsl, epcslWant) {
t.Errorf(`hc.CacheStatus() = %+v; want %+v`, epcsl, epcslWant)
}
// TabletType changed
shr = &pbq.StreamHealthResponse{
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
Serving: true,
TabletExternallyReparentedTimestamp: 0,
RealtimeStats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
}
want = &EndPointStats{
EndPoint: ep,
Cell: "cell",
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
Serving: true,
Stats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
TabletExternallyReparentedTimestamp: 0,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
epsList = hc.GetEndPointStatsFromTarget("k", "s", pbt.TabletType_REPLICA)
if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
t.Errorf(`hc.GetEndPointStatsFromTarget("k", "s", REPLICA) = %+v; want %+v`, epsList, want)
}
// RealtimeStats changed
shr = &pbq.StreamHealthResponse{
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
TabletExternallyReparentedTimestamp: 0,
RealtimeStats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
}
want = &EndPointStats{
EndPoint: ep,
Cell: "cell",
Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
Stats: &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
TabletExternallyReparentedTimestamp: 0,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
// remove endpoint
hc.deleteConn(ep)
t.Logf(`hc.RemoveEndPoint({Host: "a", PortMap: {"vt": 1}})`)
epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 0 {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want empty`, epsList)
}
}
func TestHealthCheckTimeout(t *testing.T) {
timeout := 500 * time.Millisecond
ep := topo.NewEndPoint(0, "a")
ep.PortMap["vt"] = 1
input := make(chan *querypb.StreamHealthResponse)
createFakeConn(ep, input)
t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
l := newListener()
hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, timeout, "" /* statsSuffix */).(*HealthCheckImpl)
hc.SetListener(l)
hc.AddEndPoint("cell", "", ep)
t.Logf(`hc = HealthCheck(); hc.AddEndPoint("cell", "", {Host: "a", PortMap: {"vt": 1}})`)
// one endpoint after receiving a StreamHealthResponse
shr := &querypb.StreamHealthResponse{
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Serving: true,
TabletExternallyReparentedTimestamp: 10,
RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
}
want := &EndPointStats{
EndPoint: ep,
Cell: "cell",
Target: &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
Up: true,
Serving: true,
Stats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
TabletExternallyReparentedTimestamp: 10,
}
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res := <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
epsList := hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
}
// wait for timeout period
time.Sleep(2 * timeout)
t.Logf(`Sleep(2 * timeout)`)
res = <-l.output
if res.Serving {
t.Errorf(`<-l.output: %+v; want not serving`, res)
}
epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 1 || epsList[0].Serving {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want not serving`, epsList)
}
// send a healthcheck response, it should be serving again
input <- shr
t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
res = <-l.output
if !reflect.DeepEqual(res, want) {
t.Errorf(`<-l.output: %+v; want %+v`, res, want)
}
epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
}
// close healthcheck
hc.Close()
}