func (fhc *fakeHealthCheck) addTestEndPoint(cell, host string, port int32, keyspace, shard string, tabletType pbt.TabletType, reparentTS int64, err error, conn tabletconn.TabletConn) *pbt.EndPoint {
	ep := topo.NewEndPoint(0, host)
	ep.PortMap["vt"] = port
	key := discovery.EndPointToMapKey(ep)
	item := fhc.items[key]
	if item == nil {
		fhc.AddEndPoint(cell, ep)
		item = fhc.items[key]
	}
	item.eps.Target = &pbq.Target{Keyspace: keyspace, Shard: shard, TabletType: tabletType}
	item.eps.TabletExternallyReparentedTimestamp = reparentTS
	item.eps.LastError = err
	item.conn = conn
	return ep
}
func TestWaitForEndPoints(t *testing.T) {
	shortCtx, shortCancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
	defer shortCancel()
	waitAvailableEndPointInterval = 20 * time.Millisecond

	ep := topo.NewEndPoint(0, "a")
	ep.PortMap["vt"] = 1
	input := make(chan *querypb.StreamHealthResponse)
	createFakeConn(ep, input)

	hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, 1*time.Hour, "" /* statsSuffix */)
	hc.AddEndPoint("cell", "", ep)

	// this should time out
	if err := WaitForEndPoints(shortCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != ErrWaitForEndPointsTimeout {
		t.Errorf("got wrong error: %v", err)
	}

	// this should fail, but return a non-timeout error
	cancelledCtx, cancel := context.WithCancel(context.Background())
	cancel()
	if err := WaitForEndPoints(cancelledCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err == nil || err == ErrWaitForEndPointsTimeout {
		t.Errorf("want: non-timeout error, got: %v", err)
	}

	// send the endpoint in
	shr := &querypb.StreamHealthResponse{
		Target: &querypb.Target{
			Keyspace:   "keyspace",
			Shard:      "shard",
			TabletType: topodatapb.TabletType_REPLICA,
		},
		Serving:       true,
		RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
	}
	input <- shr

	// and ask again, with longer time outs so it's not flaky
	longCtx, longCancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer longCancel()
	waitAvailableEndPointInterval = 10 * time.Millisecond
	if err := WaitForEndPoints(longCtx, hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
		t.Errorf("got error: %v", err)
	}
}
func TestWaitForEndPoints(t *testing.T) {
	waitAvailableEndPointPeriod = 10 * time.Millisecond
	waitAvailableEndPointInterval = 20 * time.Millisecond

	ep := topo.NewEndPoint(0, "a")
	ep.PortMap["vt"] = 1
	input := make(chan *querypb.StreamHealthResponse)
	createFakeConn(ep, input)

	hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, 1*time.Hour)
	hc.AddEndPoint("cell", "", ep)

	// this should time out
	if err := WaitForEndPoints(hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != ErrWaitForEndPointsTimeout {
		t.Errorf("got wrong error: %v", err)
	}

	// send the endpoint in
	shr := &querypb.StreamHealthResponse{
		Target: &querypb.Target{
			Keyspace:   "keyspace",
			Shard:      "shard",
			TabletType: topodatapb.TabletType_REPLICA,
		},
		Serving:       true,
		RealtimeStats: &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
	}
	input <- shr

	// and ask again, with longer time outs so it's not flaky
	waitAvailableEndPointPeriod = 10 * time.Second
	waitAvailableEndPointInterval = 10 * time.Millisecond
	if err := WaitForEndPoints(hc, "cell", "keyspace", "shard", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}); err != nil {
		t.Errorf("got error: %v", err)
	}
}
Exemple #4
0
func TestHealthCheck(t *testing.T) {
	ep := topo.NewEndPoint(0, "a")
	ep.PortMap["vt"] = 1
	input := make(chan *pbq.StreamHealthResponse)
	createFakeConn(ep, input)
	t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
	l := newListener()
	hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond).(*HealthCheckImpl)
	hc.SetListener(l)
	hc.AddEndPoint("cell", "", ep)
	t.Logf(`hc = HealthCheck(); hc.AddEndPoint("cell", "", {Host: "a", PortMap: {"vt": 1}})`)

	// no endpoint before getting first StreamHealthResponse
	epsList := hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 0 {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want empty`, epsList)
	}

	// one endpoint after receiving a StreamHealthResponse
	shr := &pbq.StreamHealthResponse{
		Target:  &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
		Serving: true,
		TabletExternallyReparentedTimestamp: 10,
		RealtimeStats:                       &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
	}
	want := &EndPointStats{
		EndPoint: ep,
		Cell:     "cell",
		Target:   &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
		Serving:  true,
		Stats:    &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
		TabletExternallyReparentedTimestamp: 10,
	}
	input <- shr
	t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
	res := <-l.output
	if !reflect.DeepEqual(res, want) {
		t.Errorf(`<-l.output: %+v; want %+v`, res, want)
	}
	epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
	}
	epcsl := hc.CacheStatus()
	epcslWant := EndPointsCacheStatusList{{
		Cell:   "cell",
		Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
		EndPointsStats: EndPointStatsList{{
			EndPoint: ep,
			Cell:     "cell",
			Target:   &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_MASTER},
			Serving:  true,
			Stats:    &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
			TabletExternallyReparentedTimestamp: 10,
		}},
	}}
	if !reflect.DeepEqual(epcsl, epcslWant) {
		t.Errorf(`hc.CacheStatus() = %+v; want %+v`, epcsl, epcslWant)
	}

	// TabletType changed
	shr = &pbq.StreamHealthResponse{
		Target:  &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
		Serving: true,
		TabletExternallyReparentedTimestamp: 0,
		RealtimeStats:                       &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
	}
	want = &EndPointStats{
		EndPoint: ep,
		Cell:     "cell",
		Target:   &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
		Serving:  true,
		Stats:    &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.5},
		TabletExternallyReparentedTimestamp: 0,
	}
	input <- shr
	t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, Serving: true, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.5}}`)
	res = <-l.output
	if !reflect.DeepEqual(res, want) {
		t.Errorf(`<-l.output: %+v; want %+v`, res, want)
	}
	epsList = hc.GetEndPointStatsFromTarget("k", "s", pbt.TabletType_REPLICA)
	if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
		t.Errorf(`hc.GetEndPointStatsFromTarget("k", "s", REPLICA) = %+v; want %+v`, epsList, want)
	}

	// RealtimeStats changed
	shr = &pbq.StreamHealthResponse{
		Target: &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
		TabletExternallyReparentedTimestamp: 0,
		RealtimeStats:                       &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
	}
	want = &EndPointStats{
		EndPoint: ep,
		Cell:     "cell",
		Target:   &pbq.Target{Keyspace: "k", Shard: "s", TabletType: pbt.TabletType_REPLICA},
		Stats:    &pbq.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.3},
		TabletExternallyReparentedTimestamp: 0,
	}
	input <- shr
	t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: REPLICA}, TabletExternallyReparentedTimestamp: 0, {SecondsBehindMaster: 1, CpuUsage: 0.3}}`)
	res = <-l.output
	if !reflect.DeepEqual(res, want) {
		t.Errorf(`<-l.output: %+v; want %+v`, res, want)
	}

	// remove endpoint
	hc.deleteConn(ep)
	t.Logf(`hc.RemoveEndPoint({Host: "a", PortMap: {"vt": 1}})`)
	epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 0 {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want empty`, epsList)
	}
}
Exemple #5
0
func TestHealthCheckTimeout(t *testing.T) {
	timeout := 500 * time.Millisecond
	ep := topo.NewEndPoint(0, "a")
	ep.PortMap["vt"] = 1
	input := make(chan *querypb.StreamHealthResponse)
	createFakeConn(ep, input)
	t.Logf(`createFakeConn({Host: "a", PortMap: {"vt": 1}}, c)`)
	l := newListener()
	hc := NewHealthCheck(1*time.Millisecond, 1*time.Millisecond, timeout, "" /* statsSuffix */).(*HealthCheckImpl)
	hc.SetListener(l)
	hc.AddEndPoint("cell", "", ep)
	t.Logf(`hc = HealthCheck(); hc.AddEndPoint("cell", "", {Host: "a", PortMap: {"vt": 1}})`)

	// one endpoint after receiving a StreamHealthResponse
	shr := &querypb.StreamHealthResponse{
		Target:  &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
		Serving: true,
		TabletExternallyReparentedTimestamp: 10,
		RealtimeStats:                       &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
	}
	want := &EndPointStats{
		EndPoint: ep,
		Cell:     "cell",
		Target:   &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_MASTER},
		Up:       true,
		Serving:  true,
		Stats:    &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
		TabletExternallyReparentedTimestamp: 10,
	}
	input <- shr
	t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
	res := <-l.output
	if !reflect.DeepEqual(res, want) {
		t.Errorf(`<-l.output: %+v; want %+v`, res, want)
	}
	epsList := hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
	}
	// wait for timeout period
	time.Sleep(2 * timeout)
	t.Logf(`Sleep(2 * timeout)`)
	res = <-l.output
	if res.Serving {
		t.Errorf(`<-l.output: %+v; want not serving`, res)
	}
	epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 1 || epsList[0].Serving {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want not serving`, epsList)
	}
	// send a healthcheck response, it should be serving again
	input <- shr
	t.Logf(`input <- {{Keyspace: "k", Shard: "s", TabletType: MASTER}, Serving: true, TabletExternallyReparentedTimestamp: 10, {SecondsBehindMaster: 1, CpuUsage: 0.2}}`)
	res = <-l.output
	if !reflect.DeepEqual(res, want) {
		t.Errorf(`<-l.output: %+v; want %+v`, res, want)
	}
	epsList = hc.GetEndPointStatsFromKeyspaceShard("k", "s")
	if len(epsList) != 1 || !reflect.DeepEqual(epsList[0], want) {
		t.Errorf(`hc.GetEndPointStatsFromKeyspaceShard("k", "s") = %+v; want %+v`, epsList, want)
	}
	// close healthcheck
	hc.Close()
}