// TestMetricsRecording verifies that Node statistics are periodically recorded
// as time series data.
func TestMetricsRecording(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, kvDB := serverutils.StartServer(t, base.TestServerArgs{
MetricsSampleInterval: 5 * time.Millisecond})
defer s.Stopper().Stop()
checkTimeSeriesKey := func(now int64, keyName string) error {
key := ts.MakeDataKey(keyName, "", ts.Resolution10s, now)
data := roachpb.InternalTimeSeriesData{}
return kvDB.GetProto(context.TODO(), key, &data)
}
// Verify that metrics for the current timestamp are recorded. This should
// be true very quickly.
util.SucceedsSoon(t, func() error {
now := s.Clock().PhysicalNow()
if err := checkTimeSeriesKey(now, "cr.store.livebytes.1"); err != nil {
return err
}
if err := checkTimeSeriesKey(now, "cr.node.sys.go.allocbytes.1"); err != nil {
return err
}
return nil
})
}
// TestTimeSeriesMaintenanceQueueServer verifies that the time series
// maintenance queue runs correctly on a test server.
func TestTimeSeriesMaintenanceQueueServer(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, db := serverutils.StartServer(t, base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
DisableScanner: true,
},
},
})
defer s.Stopper().Stop()
tsrv := s.(*server.TestServer)
tsdb := tsrv.TsDB()
// Populate time series data into the server. One time series, with one
// datapoint at the current time and two datapoints older than the pruning
// threshold. Datapoint timestamps are set to the midpoint of sample duration
// periods; this simplifies verification.
seriesName := "test.metric"
sourceName := "source1"
now := tsrv.Clock().PhysicalNow()
nearPast := now - (ts.Resolution10s.PruneThreshold() * 2)
farPast := now - (ts.Resolution10s.PruneThreshold() * 4)
sampleDuration := ts.Resolution10s.SampleDuration()
datapoints := []tspb.TimeSeriesDatapoint{
{
TimestampNanos: farPast - farPast%sampleDuration + sampleDuration/2,
Value: 100.0,
},
{
TimestampNanos: nearPast - (nearPast)%sampleDuration + sampleDuration/2,
Value: 200.0,
},
{
TimestampNanos: now - now%sampleDuration + sampleDuration/2,
Value: 300.0,
},
}
if err := tsdb.StoreData(context.TODO(), ts.Resolution10s, []tspb.TimeSeriesData{
{
Name: seriesName,
Source: sourceName,
Datapoints: datapoints,
},
}); err != nil {
t.Fatal(err)
}
// Generate a split key at a timestamp halfway between near past and far past.
splitKey := ts.MakeDataKey(
seriesName, sourceName, ts.Resolution10s, farPast+(nearPast-farPast)/2,
)
// Force a range split in between near past and far past. This guarantees
// that the pruning operation will issue a DeleteRange which spans ranges.
if err := db.AdminSplit(context.TODO(), splitKey); err != nil {
t.Fatal(err)
}
// getDatapoints queries all datapoints in the series from the beginning
// of time to a point in the near future.
getDatapoints := func() ([]tspb.TimeSeriesDatapoint, error) {
dps, _, err := tsdb.Query(
context.TODO(),
tspb.Query{Name: seriesName},
ts.Resolution10s,
ts.Resolution10s.SampleDuration(),
0,
now+ts.Resolution10s.SlabDuration(),
)
return dps, err
}
// Verify the datapoints are all present.
actualDatapoints, err := getDatapoints()
if err != nil {
t.Fatal(err)
}
if a, e := actualDatapoints, datapoints; !reflect.DeepEqual(a, e) {
t.Fatalf("got datapoints %v, expected %v, diff: %s", a, e, pretty.Diff(a, e))
}
// Force pruning.
storeID := roachpb.StoreID(1)
store, err := tsrv.Stores().GetStore(roachpb.StoreID(1))
if err != nil {
t.Fatalf("error retrieving store %d: %s", storeID, err)
}
store.ForceTimeSeriesMaintenanceQueueProcess()
// Verify the older datapoint has been pruned.
util.SucceedsSoon(t, func() error {
actualDatapoints, err = getDatapoints()
if err != nil {
return err
}
if a, e := actualDatapoints, datapoints[2:]; !reflect.DeepEqual(a, e) {
return fmt.Errorf("got datapoints %v, expected %v, diff: %s", a, e, pretty.Diff(a, e))
}
return nil
})
}