func Test_Receiver_flushDs(t *testing.T) {
// So we need to test that this calls queueblocking...
r := &Receiver{flusherChs: make([]chan *dsFlushRequest, 1), flushLimiter: rate.NewLimiter(10, 10)}
r.flusherChs[0] = make(chan *dsFlushRequest)
called := 0
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
if _, ok := <-r.flusherChs[0]; !ok {
break
}
called++
}
}()
ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(0, 0, "WMEAN", time.Second, 10, 10, 0, time.Time{})
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(10, time.Unix(100, 0))
ds.ProcessIncomingDataPoint(10, time.Unix(101, 0))
rds := &receiverDs{DataSource: ds}
r.SetMaxFlushRate(1)
r.flushDs(rds, false)
r.flushDs(rds, false)
close(r.flusherChs[0])
wg.Wait()
if called != 1 {
t.Errorf("flushDs call count not 1: %d", called)
}
if ds.PointCount() != 0 {
t.Errorf("ClearRRAs was not called by flushDs")
}
}
func roundRobinArchiveFromRow(rows *sql.Rows, dsStep time.Duration) (*rrd.RoundRobinArchive, error) {
var (
latest *time.Time
cf string
value float64
xff float32
id, dsId, durationMs, width, stepsPerRow, size int64
)
err := rows.Scan(&id, &dsId, &cf, &stepsPerRow, &size, &width, &xff, &value, &durationMs, &latest)
if err != nil {
log.Printf("roundRoundRobinArchiveFromRow(): error scanning row: %v", err)
return nil, err
}
if latest == nil {
latest = &time.Time{}
}
rra, err := rrd.NewRoundRobinArchive(id, dsId, cf, time.Duration(stepsPerRow)*dsStep, size, width, xff, *latest)
if err != nil {
log.Printf("roundRoundRobinArchiveFromRow(): error creating rra: %v", err)
return nil, err
}
rra.SetValue(value, time.Duration(durationMs)*time.Millisecond)
return rra, err
}
func Test_receiverDs_shouldBeFlushed(t *testing.T) {
var (
id, dsId, size, width int64
step time.Duration
cf string
xff float32
latest time.Time
)
id, dsId, step, size, width, cf, xff, latest = 1, 3, 10*time.Second, 100, 30, "WMEAN", 0.5, time.Unix(1000, 0)
ds := rrd.NewDataSource(dsId, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(id, dsId, cf, step, size, width, xff, latest)
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
f := &fakeDsFlusher{}
rds := &receiverDs{DataSource: ds, dsf: f, lastFlushRT: time.Now()}
// When rds.LastUpdate().IsZero() it should be false
if rds.shouldBeFlushed(0, 0, 0) {
t.Errorf("with rds.LastUpdate().IsZero(), rds.shouldBeFlushed == true")
}
// this will cause a LastUpdate != 0
ds.ProcessIncomingDataPoint(123, time.Now().Add(-2*time.Hour))
// so far we still have 0 points, so nothing to flush
if rds.shouldBeFlushed(0, 0, 0) {
t.Errorf("with PointCount 0, rds.shouldBeFlushed == true")
}
ds.ProcessIncomingDataPoint(123, time.Now().Add(-time.Hour))
if !rds.shouldBeFlushed(0, 0, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 0, rds.shouldBeFlushed != true")
}
if rds.shouldBeFlushed(1000, 0, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 1000, rds.shouldBeFlushed == true")
}
if rds.shouldBeFlushed(1000, 24*time.Hour, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 1000, minCache 24hr, rds.shouldBeFlushed == true")
}
if !rds.shouldBeFlushed(1000, 0, 0) {
t.Errorf("with maxCachedPoints == 1000, minCache 0, maxCache 0, rds.shouldBeFlushed != true")
}
}
func Test_dispatcherProcessOrForward(t *testing.T) {
saveFn := dispatcherForwardDPToNode
forward, fwErr := 0, error(nil)
dispatcherForwardDPToNode = func(dp *IncomingDP, node *cluster.Node, snd chan *cluster.Msg) error {
forward++
return fwErr
}
// rds
ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
// cluster
clstr := &fakeCluster{}
md := make([]byte, 20)
md[0] = 1 // Ready
node := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "local"}}
clstr.nodesForDd = []*cluster.Node{node}
clstr.ln = node
// workerChs
workerChs := make([]chan *incomingDpWithDs, 1)
workerChs[0] = make(chan *incomingDpWithDs)
sent := 0
go func() {
for {
<-workerChs[0]
sent++
}
}()
// Test if we are LocalNode
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if sent < 1 {
t.Errorf("dispatcherProcessOrForward: Nothing sent to workerChs")
}
// Now test we are NOT LN, forward
remote := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "remote"}}
clstr.nodesForDd = []*cluster.Node{remote}
n := dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if forward != 1 {
t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not called")
}
if n != 1 {
t.Errorf("dispatcherProcessOrForward: return value != 1")
}
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
fwErr = fmt.Errorf("some error")
n = dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if n != 0 {
t.Errorf("dispatcherProcessOrForward: return value != 0")
}
if !strings.Contains(string(fl.last), "some error") {
t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not logged")
}
fwErr = nil
// make an rds with points
ds = rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
rds = &receiverDs{DataSource: ds}
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if !strings.Contains(string(fl.last), "PointCount") {
t.Errorf("dispatcherProcessOrForward: Missing the PointCount warning log")
}
if rds.PointCount() != 0 {
t.Errorf("dispatcherProcessOrForward: ClearRRAs(true) not called")
}
// restore dispatcherForwardDPToNode
dispatcherForwardDPToNode = saveFn
}
func Test_workerPeriodicFlush(t *testing.T) {
// fake logger
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
// dsf
f := &fakeDsFlusher{fdsReturn: true}
// recent
recent := make(map[int64]*receiverDs)
// dsc
db := &fakeSerde{}
df := &dftDSFinder{}
c := &fakeCluster{}
dsc := newDsCache(db, df, c, nil, true)
ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
recent[7] = rds
dsc.insert(rds)
workerPeriodicFlush("workerperiodic2", f, recent, 0, 10*time.Millisecond, 10, 1)
if f.called > 0 {
t.Errorf("workerPeriodicFlush: no flush should have happened")
}
// make an rds with points
ds = rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
rds = &receiverDs{DataSource: ds}
dsc.insert(rds)
recent[7] = rds
debug = true
leftover := workerPeriodicFlush("workerperiodic3", f, recent, 0, 10*time.Millisecond, 0, 1)
if f.called == 0 {
t.Errorf("workerPeriodicFlush: should have called flushDs")
}
if len(recent) != 0 {
t.Errorf("workerPeriodicFlush: should have deleted the point after flushDs")
}
if len(leftover) != 0 {
t.Errorf("workerPeriodicFlush: leftover should be empty")
}
f.fdsReturn = false
f.called = 0
recent[7] = rds
ds.ProcessIncomingDataPoint(123, time.Unix(4000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(5000, 0))
leftover = workerPeriodicFlush("workerperiodic4", f, recent, 0, 10*time.Millisecond, 0, 0)
if f.called == 0 {
t.Errorf("workerPeriodicFlush: should have called flushDs")
}
if len(recent) != 0 {
t.Errorf("workerPeriodicFlush: should have deleted the point on flushDs (2)")
}
if len(leftover) == 0 {
t.Errorf("workerPeriodicFlush: leftover should NOT be empty, it should have the point from recent")
}
}
func Test_theWorker(t *testing.T) {
// fake logger
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
ident := "FOO"
wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: ident}
dsf := &fakeDsFlusher{fdsReturn: true}
workerCh := make(chan *incomingDpWithDs)
saveFn1 := workerPeriodicFlush
wpfCalled := 0
workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlush int) map[int64]*receiverDs {
wpfCalled++
return map[int64]*receiverDs{1: nil, 2: nil}
}
sr := &fakeSr{}
wc.startWg.Add(1)
go worker(wc, dsf, workerCh, 0, 0, 10, 10*time.Millisecond, sr)
wc.startWg.Wait()
if !strings.Contains(string(fl.last), ident) {
t.Errorf("worker: missing worker started log entry for ident: %s", ident)
}
debug = true
// make an rds with points
ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
rds := &receiverDs{DataSource: ds}
dp := &IncomingDP{Name: "foo", TimeStamp: time.Unix(2000, 0), Value: 123}
workerCh <- &incomingDpWithDs{dp, rds}
dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(3000, 0), Value: 123}
workerCh <- &incomingDpWithDs{dp, rds}
pc := ds.PointCount()
if pc == 0 {
t.Errorf("After dps being sent to workerCh, ds should have some points: %d", pc)
}
time.Sleep(50 * time.Millisecond) // wait for a flush or two
if wpfCalled == 0 {
t.Errorf("worker: at least one periodic flush should have happened")
}
// trigger an error
dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(5000, 0), Value: math.Inf(-1)}
workerCh <- &incomingDpWithDs{dp, rds}
close(workerCh)
wc.wg.Wait()
if !strings.Contains(string(fl.last), "not a valid data point") {
t.Errorf("worker: missing 'not a valid data point' log entry")
}
// restore funcs
workerPeriodicFlush = saveFn1
}