func Test_Receiver_flushDs(t *testing.T) {
// So we need to test that this calls queueblocking...
r := &Receiver{flusherChs: make([]chan *dsFlushRequest, 1), flushLimiter: rate.NewLimiter(10, 10)}
r.flusherChs[0] = make(chan *dsFlushRequest)
called := 0
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
if _, ok := <-r.flusherChs[0]; !ok {
break
}
called++
}
}()
ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(0, 0, "WMEAN", time.Second, 10, 10, 0, time.Time{})
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(10, time.Unix(100, 0))
ds.ProcessIncomingDataPoint(10, time.Unix(101, 0))
rds := &receiverDs{DataSource: ds}
r.SetMaxFlushRate(1)
r.flushDs(rds, false)
r.flushDs(rds, false)
close(r.flusherChs[0])
wg.Wait()
if called != 1 {
t.Errorf("flushDs call count not 1: %d", called)
}
if ds.PointCount() != 0 {
t.Errorf("ClearRRAs was not called by flushDs")
}
}
func dataSourceFromRow(rows *sql.Rows) (*rrd.DataSource, error) {
var (
last_ds *float64
lastupdate *time.Time
durationMs, stepMs, hbMs int64
value float64
id int64
name string
)
err := rows.Scan(&id, &name, &stepMs, &hbMs, &lastupdate, &last_ds, &value, &durationMs)
if err != nil {
log.Printf("dataSourceFromRow(): error scanning row: %v", err)
return nil, err
}
if lastupdate == nil {
lastupdate = &time.Time{}
}
if last_ds == nil {
last_ds = new(float64)
}
ds := rrd.NewDataSource(id, name,
time.Duration(stepMs)*time.Millisecond,
time.Duration(hbMs)*time.Millisecond,
*lastupdate,
*last_ds)
ds.SetValue(value, time.Duration(durationMs)*time.Millisecond)
return ds, err
}
func (f *fakeSerde) CreateOrReturnDataSource(name string, dsSpec *serde.DSSpec) (*rrd.DataSource, error) {
f.createCalled++
if f.fakeErr {
return nil, fmt.Errorf("some error")
}
return rrd.NewDataSource(7, name, 0, 0, time.Time{}, 0), nil
}
func Test_flusher(t *testing.T) {
wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: "FOO"}
dsf := &fFlusher{}
sr := &fakeSr{}
fc := make(chan *dsFlushRequest)
wc.startWg.Add(1)
go flusher(wc, dsf, sr, fc)
wc.startWg.Wait()
ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
resp := make(chan bool)
fc <- &dsFlushRequest{ds: ds, resp: resp}
<-resp
if dsf.called != 1 {
t.Errorf("FlushDataSource() not called.")
}
if sr.called != 2 {
t.Errorf("reportStatCount() should have been called 2 times.")
}
close(fc)
wc.wg.Wait()
}
func Test_dsCache_insert(t *testing.T) {
d := newDsCache(nil, nil, nil, nil, true)
ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
d.insert(rds)
if rds2 := d.getById(7); rds2 != rds {
t.Errorf("insert: getById did not return correct value")
}
if rds2 := d.getByName("foo"); rds2 != rds {
t.Errorf("insert: getByName did not return correct value")
}
}
func Test_receiverDs_shouldBeFlushed(t *testing.T) {
var (
id, dsId, size, width int64
step time.Duration
cf string
xff float32
latest time.Time
)
id, dsId, step, size, width, cf, xff, latest = 1, 3, 10*time.Second, 100, 30, "WMEAN", 0.5, time.Unix(1000, 0)
ds := rrd.NewDataSource(dsId, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(id, dsId, cf, step, size, width, xff, latest)
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
f := &fakeDsFlusher{}
rds := &receiverDs{DataSource: ds, dsf: f, lastFlushRT: time.Now()}
// When rds.LastUpdate().IsZero() it should be false
if rds.shouldBeFlushed(0, 0, 0) {
t.Errorf("with rds.LastUpdate().IsZero(), rds.shouldBeFlushed == true")
}
// this will cause a LastUpdate != 0
ds.ProcessIncomingDataPoint(123, time.Now().Add(-2*time.Hour))
// so far we still have 0 points, so nothing to flush
if rds.shouldBeFlushed(0, 0, 0) {
t.Errorf("with PointCount 0, rds.shouldBeFlushed == true")
}
ds.ProcessIncomingDataPoint(123, time.Now().Add(-time.Hour))
if !rds.shouldBeFlushed(0, 0, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 0, rds.shouldBeFlushed != true")
}
if rds.shouldBeFlushed(1000, 0, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 1000, rds.shouldBeFlushed == true")
}
if rds.shouldBeFlushed(1000, 24*time.Hour, 24*time.Hour) {
t.Errorf("with maxCachedPoints == 1000, minCache 24hr, rds.shouldBeFlushed == true")
}
if !rds.shouldBeFlushed(1000, 0, 0) {
t.Errorf("with maxCachedPoints == 1000, minCache 0, maxCache 0, rds.shouldBeFlushed != true")
}
}
func Test_workerChannels_queue(t *testing.T) {
var wcs workerChannels = make([]chan *incomingDpWithDs, 2)
wcs[0] = make(chan *incomingDpWithDs)
wcs[1] = make(chan *incomingDpWithDs)
ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
called := 0
go func() {
<-wcs[0]
called++
}()
wcs.queue(nil, rds)
if called != 1 {
t.Errorf("id 0 should be send to worker 0")
}
}
func Test_receiverDs_Relinquish(t *testing.T) {
ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
f := &fakeDsFlusher{}
rds := &receiverDs{DataSource: ds, dsf: f}
err := rds.Relinquish()
if err != nil {
t.Errorf("rds.Relinquish: err != nil: %v", err)
}
if f.called != 0 {
t.Errorf("if lastupdate is zero, ds should not be flushed")
}
ds.ProcessIncomingDataPoint(123, time.Unix(1000, 0))
err = rds.Relinquish()
if err != nil {
t.Errorf("rds.Relinquish (2): err != nil: %v", err)
}
if f.called != 1 {
t.Errorf("if lastupdate is not zero, ds should be flushed")
}
// test Acquire while we're at it
err = rds.Acquire()
if err != nil {
t.Errorf("Acquire: err != nil")
}
if !rds.stale {
t.Errorf("Acquire: !rds.stale")
}
if f.called != 1 {
t.Errorf("Acquire: should not call flush")
}
// receiverDs methods
if rds.Type() != "DataSource" {
t.Errorf(`rds.Type() != "DataSource"`)
}
if rds.GetName() != "foo" {
t.Errorf(`rds.GetName() != "foo"`)
}
}
func Test_flusherChannels_queueBlocking(t *testing.T) {
var fcs flusherChannels = make([]chan *dsFlushRequest, 2)
fcs[0] = make(chan *dsFlushRequest)
fcs[1] = make(chan *dsFlushRequest)
called := 0
go func() {
fr := <-fcs[0]
fr.resp <- true
called++
}()
ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
fcs.queueBlocking(rds, true)
if called != 1 {
t.Errorf("id 0 should be send to flusher 0")
}
}
func Test_dsCache_preLoad(t *testing.T) {
db := &fakeSerde{}
d := newDsCache(db, nil, nil, nil, true)
d.preLoad()
if db.fetchCalled == 0 {
t.Errorf("db.fetchCalled == 0")
}
ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
db.returnDss = []*rrd.DataSource{ds}
d.preLoad()
if len(d.byName) == 0 {
t.Errorf("len(d.byName) == 0")
}
db.fakeErr = true
if err := d.preLoad(); err == nil {
t.Errorf("preLoad: err == nil")
}
}
func Test_dispatcherProcessOrForward(t *testing.T) {
saveFn := dispatcherForwardDPToNode
forward, fwErr := 0, error(nil)
dispatcherForwardDPToNode = func(dp *IncomingDP, node *cluster.Node, snd chan *cluster.Msg) error {
forward++
return fwErr
}
// rds
ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
// cluster
clstr := &fakeCluster{}
md := make([]byte, 20)
md[0] = 1 // Ready
node := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "local"}}
clstr.nodesForDd = []*cluster.Node{node}
clstr.ln = node
// workerChs
workerChs := make([]chan *incomingDpWithDs, 1)
workerChs[0] = make(chan *incomingDpWithDs)
sent := 0
go func() {
for {
<-workerChs[0]
sent++
}
}()
// Test if we are LocalNode
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if sent < 1 {
t.Errorf("dispatcherProcessOrForward: Nothing sent to workerChs")
}
// Now test we are NOT LN, forward
remote := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "remote"}}
clstr.nodesForDd = []*cluster.Node{remote}
n := dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if forward != 1 {
t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not called")
}
if n != 1 {
t.Errorf("dispatcherProcessOrForward: return value != 1")
}
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
fwErr = fmt.Errorf("some error")
n = dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if n != 0 {
t.Errorf("dispatcherProcessOrForward: return value != 0")
}
if !strings.Contains(string(fl.last), "some error") {
t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not logged")
}
fwErr = nil
// make an rds with points
ds = rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
rds = &receiverDs{DataSource: ds}
dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
if !strings.Contains(string(fl.last), "PointCount") {
t.Errorf("dispatcherProcessOrForward: Missing the PointCount warning log")
}
if rds.PointCount() != 0 {
t.Errorf("dispatcherProcessOrForward: ClearRRAs(true) not called")
}
// restore dispatcherForwardDPToNode
dispatcherForwardDPToNode = saveFn
}
func Test_workerPeriodicFlush(t *testing.T) {
// fake logger
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
// dsf
f := &fakeDsFlusher{fdsReturn: true}
// recent
recent := make(map[int64]*receiverDs)
// dsc
db := &fakeSerde{}
df := &dftDSFinder{}
c := &fakeCluster{}
dsc := newDsCache(db, df, c, nil, true)
ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
rds := &receiverDs{DataSource: ds}
recent[7] = rds
dsc.insert(rds)
workerPeriodicFlush("workerperiodic2", f, recent, 0, 10*time.Millisecond, 10, 1)
if f.called > 0 {
t.Errorf("workerPeriodicFlush: no flush should have happened")
}
// make an rds with points
ds = rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
rds = &receiverDs{DataSource: ds}
dsc.insert(rds)
recent[7] = rds
debug = true
leftover := workerPeriodicFlush("workerperiodic3", f, recent, 0, 10*time.Millisecond, 0, 1)
if f.called == 0 {
t.Errorf("workerPeriodicFlush: should have called flushDs")
}
if len(recent) != 0 {
t.Errorf("workerPeriodicFlush: should have deleted the point after flushDs")
}
if len(leftover) != 0 {
t.Errorf("workerPeriodicFlush: leftover should be empty")
}
f.fdsReturn = false
f.called = 0
recent[7] = rds
ds.ProcessIncomingDataPoint(123, time.Unix(4000, 0))
ds.ProcessIncomingDataPoint(123, time.Unix(5000, 0))
leftover = workerPeriodicFlush("workerperiodic4", f, recent, 0, 10*time.Millisecond, 0, 0)
if f.called == 0 {
t.Errorf("workerPeriodicFlush: should have called flushDs")
}
if len(recent) != 0 {
t.Errorf("workerPeriodicFlush: should have deleted the point on flushDs (2)")
}
if len(leftover) == 0 {
t.Errorf("workerPeriodicFlush: leftover should NOT be empty, it should have the point from recent")
}
}
func Test_theWorker(t *testing.T) {
// fake logger
fl := &fakeLogger{}
log.SetOutput(fl)
defer func() {
// restore default output
log.SetOutput(os.Stderr)
}()
ident := "FOO"
wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: ident}
dsf := &fakeDsFlusher{fdsReturn: true}
workerCh := make(chan *incomingDpWithDs)
saveFn1 := workerPeriodicFlush
wpfCalled := 0
workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlush int) map[int64]*receiverDs {
wpfCalled++
return map[int64]*receiverDs{1: nil, 2: nil}
}
sr := &fakeSr{}
wc.startWg.Add(1)
go worker(wc, dsf, workerCh, 0, 0, 10, 10*time.Millisecond, sr)
wc.startWg.Wait()
if !strings.Contains(string(fl.last), ident) {
t.Errorf("worker: missing worker started log entry for ident: %s", ident)
}
debug = true
// make an rds with points
ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
rds := &receiverDs{DataSource: ds}
dp := &IncomingDP{Name: "foo", TimeStamp: time.Unix(2000, 0), Value: 123}
workerCh <- &incomingDpWithDs{dp, rds}
dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(3000, 0), Value: 123}
workerCh <- &incomingDpWithDs{dp, rds}
pc := ds.PointCount()
if pc == 0 {
t.Errorf("After dps being sent to workerCh, ds should have some points: %d", pc)
}
time.Sleep(50 * time.Millisecond) // wait for a flush or two
if wpfCalled == 0 {
t.Errorf("worker: at least one periodic flush should have happened")
}
// trigger an error
dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(5000, 0), Value: math.Inf(-1)}
workerCh <- &incomingDpWithDs{dp, rds}
close(workerCh)
wc.wg.Wait()
if !strings.Contains(string(fl.last), "not a valid data point") {
t.Errorf("worker: missing 'not a valid data point' log entry")
}
// restore funcs
workerPeriodicFlush = saveFn1
}
