예제 #1
0
func Test_Receiver_flushDs(t *testing.T) {
	// So we need to test that this calls queueblocking...
	r := &Receiver{flusherChs: make([]chan *dsFlushRequest, 1), flushLimiter: rate.NewLimiter(10, 10)}
	r.flusherChs[0] = make(chan *dsFlushRequest)
	called := 0
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			if _, ok := <-r.flusherChs[0]; !ok {
				break
			}
			called++
		}
	}()
	ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(0, 0, "WMEAN", time.Second, 10, 10, 0, time.Time{})
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(10, time.Unix(100, 0))
	ds.ProcessIncomingDataPoint(10, time.Unix(101, 0))
	rds := &receiverDs{DataSource: ds}
	r.SetMaxFlushRate(1)
	r.flushDs(rds, false)
	r.flushDs(rds, false)
	close(r.flusherChs[0])
	wg.Wait()
	if called != 1 {
		t.Errorf("flushDs call count not 1: %d", called)
	}
	if ds.PointCount() != 0 {
		t.Errorf("ClearRRAs was not called by flushDs")
	}
}
예제 #2
0
파일: postgres.go 프로젝트: tgres/tgres
func dataSourceFromRow(rows *sql.Rows) (*rrd.DataSource, error) {
	var (
		last_ds                  *float64
		lastupdate               *time.Time
		durationMs, stepMs, hbMs int64
		value                    float64
		id                       int64
		name                     string
	)
	err := rows.Scan(&id, &name, &stepMs, &hbMs, &lastupdate, &last_ds, &value, &durationMs)
	if err != nil {
		log.Printf("dataSourceFromRow(): error scanning row: %v", err)
		return nil, err
	}

	if lastupdate == nil {
		lastupdate = &time.Time{}
	}
	if last_ds == nil {
		last_ds = new(float64)
	}

	ds := rrd.NewDataSource(id, name,
		time.Duration(stepMs)*time.Millisecond,
		time.Duration(hbMs)*time.Millisecond,
		*lastupdate,
		*last_ds)

	ds.SetValue(value, time.Duration(durationMs)*time.Millisecond)
	return ds, err
}
예제 #3
0
파일: dscache_test.go 프로젝트: tgres/tgres
func (f *fakeSerde) CreateOrReturnDataSource(name string, dsSpec *serde.DSSpec) (*rrd.DataSource, error) {
	f.createCalled++
	if f.fakeErr {
		return nil, fmt.Errorf("some error")
	}
	return rrd.NewDataSource(7, name, 0, 0, time.Time{}, 0), nil
}
예제 #4
0
파일: flusher_test.go 프로젝트: tgres/tgres
func Test_flusher(t *testing.T) {

	wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: "FOO"}
	dsf := &fFlusher{}
	sr := &fakeSr{}
	fc := make(chan *dsFlushRequest)

	wc.startWg.Add(1)
	go flusher(wc, dsf, sr, fc)
	wc.startWg.Wait()

	ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
	resp := make(chan bool)
	fc <- &dsFlushRequest{ds: ds, resp: resp}
	<-resp

	if dsf.called != 1 {
		t.Errorf("FlushDataSource() not called.")
	}

	if sr.called != 2 {
		t.Errorf("reportStatCount() should have been called 2 times.")
	}

	close(fc)
	wc.wg.Wait()
}
예제 #5
0
파일: dscache_test.go 프로젝트: tgres/tgres
func Test_dsCache_insert(t *testing.T) {
	d := newDsCache(nil, nil, nil, nil, true)

	ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}

	d.insert(rds)
	if rds2 := d.getById(7); rds2 != rds {
		t.Errorf("insert: getById did not return correct value")
	}
	if rds2 := d.getByName("foo"); rds2 != rds {
		t.Errorf("insert: getByName did not return correct value")
	}
}
예제 #6
0
파일: dscache_test.go 프로젝트: tgres/tgres
func Test_receiverDs_shouldBeFlushed(t *testing.T) {
	var (
		id, dsId, size, width int64
		step                  time.Duration
		cf                    string
		xff                   float32
		latest                time.Time
	)

	id, dsId, step, size, width, cf, xff, latest = 1, 3, 10*time.Second, 100, 30, "WMEAN", 0.5, time.Unix(1000, 0)
	ds := rrd.NewDataSource(dsId, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(id, dsId, cf, step, size, width, xff, latest)
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	f := &fakeDsFlusher{}
	rds := &receiverDs{DataSource: ds, dsf: f, lastFlushRT: time.Now()}

	// When rds.LastUpdate().IsZero() it should be false
	if rds.shouldBeFlushed(0, 0, 0) {
		t.Errorf("with rds.LastUpdate().IsZero(), rds.shouldBeFlushed == true")
	}

	// this will cause a LastUpdate != 0
	ds.ProcessIncomingDataPoint(123, time.Now().Add(-2*time.Hour))

	// so far we still have 0 points, so nothing to flush
	if rds.shouldBeFlushed(0, 0, 0) {
		t.Errorf("with PointCount 0, rds.shouldBeFlushed == true")
	}

	ds.ProcessIncomingDataPoint(123, time.Now().Add(-time.Hour))

	if !rds.shouldBeFlushed(0, 0, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 0, rds.shouldBeFlushed != true")
	}

	if rds.shouldBeFlushed(1000, 0, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 1000, rds.shouldBeFlushed == true")
	}

	if rds.shouldBeFlushed(1000, 24*time.Hour, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 1000, minCache 24hr, rds.shouldBeFlushed == true")
	}

	if !rds.shouldBeFlushed(1000, 0, 0) {
		t.Errorf("with maxCachedPoints == 1000, minCache 0, maxCache 0, rds.shouldBeFlushed != true")
	}

}
예제 #7
0
func Test_workerChannels_queue(t *testing.T) {
	var wcs workerChannels = make([]chan *incomingDpWithDs, 2)
	wcs[0] = make(chan *incomingDpWithDs)
	wcs[1] = make(chan *incomingDpWithDs)

	ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}
	called := 0
	go func() {
		<-wcs[0]
		called++
	}()
	wcs.queue(nil, rds)
	if called != 1 {
		t.Errorf("id 0 should be send to worker 0")
	}
}
예제 #8
0
파일: dscache_test.go 프로젝트: tgres/tgres
func Test_receiverDs_Relinquish(t *testing.T) {
	ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	f := &fakeDsFlusher{}
	rds := &receiverDs{DataSource: ds, dsf: f}

	err := rds.Relinquish()
	if err != nil {
		t.Errorf("rds.Relinquish: err != nil: %v", err)
	}
	if f.called != 0 {
		t.Errorf("if lastupdate is zero, ds should not be flushed")
	}

	ds.ProcessIncomingDataPoint(123, time.Unix(1000, 0))
	err = rds.Relinquish()
	if err != nil {
		t.Errorf("rds.Relinquish (2): err != nil: %v", err)
	}
	if f.called != 1 {
		t.Errorf("if lastupdate is not zero, ds should be flushed")
	}

	// test Acquire while we're at it
	err = rds.Acquire()
	if err != nil {
		t.Errorf("Acquire: err != nil")
	}
	if !rds.stale {
		t.Errorf("Acquire: !rds.stale")
	}
	if f.called != 1 {
		t.Errorf("Acquire: should not call flush")
	}

	// receiverDs methods
	if rds.Type() != "DataSource" {
		t.Errorf(`rds.Type() != "DataSource"`)
	}

	if rds.GetName() != "foo" {
		t.Errorf(`rds.GetName() != "foo"`)
	}
}
예제 #9
0
파일: flusher_test.go 프로젝트: tgres/tgres
func Test_flusherChannels_queueBlocking(t *testing.T) {
	var fcs flusherChannels = make([]chan *dsFlushRequest, 2)
	fcs[0] = make(chan *dsFlushRequest)
	fcs[1] = make(chan *dsFlushRequest)

	called := 0
	go func() {
		fr := <-fcs[0]
		fr.resp <- true
		called++
	}()

	ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}
	fcs.queueBlocking(rds, true)

	if called != 1 {
		t.Errorf("id 0 should be send to flusher 0")
	}
}
예제 #10
0
파일: dscache_test.go 프로젝트: tgres/tgres
func Test_dsCache_preLoad(t *testing.T) {
	db := &fakeSerde{}
	d := newDsCache(db, nil, nil, nil, true)

	d.preLoad()
	if db.fetchCalled == 0 {
		t.Errorf("db.fetchCalled == 0")
	}

	ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	db.returnDss = []*rrd.DataSource{ds}
	d.preLoad()
	if len(d.byName) == 0 {
		t.Errorf("len(d.byName) == 0")
	}

	db.fakeErr = true
	if err := d.preLoad(); err == nil {
		t.Errorf("preLoad: err == nil")
	}
}
예제 #11
0
func Test_dispatcherProcessOrForward(t *testing.T) {

	saveFn := dispatcherForwardDPToNode
	forward, fwErr := 0, error(nil)
	dispatcherForwardDPToNode = func(dp *IncomingDP, node *cluster.Node, snd chan *cluster.Msg) error {
		forward++
		return fwErr
	}

	// rds
	ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}

	// cluster
	clstr := &fakeCluster{}
	md := make([]byte, 20)
	md[0] = 1 // Ready
	node := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "local"}}
	clstr.nodesForDd = []*cluster.Node{node}
	clstr.ln = node

	// workerChs
	workerChs := make([]chan *incomingDpWithDs, 1)
	workerChs[0] = make(chan *incomingDpWithDs)
	sent := 0
	go func() {
		for {
			<-workerChs[0]
			sent++
		}
	}()

	// Test if we are LocalNode
	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if sent < 1 {
		t.Errorf("dispatcherProcessOrForward: Nothing sent to workerChs")
	}

	// Now test we are NOT LN, forward
	remote := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "remote"}}
	clstr.nodesForDd = []*cluster.Node{remote}

	n := dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if forward != 1 {
		t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not called")
	}
	if n != 1 {
		t.Errorf("dispatcherProcessOrForward: return value != 1")
	}

	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	fwErr = fmt.Errorf("some error")
	n = dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if n != 0 {
		t.Errorf("dispatcherProcessOrForward: return value != 0")
	}
	if !strings.Contains(string(fl.last), "some error") {
		t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not logged")
	}
	fwErr = nil

	// make an rds with points
	ds = rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
	rds = &receiverDs{DataSource: ds}

	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if !strings.Contains(string(fl.last), "PointCount") {
		t.Errorf("dispatcherProcessOrForward: Missing the PointCount warning log")
	}
	if rds.PointCount() != 0 {
		t.Errorf("dispatcherProcessOrForward: ClearRRAs(true) not called")
	}

	// restore dispatcherForwardDPToNode
	dispatcherForwardDPToNode = saveFn
}
예제 #12
0
파일: worker_test.go 프로젝트: tgres/tgres
func Test_workerPeriodicFlush(t *testing.T) {

	// fake logger
	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	// dsf
	f := &fakeDsFlusher{fdsReturn: true}

	// recent
	recent := make(map[int64]*receiverDs)

	// dsc
	db := &fakeSerde{}
	df := &dftDSFinder{}
	c := &fakeCluster{}
	dsc := newDsCache(db, df, c, nil, true)

	ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}
	recent[7] = rds
	dsc.insert(rds)

	workerPeriodicFlush("workerperiodic2", f, recent, 0, 10*time.Millisecond, 10, 1)

	if f.called > 0 {
		t.Errorf("workerPeriodicFlush: no flush should have happened")
	}

	// make an rds with points
	ds = rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
	rds = &receiverDs{DataSource: ds}
	dsc.insert(rds)
	recent[7] = rds
	debug = true

	leftover := workerPeriodicFlush("workerperiodic3", f, recent, 0, 10*time.Millisecond, 0, 1)
	if f.called == 0 {
		t.Errorf("workerPeriodicFlush: should have called flushDs")
	}
	if len(recent) != 0 {
		t.Errorf("workerPeriodicFlush: should have deleted the point after flushDs")
	}
	if len(leftover) != 0 {
		t.Errorf("workerPeriodicFlush: leftover should be empty")
	}

	f.fdsReturn = false
	f.called = 0
	recent[7] = rds
	ds.ProcessIncomingDataPoint(123, time.Unix(4000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(5000, 0))
	leftover = workerPeriodicFlush("workerperiodic4", f, recent, 0, 10*time.Millisecond, 0, 0)
	if f.called == 0 {
		t.Errorf("workerPeriodicFlush: should have called flushDs")
	}
	if len(recent) != 0 {
		t.Errorf("workerPeriodicFlush: should have deleted the point on flushDs (2)")
	}
	if len(leftover) == 0 {
		t.Errorf("workerPeriodicFlush: leftover should NOT be empty, it should have the point from recent")
	}

}
예제 #13
0
파일: worker_test.go 프로젝트: tgres/tgres
func Test_theWorker(t *testing.T) {

	// fake logger
	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	ident := "FOO"
	wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: ident}
	dsf := &fakeDsFlusher{fdsReturn: true}
	workerCh := make(chan *incomingDpWithDs)

	saveFn1 := workerPeriodicFlush

	wpfCalled := 0
	workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlush int) map[int64]*receiverDs {
		wpfCalled++
		return map[int64]*receiverDs{1: nil, 2: nil}
	}

	sr := &fakeSr{}

	wc.startWg.Add(1)
	go worker(wc, dsf, workerCh, 0, 0, 10, 10*time.Millisecond, sr)
	wc.startWg.Wait()

	if !strings.Contains(string(fl.last), ident) {
		t.Errorf("worker: missing worker started log entry for ident: %s", ident)
	}

	debug = true

	// make an rds with points
	ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	rds := &receiverDs{DataSource: ds}
	dp := &IncomingDP{Name: "foo", TimeStamp: time.Unix(2000, 0), Value: 123}
	workerCh <- &incomingDpWithDs{dp, rds}
	dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(3000, 0), Value: 123}
	workerCh <- &incomingDpWithDs{dp, rds}

	pc := ds.PointCount()
	if pc == 0 {
		t.Errorf("After dps being sent to workerCh, ds should have some points: %d", pc)
	}

	time.Sleep(50 * time.Millisecond) // wait for a flush or two
	if wpfCalled == 0 {
		t.Errorf("worker: at least one periodic flush should have happened")
	}

	// trigger an error
	dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(5000, 0), Value: math.Inf(-1)}
	workerCh <- &incomingDpWithDs{dp, rds}

	close(workerCh)
	wc.wg.Wait()

	if !strings.Contains(string(fl.last), "not a valid data point") {
		t.Errorf("worker: missing 'not a valid data point' log entry")
	}

	// restore funcs
	workerPeriodicFlush = saveFn1
}