예제 #1
0
func Test_Receiver_flushDs(t *testing.T) {
	// So we need to test that this calls queueblocking...
	r := &Receiver{flusherChs: make([]chan *dsFlushRequest, 1), flushLimiter: rate.NewLimiter(10, 10)}
	r.flusherChs[0] = make(chan *dsFlushRequest)
	called := 0
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			if _, ok := <-r.flusherChs[0]; !ok {
				break
			}
			called++
		}
	}()
	ds := rrd.NewDataSource(0, "", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(0, 0, "WMEAN", time.Second, 10, 10, 0, time.Time{})
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(10, time.Unix(100, 0))
	ds.ProcessIncomingDataPoint(10, time.Unix(101, 0))
	rds := &receiverDs{DataSource: ds}
	r.SetMaxFlushRate(1)
	r.flushDs(rds, false)
	r.flushDs(rds, false)
	close(r.flusherChs[0])
	wg.Wait()
	if called != 1 {
		t.Errorf("flushDs call count not 1: %d", called)
	}
	if ds.PointCount() != 0 {
		t.Errorf("ClearRRAs was not called by flushDs")
	}
}
예제 #2
0
파일: postgres.go 프로젝트: tgres/tgres
func roundRobinArchiveFromRow(rows *sql.Rows, dsStep time.Duration) (*rrd.RoundRobinArchive, error) {
	var (
		latest *time.Time
		cf     string
		value  float64
		xff    float32

		id, dsId, durationMs, width, stepsPerRow, size int64
	)
	err := rows.Scan(&id, &dsId, &cf, &stepsPerRow, &size, &width, &xff, &value, &durationMs, &latest)
	if err != nil {
		log.Printf("roundRoundRobinArchiveFromRow(): error scanning row: %v", err)
		return nil, err
	}

	if latest == nil {
		latest = &time.Time{}
	}
	rra, err := rrd.NewRoundRobinArchive(id, dsId, cf, time.Duration(stepsPerRow)*dsStep, size, width, xff, *latest)
	if err != nil {
		log.Printf("roundRoundRobinArchiveFromRow(): error creating rra: %v", err)
		return nil, err
	}

	rra.SetValue(value, time.Duration(durationMs)*time.Millisecond)
	return rra, err
}
예제 #3
0
파일: dscache_test.go 프로젝트: tgres/tgres
func Test_receiverDs_shouldBeFlushed(t *testing.T) {
	var (
		id, dsId, size, width int64
		step                  time.Duration
		cf                    string
		xff                   float32
		latest                time.Time
	)

	id, dsId, step, size, width, cf, xff, latest = 1, 3, 10*time.Second, 100, 30, "WMEAN", 0.5, time.Unix(1000, 0)
	ds := rrd.NewDataSource(dsId, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(id, dsId, cf, step, size, width, xff, latest)
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	f := &fakeDsFlusher{}
	rds := &receiverDs{DataSource: ds, dsf: f, lastFlushRT: time.Now()}

	// When rds.LastUpdate().IsZero() it should be false
	if rds.shouldBeFlushed(0, 0, 0) {
		t.Errorf("with rds.LastUpdate().IsZero(), rds.shouldBeFlushed == true")
	}

	// this will cause a LastUpdate != 0
	ds.ProcessIncomingDataPoint(123, time.Now().Add(-2*time.Hour))

	// so far we still have 0 points, so nothing to flush
	if rds.shouldBeFlushed(0, 0, 0) {
		t.Errorf("with PointCount 0, rds.shouldBeFlushed == true")
	}

	ds.ProcessIncomingDataPoint(123, time.Now().Add(-time.Hour))

	if !rds.shouldBeFlushed(0, 0, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 0, rds.shouldBeFlushed != true")
	}

	if rds.shouldBeFlushed(1000, 0, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 1000, rds.shouldBeFlushed == true")
	}

	if rds.shouldBeFlushed(1000, 24*time.Hour, 24*time.Hour) {
		t.Errorf("with maxCachedPoints == 1000, minCache 24hr, rds.shouldBeFlushed == true")
	}

	if !rds.shouldBeFlushed(1000, 0, 0) {
		t.Errorf("with maxCachedPoints == 1000, minCache 0, maxCache 0, rds.shouldBeFlushed != true")
	}

}
예제 #4
0
func Test_dispatcherProcessOrForward(t *testing.T) {

	saveFn := dispatcherForwardDPToNode
	forward, fwErr := 0, error(nil)
	dispatcherForwardDPToNode = func(dp *IncomingDP, node *cluster.Node, snd chan *cluster.Msg) error {
		forward++
		return fwErr
	}

	// rds
	ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}

	// cluster
	clstr := &fakeCluster{}
	md := make([]byte, 20)
	md[0] = 1 // Ready
	node := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "local"}}
	clstr.nodesForDd = []*cluster.Node{node}
	clstr.ln = node

	// workerChs
	workerChs := make([]chan *incomingDpWithDs, 1)
	workerChs[0] = make(chan *incomingDpWithDs)
	sent := 0
	go func() {
		for {
			<-workerChs[0]
			sent++
		}
	}()

	// Test if we are LocalNode
	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if sent < 1 {
		t.Errorf("dispatcherProcessOrForward: Nothing sent to workerChs")
	}

	// Now test we are NOT LN, forward
	remote := &cluster.Node{Node: &memberlist.Node{Meta: md, Name: "remote"}}
	clstr.nodesForDd = []*cluster.Node{remote}

	n := dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if forward != 1 {
		t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not called")
	}
	if n != 1 {
		t.Errorf("dispatcherProcessOrForward: return value != 1")
	}

	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	fwErr = fmt.Errorf("some error")
	n = dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if n != 0 {
		t.Errorf("dispatcherProcessOrForward: return value != 0")
	}
	if !strings.Contains(string(fl.last), "some error") {
		t.Errorf("dispatcherProcessOrForward: dispatcherForwardDPToNode not logged")
	}
	fwErr = nil

	// make an rds with points
	ds = rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
	rds = &receiverDs{DataSource: ds}

	dispatcherProcessOrForward(rds, clstr, workerChs, nil, nil)
	if !strings.Contains(string(fl.last), "PointCount") {
		t.Errorf("dispatcherProcessOrForward: Missing the PointCount warning log")
	}
	if rds.PointCount() != 0 {
		t.Errorf("dispatcherProcessOrForward: ClearRRAs(true) not called")
	}

	// restore dispatcherForwardDPToNode
	dispatcherForwardDPToNode = saveFn
}
예제 #5
0
파일: worker_test.go 프로젝트: tgres/tgres
func Test_workerPeriodicFlush(t *testing.T) {

	// fake logger
	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	// dsf
	f := &fakeDsFlusher{fdsReturn: true}

	// recent
	recent := make(map[int64]*receiverDs)

	// dsc
	db := &fakeSerde{}
	df := &dftDSFinder{}
	c := &fakeCluster{}
	dsc := newDsCache(db, df, c, nil, true)

	ds := rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	rds := &receiverDs{DataSource: ds}
	recent[7] = rds
	dsc.insert(rds)

	workerPeriodicFlush("workerperiodic2", f, recent, 0, 10*time.Millisecond, 10, 1)

	if f.called > 0 {
		t.Errorf("workerPeriodicFlush: no flush should have happened")
	}

	// make an rds with points
	ds = rrd.NewDataSource(7, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	ds.ProcessIncomingDataPoint(123, time.Unix(2000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(3000, 0))
	rds = &receiverDs{DataSource: ds}
	dsc.insert(rds)
	recent[7] = rds
	debug = true

	leftover := workerPeriodicFlush("workerperiodic3", f, recent, 0, 10*time.Millisecond, 0, 1)
	if f.called == 0 {
		t.Errorf("workerPeriodicFlush: should have called flushDs")
	}
	if len(recent) != 0 {
		t.Errorf("workerPeriodicFlush: should have deleted the point after flushDs")
	}
	if len(leftover) != 0 {
		t.Errorf("workerPeriodicFlush: leftover should be empty")
	}

	f.fdsReturn = false
	f.called = 0
	recent[7] = rds
	ds.ProcessIncomingDataPoint(123, time.Unix(4000, 0))
	ds.ProcessIncomingDataPoint(123, time.Unix(5000, 0))
	leftover = workerPeriodicFlush("workerperiodic4", f, recent, 0, 10*time.Millisecond, 0, 0)
	if f.called == 0 {
		t.Errorf("workerPeriodicFlush: should have called flushDs")
	}
	if len(recent) != 0 {
		t.Errorf("workerPeriodicFlush: should have deleted the point on flushDs (2)")
	}
	if len(leftover) == 0 {
		t.Errorf("workerPeriodicFlush: leftover should NOT be empty, it should have the point from recent")
	}

}
예제 #6
0
파일: worker_test.go 프로젝트: tgres/tgres
func Test_theWorker(t *testing.T) {

	// fake logger
	fl := &fakeLogger{}
	log.SetOutput(fl)
	defer func() {
		// restore default output
		log.SetOutput(os.Stderr)
	}()

	ident := "FOO"
	wc := &wrkCtl{wg: &sync.WaitGroup{}, startWg: &sync.WaitGroup{}, id: ident}
	dsf := &fakeDsFlusher{fdsReturn: true}
	workerCh := make(chan *incomingDpWithDs)

	saveFn1 := workerPeriodicFlush

	wpfCalled := 0
	workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlush int) map[int64]*receiverDs {
		wpfCalled++
		return map[int64]*receiverDs{1: nil, 2: nil}
	}

	sr := &fakeSr{}

	wc.startWg.Add(1)
	go worker(wc, dsf, workerCh, 0, 0, 10, 10*time.Millisecond, sr)
	wc.startWg.Wait()

	if !strings.Contains(string(fl.last), ident) {
		t.Errorf("worker: missing worker started log entry for ident: %s", ident)
	}

	debug = true

	// make an rds with points
	ds := rrd.NewDataSource(0, "foo", 0, 0, time.Time{}, 0)
	rra, _ := rrd.NewRoundRobinArchive(1, 0, "WMEAN", 10*time.Second, 100, 30, 0.5, time.Unix(1000, 0))
	ds.SetRRAs([]*rrd.RoundRobinArchive{rra})
	rds := &receiverDs{DataSource: ds}
	dp := &IncomingDP{Name: "foo", TimeStamp: time.Unix(2000, 0), Value: 123}
	workerCh <- &incomingDpWithDs{dp, rds}
	dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(3000, 0), Value: 123}
	workerCh <- &incomingDpWithDs{dp, rds}

	pc := ds.PointCount()
	if pc == 0 {
		t.Errorf("After dps being sent to workerCh, ds should have some points: %d", pc)
	}

	time.Sleep(50 * time.Millisecond) // wait for a flush or two
	if wpfCalled == 0 {
		t.Errorf("worker: at least one periodic flush should have happened")
	}

	// trigger an error
	dp = &IncomingDP{Name: "foo", TimeStamp: time.Unix(5000, 0), Value: math.Inf(-1)}
	workerCh <- &incomingDpWithDs{dp, rds}

	close(workerCh)
	wc.wg.Wait()

	if !strings.Contains(string(fl.last), "not a valid data point") {
		t.Errorf("worker: missing 'not a valid data point' log entry")
	}

	// restore funcs
	workerPeriodicFlush = saveFn1
}