Example #1
0
func createApplicationStats(
	appList *datastructs.ApplicationList,
	logger *log.Logger,
	tagvAdder suggest.Adder,
	maybeNilMemoryManager *memoryManagerType) *datastructs.ApplicationStatuses {
	var astore *store.Store
	fmt.Println("Initialization started.")
	if maybeNilMemoryManager != nil {
		memoryManager := maybeNilMemoryManager
		astore = store.NewStoreBytesPerPage(
			*fBytesPerPage,
			1,
			*fThreshhold,
			*fDegree)
		astore.SetExpanding(true)
		memoryManager.SetMemory(astore)
		if err := memoryManager.RegisterMetrics(); err != nil {
			log.Fatal(err)
		}
	} else {
		astore = store.NewStoreBytesPerPage(
			*fBytesPerPage, *fPageCount, *fThreshhold, *fDegree)
	}
	dirSpec, err := tricorder.RegisterDirectory("/store")
	if err != nil {
		log.Fatal(err)
	}
	if err := astore.RegisterMetrics(dirSpec); err != nil {
		log.Fatal(err)
	}
	stats := datastructs.NewApplicationStatuses(appList, astore)
	mdbChannel := mdbd.StartMdbDaemon(*fMdbFile, logger)
	machines := <-mdbChannel
	theHostNames := hostNames(machines.Machines)
	for _, aName := range theHostNames {
		tagvAdder.Add(aName)
	}
	stats.MarkHostsActiveExclusively(
		duration.TimeToFloat(time.Now()), theHostNames)
	fmt.Println("Initialization complete.")
	// Endpoint refresher goroutine
	go func() {
		for {
			machines := <-mdbChannel
			stats.MarkHostsActiveExclusively(
				duration.TimeToFloat(time.Now()),
				hostNames(machines.Machines))
		}
	}()
	return stats
}
Example #2
0
// ToFloat converts a value of this type to a float64
// ToFloat panics if this type doesn't support conversion to float64
func (t Type) ToFloat(x interface{}) float64 {
	switch t {
	case Int8:
		return float64(x.(int8))
	case Int16:
		return float64(x.(int16))
	case Int32:
		return float64(x.(int32))
	case Int64:
		return float64(x.(int64))
	case Uint8:
		return float64(x.(uint8))
	case Uint16:
		return float64(x.(uint16))
	case Uint32:
		return float64(x.(uint32))
	case Uint64:
		return float64(x.(uint64))
	case Float32:
		return float64(x.(float32))
	case Float64:
		return x.(float64)
	case GoTime:
		return duration.TimeToFloat(x.(time.Time))
	case GoDuration:
		return duration.ToFloat(x.(time.Duration))
	default:
		panic("Type doesn't support conversion to float")
	}
}
Example #3
0
func newStoreMetricsType() *storeMetricsType {
	return &storeMetricsType{
		PagesPerMetricDist: kBucketer.NewNonCumulativeDistribution(),
		values: storePrimitiveMetricsType{
			LatestEvictedTimeStamp: duration.TimeToFloat(time.Now()),
		},
	}
}
Example #4
0
func (n *namedIteratorSameValueType) Next(r *store.Record) bool {
	if n.numWritten == n.numToWrite {
		return false
	}
	r.Info = n.info
	r.Value = n.value
	r.TimeStamp = duration.TimeToFloat(kNow)
	r.Active = true
	n.numWritten++
	return true
}
Example #5
0
func (n *namedIteratorForTestingType) Next(r *store.Record) bool {
	if n.numWritten == n.numToWrite {
		return false
	}
	r.Info = kInt64MetricInfo
	r.Value = int64(n.numWritten)
	r.TimeStamp = duration.TimeToFloat(kNow)
	r.Active = true
	n.numWritten++
	return true
}
Example #6
0
func (l *loggerType) LogResponse(
	e *collector.Endpoint,
	list metrics.List,
	timestamp time.Time) error {
	ts := duration.TimeToFloat(timestamp)
	added, err := l.Store.AddBatch(
		e,
		ts,
		list)
	if err == nil {
		l.reportNewNamesForSuggest(list)
		l.AppStats.LogChangedMetricCount(e, added)
		l.ChangedMetricsDist.Add(float64(added))
		l.TotalCounts.Update(l.Store, e)
	}
	// This error just means that the endpoint was marked inactive
	// during polling.
	if err == store.ErrInactive {
		return nil
	}
	return err
}
Example #7
0
func asFloat64(r *pstore.Record) float64 {
	switch r.Kind {
	case types.Bool:
		if r.Value.(bool) {
			return 1.0
		}
		return 0.0
	case types.Int8:
		return float64(r.Value.(int8))
	case types.Int16:
		return float64(r.Value.(int16))
	case types.Int32:
		return float64(r.Value.(int32))
	case types.Int64:
		return float64(r.Value.(int64))
	case types.Uint8:
		return float64(r.Value.(uint8))
	case types.Uint16:
		return float64(r.Value.(uint16))
	case types.Uint32:
		return float64(r.Value.(uint32))
	case types.Uint64:
		return float64(r.Value.(uint64))
	case types.Float32:
		return float64(r.Value.(float32))
	case types.Float64:
		return r.Value.(float64)
	case types.GoTime:
		return duration.TimeToFloat(r.Value.(time.Time))
	case types.GoDuration:
		return duration.ToFloat(
			r.Value.(time.Duration)) * units.FromSeconds(
			r.Unit)
	default:
		panic("Unsupported type")

	}
}
Example #8
0
// gatherDataForEndpoint serves api/hosts pages.
// metricStore is the metric store.
// endpoint is the endpoint from which we are getting historical metrics.
// canonicalPath is the path of the metrics or the empty string for all
// metrics. canonicalPath is returned from canonicalisePath().
// history is the amount of time to go back in minutes.
// If isSingleton is true, fetched metrics have to match canonicalPath
// exactly.
// Otherwise fetched metrics have to be found underneath canonicalPath.
// On no match, gatherDataForEndpoint returns an empty
// messages.EndpointMetricsList instance
func gatherDataForEndpoint(
	metricStore *store.Store,
	endpoint *collector.Endpoint,
	canonicalPath string,
	history int,
	isSingleton bool) (result messages.EndpointMetricList) {
	result = make(messages.EndpointMetricList, 0)
	now := duration.TimeToFloat(time.Now())
	appender := newEndpointMetricsAppender(&result)
	if canonicalPath == "" {
		metricStore.ByEndpointStrategy(
			endpoint,
			now-60.0*float64(history),
			math.Inf(1),
			store.GroupMetricByKey,
			appender)
	} else {
		metricStore.ByNameAndEndpointStrategy(
			canonicalPath,
			endpoint,
			now-60.0*float64(history),
			math.Inf(1),
			store.GroupMetricByKey,
			appender)
		if len(result) == 0 && !isSingleton {
			metricStore.ByPrefixAndEndpointStrategy(
				canonicalPath+"/",
				endpoint,
				now-60.0*float64(history),
				math.Inf(1),
				store.GroupMetricByKey,
				appender)
		}

	}
	sortMetricsByPath(result)
	return
}
Example #9
0
// LookupBatch looks up all the metrics in one go and returns the
// following:
// fetched: timeSeries already in this collection keyed by Metric.
//  values must be added to these manually.
// newOnes: timeSeries just added as a result of this lookup. Since these
// are new, the first value added automatically.
// notFetched: timeSeries in this collection but not fetched. These
// are the time series that should be marked inactive.
// ok is true if metrics can be added to this instance or false if this
// instance is inactive and closed for new metrics.
func (c *timeSeriesCollectionType) LookupBatch(
	timestamp float64, mlist metrics.List) (
	fetched map[*timeSeriesType]interface{},
	newOnes, notFetched []*timeSeriesType,
	fetchedTimeStamps map[*timestampSeriesType]float64,
	newTs []*timestampSeriesType,
	notFetchedTimeStamps []*timestampSeriesType,
	err error) {
	if err = metrics.VerifyList(mlist); err != nil {
		return
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	if !c.active {
		err = ErrInactive
		return
	}
	valueByMetric := make(map[*MetricInfo]interface{})
	timestampByGroupId := make(map[int]float64)
	groupIds := make(map[int]bool)
	fetched = make(map[*timeSeriesType]interface{})
	fetchedTimeStamps = make(map[*timestampSeriesType]float64)
	mlen := mlist.Len()
	for i := 0; i < mlen; i++ {
		var avalue metrics.Value
		mlist.Index(i, &avalue)
		kind, subType := types.FromGoValueWithSubType(avalue.Value)
		id := c.metricInfoStore.Register(&avalue, kind, subType)
		if kind == types.Dist {
			distributionRollOvers := c.distributionRollOversByPath[avalue.Path]
			if distributionRollOvers == nil {
				distributionRollOvers = &distributionRollOverType{}
				c.distributionRollOversByPath[avalue.Path] = distributionRollOvers
			}
			distribution := avalue.Value.(*messages.Distribution)
			rollOverCount := distributionRollOvers.UpdateAndFetchRollOverCount(
				id.Ranges(), distribution.Generation)
			currentDistributionTotals := &DistributionTotals{
				Counts:        distExtractCounts(distribution),
				Sum:           distribution.Sum,
				RollOverCount: rollOverCount,
			}
			valueByMetric[id] = currentDistributionTotals
		} else {
			valueByMetric[id] = avalue.Value
		}
		groupIds[id.GroupId()] = true
		if !avalue.TimeStamp.IsZero() {
			timestampByGroupId[id.GroupId()] = duration.TimeToFloat(avalue.TimeStamp)
		}
	}
	// If a group ID is missing a timestamp, give it the
	// timestamp from scotty.
	for groupId := range groupIds {
		if _, ok := timestampByGroupId[groupId]; !ok {
			timestampByGroupId[groupId] = timestamp
		}
	}
	// populate notFetched
	for id, series := range c.timeSeries {
		if _, ok := valueByMetric[id]; !ok {
			notFetched = append(notFetched, series)
		}
	}
	// populate fetched and newOnes
	for id, value := range valueByMetric {
		if c.timeSeries[id] == nil {
			thisTs := timestampByGroupId[id.GroupId()]
			c.timeSeries[id] = newTimeSeriesType(
				id, thisTs, value, c.metrics)
			newOnes = append(newOnes, c.timeSeries[id])
		} else {
			fetched[c.timeSeries[id]] = value
		}
	}
	// populate notFetchedTimeStamps
	for groupId, series := range c.timestampSeries {
		if _, ok := timestampByGroupId[groupId]; !ok {
			notFetchedTimeStamps = append(
				notFetchedTimeStamps, series)
		}
	}
	// populate fetchedTimeStamps and newTs
	for groupId, ts := range timestampByGroupId {
		if c.timestampSeries[groupId] == nil {
			c.timestampSeries[groupId] = newTimeStampSeriesType(
				groupId, ts, c.metrics)
			newTs = append(newTs, c.timestampSeries[groupId])
		} else {
			fetchedTimeStamps[c.timestampSeries[groupId]] = ts
		}
	}
	return
}
Example #10
0
// TODO: Remove once we know the grafana bug involving duplicate timestamps
// is fixed.
func newPathAndMillisType(record *pstore.Record) pathAndMillisType {
	return pathAndMillisType{
		Path:   record.Path,
		Millis: int64(duration.TimeToFloat(record.Timestamp) * 1000.0)}
}