Exemple #1
0
func (self *hawkularSink) StoreTimeseries(ts []sink_api.Timeseries) error {
	if len(ts) > 0 {
		tmhs := make(map[string][]metrics.MetricHeader)

		if &self.labelTenant == nil {
			tmhs[self.client.Tenant] = make([]metrics.MetricHeader, 0, len(ts))
		}

		wg := &sync.WaitGroup{}

		for _, t := range ts {

			tenant := self.client.Tenant

			if &self.labelTenant != nil {
				if v, found := t.Point.Labels[self.labelTenant]; found {
					tenant = v
				}
			}

			// Registering should not block the processing
			wg.Add(1)
			go func(t *sink_api.Timeseries, tenant string) {
				defer wg.Done()
				self.registerIfNecessary(t, metrics.Tenant(tenant))
			}(&t, tenant)

			if t.MetricDescriptor.ValueType == sink_api.ValueBool {
				// TODO: Model to availability type once we see some real world examples
				break
			}

			mH, err := self.pointToMetricHeader(&t)
			if err != nil {
				// One transformation error should not prevent the whole process
				glog.Errorf(err.Error())
				continue
			}

			if _, found := tmhs[tenant]; !found {
				tmhs[tenant] = make([]metrics.MetricHeader, 0)
			}

			tmhs[tenant] = append(tmhs[tenant], *mH)
		}

		for k, v := range tmhs {
			wg.Add(1)
			go func(v []metrics.MetricHeader, k string) {
				defer wg.Done()
				if err := self.client.Write(v, metrics.Tenant(k)); err != nil {
					glog.Errorf(err.Error())
				}
			}(v, k)
		}
		wg.Wait()
	}
	return nil
}
Exemple #2
0
func (hs *hawkularSource) GetUsagePercentile(kind api.ResourceName, perc int64, image, namespace string, exactMatch bool, start, end time.Time) (int64, int64, error) {
	q := tagQuery(kind, image, exactMatch)

	m := make([]metrics.Modifier, len(hs.modifiers), 2+len(hs.modifiers))
	copy(m, hs.modifiers)

	if namespace != api.NamespaceAll {
		m = append(m, metrics.Tenant(namespace))
	}

	p, err := metrics.ConvertToFloat64(perc)
	if err != nil {
		return 0, 0, err
	}

	m = append(m, metrics.Filters(metrics.TagsFilter(q), metrics.BucketsFilter(1), metrics.StartTimeFilter(start), metrics.EndTimeFilter(end), metrics.PercentilesFilter([]float64{p})))

	bp, err := hs.client.ReadBuckets(metrics.Counter, m...)
	if err != nil {
		return 0, 0, err
	}

	if len(bp) > 0 && len(bp[0].Percentiles) > 0 {
		return int64(bp[0].Percentiles[0].Value), int64(bp[0].Samples), nil
	}
	return 0, 0, nil
}
Exemple #3
0
func (h *hawkularSink) sendData(tmhs map[string][]metrics.MetricHeader, wg *sync.WaitGroup) {
	for k, v := range tmhs {
		parts := toBatches(v, h.batchSize)
		close(parts)

		for p := range parts {
			wg.Add(1)
			go func(batch []metrics.MetricHeader, tenant string) {
				defer wg.Done()

				m := make([]metrics.Modifier, len(h.modifiers), len(h.modifiers)+1)
				copy(m, h.modifiers)
				m = append(m, metrics.Tenant(tenant))
				if err := h.client.Write(batch, m...); err != nil {
					glog.Errorf(err.Error())
				}
			}(p, k)
		}
	}
}
Exemple #4
0
func (h *hawkularSink) ExportData(db *core.DataBatch) {
	totalCount := 0
	for _, ms := range db.MetricSets {
		totalCount += len(ms.MetricValues)
		totalCount += len(ms.LabeledMetrics)
	}

	if len(db.MetricSets) > 0 {
		tmhs := make(map[string][]metrics.MetricHeader)

		if len(h.labelTenant) == 0 {
			tmhs[h.client.Tenant] = make([]metrics.MetricHeader, 0, totalCount)
		}

		wg := &sync.WaitGroup{}

		for _, ms := range db.MetricSets {

			// // Transform ms.MetricValues to LabeledMetrics first
			lms := metricValueToLabeledMetric(ms.MetricValues)
			ms.LabeledMetrics = append(ms.LabeledMetrics, lms...)

		Store:
			for _, labeledMetric := range ms.LabeledMetrics {

				for _, filter := range h.filters {
					if !filter(ms, labeledMetric.Name) {
						continue Store
					}
				}

				tenant := h.client.Tenant

				if len(h.labelTenant) > 0 {
					if v, found := ms.Labels[h.labelTenant]; found {
						tenant = v
					}
				}

				wg.Add(1)
				go func(ms *core.MetricSet, labeledMetric core.LabeledMetric, tenant string) {
					defer wg.Done()
					h.registerLabeledIfNecessary(ms, labeledMetric, metrics.Tenant(tenant))
				}(ms, labeledMetric, tenant)

				mH, err := h.pointToLabeledMetricHeader(ms, labeledMetric, db.Timestamp)
				if err != nil {
					// One transformation error should not prevent the whole process
					glog.Errorf(err.Error())
					continue
				}

				if _, found := tmhs[tenant]; !found {
					tmhs[tenant] = make([]metrics.MetricHeader, 0)
				}

				tmhs[tenant] = append(tmhs[tenant], *mH)
			}
		}
		h.sendData(tmhs, wg) // Send to a limited channel? Only batches.. egg.
		wg.Wait()
	}
}