func (self *hawkularSink) StoreTimeseries(ts []sink_api.Timeseries) error {
if len(ts) > 0 {
tmhs := make(map[string][]metrics.MetricHeader)
if &self.labelTenant == nil {
tmhs[self.client.Tenant] = make([]metrics.MetricHeader, 0, len(ts))
}
wg := &sync.WaitGroup{}
for _, t := range ts {
tenant := self.client.Tenant
if &self.labelTenant != nil {
if v, found := t.Point.Labels[self.labelTenant]; found {
tenant = v
}
}
// Registering should not block the processing
wg.Add(1)
go func(t *sink_api.Timeseries, tenant string) {
defer wg.Done()
self.registerIfNecessary(t, metrics.Tenant(tenant))
}(&t, tenant)
if t.MetricDescriptor.ValueType == sink_api.ValueBool {
// TODO: Model to availability type once we see some real world examples
break
}
mH, err := self.pointToMetricHeader(&t)
if err != nil {
// One transformation error should not prevent the whole process
glog.Errorf(err.Error())
continue
}
if _, found := tmhs[tenant]; !found {
tmhs[tenant] = make([]metrics.MetricHeader, 0)
}
tmhs[tenant] = append(tmhs[tenant], *mH)
}
for k, v := range tmhs {
wg.Add(1)
go func(v []metrics.MetricHeader, k string) {
defer wg.Done()
if err := self.client.Write(v, metrics.Tenant(k)); err != nil {
glog.Errorf(err.Error())
}
}(v, k)
}
wg.Wait()
}
return nil
}
func (hs *hawkularSource) GetUsagePercentile(kind api.ResourceName, perc int64, image, namespace string, exactMatch bool, start, end time.Time) (int64, int64, error) {
q := tagQuery(kind, image, exactMatch)
m := make([]metrics.Modifier, len(hs.modifiers), 2+len(hs.modifiers))
copy(m, hs.modifiers)
if namespace != api.NamespaceAll {
m = append(m, metrics.Tenant(namespace))
}
p, err := metrics.ConvertToFloat64(perc)
if err != nil {
return 0, 0, err
}
m = append(m, metrics.Filters(metrics.TagsFilter(q), metrics.BucketsFilter(1), metrics.StartTimeFilter(start), metrics.EndTimeFilter(end), metrics.PercentilesFilter([]float64{p})))
bp, err := hs.client.ReadBuckets(metrics.Counter, m...)
if err != nil {
return 0, 0, err
}
if len(bp) > 0 && len(bp[0].Percentiles) > 0 {
return int64(bp[0].Percentiles[0].Value), int64(bp[0].Samples), nil
}
return 0, 0, nil
}
func (h *hawkularSink) sendData(tmhs map[string][]metrics.MetricHeader, wg *sync.WaitGroup) {
for k, v := range tmhs {
parts := toBatches(v, h.batchSize)
close(parts)
for p := range parts {
wg.Add(1)
go func(batch []metrics.MetricHeader, tenant string) {
defer wg.Done()
m := make([]metrics.Modifier, len(h.modifiers), len(h.modifiers)+1)
copy(m, h.modifiers)
m = append(m, metrics.Tenant(tenant))
if err := h.client.Write(batch, m...); err != nil {
glog.Errorf(err.Error())
}
}(p, k)
}
}
}
func (h *hawkularSink) ExportData(db *core.DataBatch) {
totalCount := 0
for _, ms := range db.MetricSets {
totalCount += len(ms.MetricValues)
totalCount += len(ms.LabeledMetrics)
}
if len(db.MetricSets) > 0 {
tmhs := make(map[string][]metrics.MetricHeader)
if len(h.labelTenant) == 0 {
tmhs[h.client.Tenant] = make([]metrics.MetricHeader, 0, totalCount)
}
wg := &sync.WaitGroup{}
for _, ms := range db.MetricSets {
// // Transform ms.MetricValues to LabeledMetrics first
lms := metricValueToLabeledMetric(ms.MetricValues)
ms.LabeledMetrics = append(ms.LabeledMetrics, lms...)
Store:
for _, labeledMetric := range ms.LabeledMetrics {
for _, filter := range h.filters {
if !filter(ms, labeledMetric.Name) {
continue Store
}
}
tenant := h.client.Tenant
if len(h.labelTenant) > 0 {
if v, found := ms.Labels[h.labelTenant]; found {
tenant = v
}
}
wg.Add(1)
go func(ms *core.MetricSet, labeledMetric core.LabeledMetric, tenant string) {
defer wg.Done()
h.registerLabeledIfNecessary(ms, labeledMetric, metrics.Tenant(tenant))
}(ms, labeledMetric, tenant)
mH, err := h.pointToLabeledMetricHeader(ms, labeledMetric, db.Timestamp)
if err != nil {
// One transformation error should not prevent the whole process
glog.Errorf(err.Error())
continue
}
if _, found := tmhs[tenant]; !found {
tmhs[tenant] = make([]metrics.MetricHeader, 0)
}
tmhs[tenant] = append(tmhs[tenant], *mH)
}
}
h.sendData(tmhs, wg) // Send to a limited channel? Only batches.. egg.
wg.Wait()
}
}