예제 #1
0
func (c *CassandraDB) WriteRow(wRecord db.WriteRecord, src string) (rowKey string, err error) {
	if len(wRecord.AggregatesColumnNames) != len(wRecord.Aggregates) {
		return "", errors.New("Aggregates names and data don't match.")
	}

	var timestamp int64
	if wRecord.RecordTimestamp != nil {
		timestamp = *wRecord.RecordTimestamp
	}
	uuidString := uuid.New()
	rowKey = dbcommon.MakeRowKey(src, timestamp, uuidString)

	////////////////////////////////////////////////////////////////////////////
	// Process points.

	ptsMap := make(map[string][]float64) // Used for creating missing aggregates in aggregate processing loop.
	// aggs is a map from metric name to map from aggregate to value.
	aggs := make(map[string]map[string]*float64)

	if len(wRecord.Points) > 0 {
		var ptsDataType pb.DataType
		dataTypeInt32, setPtsDataType := pb.DataType_value[strings.ToUpper(wRecord.PointsDataType)]
		if setPtsDataType { // Only set proto field if explicitly set by user.  Otherwise rely on default.
			ptsDataType = pb.DataType(dataTypeInt32)
		}

		ptsRow := &gossie.Row{[]byte(rowKey), nil}
		for _, pointRecord := range wRecord.Points {
			if (len(pointRecord.Timestamps) > 0) && (len(pointRecord.Data) != len(pointRecord.Timestamps)) {
				return "", errors.New("Points data and timestamps don't match for: " + pointRecord.Name)
			}

			p := &pb.Points{}
			if setPtsDataType { // Only set proto field if explicitly set by user.  Otherwise rely on default.
				dType := ptsDataType // Make new allocation for each one.
				p.Type = &dType
			}
			ptsMap[pointRecord.Name] = pointRecord.Data // Save for creating missing aggregates.
			// This blank map ensures we create any missing aggregates in the aggregate handling loops below.
			aggs[pointRecord.Name] = make(map[string]*float64)

			p.ValuesDouble = pointRecord.Data
			p.MakeDeltaValuesScaled(p.GetType())

			var previousTS int64
			for _, timestamp := range pointRecord.Timestamps {
				p.DeltaTimestamps = append(p.DeltaTimestamps, timestamp-previousTS)
				previousTS = timestamp
			}

			serializedData, e := proto.Marshal(p)
			if e != nil {
				return "", e
			}

			ptsRow.Columns = append(ptsRow.Columns, &gossie.Column{
				Name:  []byte(pointRecord.Name),
				Value: serializedData})
		}

		tPointsWrite := time.Now()
		if err := c.pool.Writer().Insert(dbcommon.CFPoints, ptsRow).Run(); err != nil {
			return "", err
		}
		glog.V(2).Infof("PERF: DB points write time: %v\n", time.Now().Sub(tPointsWrite))
	}

	////////////////////////////////////////////////////////////////////////////
	// Process aggregates (or make aggregates from points).

	if (len(wRecord.AggregatesColumnNames) + len(wRecord.Points)) > 0 {
		var aggDataType pb.DataType
		dataTypeInt32, setAggDataType := pb.DataType_value[strings.ToUpper(wRecord.AggregatesDataType)]
		if setAggDataType { // Only set proto field if explicitly set by user.  Otherwise rely on default.
			aggDataType = pb.DataType(dataTypeInt32)
		}

		// Parse aggregatesColumnNames and build aggs map of map.
		for idx, fullName := range wRecord.AggregatesColumnNames {
			metricName, aggregateName := common.GetMetricComponents(fullName)
			if metricName == "" {
				return "", errors.New("Missing metric name in:" + fullName)
			}
			if aggregateName == "" {
				return "", errors.New("Missing aggregate name in:" + fullName)
			}

			if aggs[metricName] == nil {
				aggs[metricName] = make(map[string]*float64)
			}
			value := wRecord.Aggregates[idx]
			aggs[metricName][aggregateName] = value
		}

		// Iterate over data in aggs map of map.
		aggRow := &gossie.Row{[]byte(rowKey), nil}
		for metricName, aggMap := range aggs {
			a := new(pb.Aggregation)
			if setAggDataType { // Only set proto field if explicitly set by user.  Otherwise rely on default.
				dType := aggDataType // Make new allocation for each one.
				a.Type = &dType
			}
			a.Double = &pb.Aggregation_AggregationDouble{}
			for aggregateName, valuePtr := range aggMap {
				a.SetDoubleField(aggregateName, valuePtr)
			}
			a.CreateMissingDoubleAggregates(ptsMap[metricName])

			a.MakeScaled(a.GetType())

			serializedData, e := proto.Marshal(a)
			if e != nil {
				return "", e
			}

			aggRow.Columns = append(aggRow.Columns, &gossie.Column{
				Name:  []byte(metricName),
				Value: serializedData,
			})
		}

		tAggregatesWrite := time.Now()
		if err := c.pool.Writer().Insert(dbcommon.CFAggregates, aggRow).Run(); err != nil {
			return "", err
		}
		glog.V(2).Infof("PERF: DB aggregates write time: %v\n", time.Now().Sub(tAggregatesWrite))
	}

	////////////////////////////////////////////////////////////////////////////
	// Process configs.

	if len(wRecord.ConfigPairs) > 0 {
		cfgRow := &gossie.Row{[]byte(rowKey), nil}
		for k, v := range wRecord.ConfigPairs {
			cfgRow.Columns = append(cfgRow.Columns, &gossie.Column{
				Name:  []byte(k),
				Value: []byte(v),
			})
		}
		if err := c.pool.Writer().Insert(dbcommon.CFConfigs, cfgRow).Run(); err != nil {
			return "", err
		}
	}

	////////////////////////////////////////////////////////////////////////////
	// Process src.

	if (len(wRecord.Points) + len(wRecord.AggregatesColumnNames) + len(wRecord.ConfigPairs)) > 0 {
		srcRow := &gossie.Row{[]byte(rowKey), nil}
		srcRow.Columns = append(srcRow.Columns, &gossie.Column{
			Name:  []byte(src),
			Value: []byte{},
		})
		return rowKey, c.pool.Writer().Insert(dbcommon.CFSource, srcRow).Run()
	} else {
		return "", errors.New("No data to write.")
	}
}
예제 #2
0
func MakeSrcsInlineGraphContent(d db.DB, b *bytes.Buffer, rawQuery string) error {
	dTable, err := getDataTable(d, rawQuery)
	if err != nil {
		return err
	}

	q, _ := url.ParseQuery(rawQuery)
	showShadow := q.Get("showShadow") == "1"

	aggregatesListStr := q.Get("aggregates")
	aggregatesMap := make(map[string]int) // Map from aggregate to index.
	if len(aggregatesListStr) > 0 {
		for idx, aggregate := range strings.Split(aggregatesListStr, ",") {
			aggregatesMap[aggregate] = idx
		}
	}

	if showShadow {
		if len(dTable.ColumnNames) < 4 || ((len(dTable.ColumnNames)-1)%3) != 0 {
			return errors.New("Need exactly 3 aggregates to show shadow plots.")
		}

		var data [][]interface{}
		var columnNames []string
		for i := 0; i < len(dTable.ColumnNames); i += 3 {
			metric, _ := common.GetMetricComponents(dTable.ColumnNames[i])
			columnNames = append(columnNames, metric)
		}

		// aggRemap is used to reorder the aggregates to reflect the query param order.
		aggRemap := make(map[int]int) // Remap actual index to target index.
		for i := 1; i < 4; i++ {
			_, aggregate := common.GetMetricComponents(dTable.ColumnNames[i])
			aggRemap[i-1] = aggregatesMap[aggregate]
		}

		for _, dRow := range dTable.Data {
			newRow := []interface{}{(*dRow)[0]} // Start row with X value.
			for i := 1; i < len(*dRow); i += 3 {
				v0 := (*dRow)[i+aggRemap[0]]
				v1 := (*dRow)[i+aggRemap[1]]
				v2 := (*dRow)[i+aggRemap[2]]
				newRow = append(newRow, []*float64{v0, v1, v2})
			}
			data = append(data, newRow)
		}
		tTemplate := time.Now()
		err = templateloader.Templates.ExecuteTemplate(b, "in-graph.template-html", struct {
			Data        [][]interface{}
			ColumnNames []string
			ShowShadow  bool
			XLabel      string
		}{
			Data:        data,
			ColumnNames: columnNames,
			ShowShadow:  true,
			XLabel:      dTable.ColumnNames[0],
		})
		glog.V(2).Infof("PERF: template generation time: %v\n", time.Now().Sub(tTemplate))
	} else {
		tTemplate := time.Now()
		err = templateloader.Templates.ExecuteTemplate(b, "in-graph.template-html", struct {
			Data        []*[]*float64
			ColumnNames []string
			ShowShadow  bool
			XLabel      string
		}{
			Data:        dTable.Data,
			ColumnNames: dTable.ColumnNames,
			ShowShadow:  false,
			XLabel:      dTable.ColumnNames[0],
		})
		glog.V(2).Infof("PERF: template generation time: %v\n", time.Now().Sub(tTemplate))
	}
	if err != nil {
		return err
	}

	return nil
}