Example #1
0
func getActiveUsersByDimension(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite, dimension string, topN int) error {
	call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers").Dimensions("rt:" + dimension)
	data, err := call.Do()
	if err != nil {
		return err
	}
	tags := opentsdb.TagSet{"site": site.Name}
	rows := make(kvList, len(data.Rows))
	for i, row := range data.Rows {
		// key will always be an string of the dimension we care about.
		// For example, 'Chrome' would be a key for the 'browser' dimension.
		key, _ := opentsdb.Clean(row[0])
		if key == "" {
			key = "__blank__"
		}
		value, err := strconv.Atoi(row[1])
		if err != nil {
			return fmt.Errorf("Error parsing GA data: %s", err)
		}
		rows[i] = kv{key: key, value: value}
	}
	sort.Sort(sort.Reverse(rows))
	if topN != -1 && topN < len(rows) {
		topRows := make(kvList, topN)
		topRows = rows[:topN]
		rows = topRows
	}

	for _, row := range rows {
		Add(md, "google.analytics.realtime.activeusers.by_"+dimension, row.value, opentsdb.TagSet{dimension: row.key}.Merge(tags), metadata.Gauge, metadata.ActiveUsers, descActiveUsers)
	}
	return nil
}
Example #2
0
File: collect.go Project: pdf/bosun
func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}
Example #3
0
func DatabaseAddCollector(c conf.Database) {
	if c.Type != "mysql" {
		slog.Fatalf("%v: %v", "invalid Database Type", c.Type)
	}
	if c.DBName != "" {
		cleaned, _ := opentsdb.Clean(c.DBName)
		if c.DBName != cleaned {
			slog.Fatalf("%v: %v", "invalid Database DBName", c.DBName)
		}
	}
	if c.InstId < 1 {
		c.InstId = 1
	}
	if c.MaxOpenConns < 1 {
		c.MaxOpenConns = 2
	}
	if c.Username == "" {
		c.Username = "******"
	}
	if c.Protocol == "" {
		c.Protocol = "tcp"
	}
	if c.Address == "" {
		c.Address = "127.0.0.1"
	}
	if c.Port < 1 {
		c.Port = 3306
	}

	var tags opentsdb.TagSet
	if c.DBName == "" {
		tags = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)}
	} else {
		tags = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)}
	}

	collectors = append(collectors, &ContinuousCollector{
		F: func(collectorStatsChan chan<- *ContinuousCollectorStats) {
			DatabaseCollect(c, collectorStatsChan)
		},
		name: c.Type,
		tags: tags,
	})
}
Example #4
0
func DatabaseGetQueries(c conf.Database) []DatabaseQuery {
	queries := make([]DatabaseQuery, 0, len(c.Query))

	for _, query := range c.Query {
		cleaned, _ := opentsdb.Clean(query.Name)

		if query.Name != cleaned {
			slog.Warningf("%v: %v: %v: %v", "Database", c.Type, "invaid metric name", query.Name)
			continue
		}

		databaseQuery := DatabaseQuery{name: query.Name, query: query.Query, description: query.Description, hasTime: query.HasTime}

		if query.Interval < 1 {
			databaseQuery.interval = DefaultFreq
		} else {
			databaseQuery.interval = time.Duration(query.Interval) * time.Second
		}

		queries = append(queries, databaseQuery)
	}

	return queries
}
Example #5
0
func DatabaseParseResults(c conf.Database, query *DatabaseQuery, results *[][]string) (opentsdb.MultiDataPoint, error) {
	if len(*results) < 1 {
		return nil, nil
	}

	numTagLoops := len((*results)[0]) - 1
	if query.hasTime {
		numTagLoops--
	}
	if numTagLoops < 0 {
		return nil, fmt.Errorf("invalid number of columns")
	}
	numTagLoops = (numTagLoops - (numTagLoops % 2)) / 2

	var md opentsdb.MultiDataPoint
	var tagName string
	var tagValue string
	var tagsBase opentsdb.TagSet
	baseName := c.Type + "."
	if c.DBName == "" {
		tagsBase = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)}
	} else {
		tagsBase = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)}
	}

	for _, result := range *results {
		if result[0] == "" {
			continue
		}

		// Check result[0] is vaild metric value?

		tags := tagsBase.Copy()
		for i := 0; i < numTagLoops; i++ {
			if query.hasTime {
				tagName = strings.Replace(result[(i*2)+2], " ", "_", -1)
				tagValue = strings.Replace(result[(i*2)+3], " ", "_", -1)
			} else {
				tagName = strings.Replace(result[(i*2)+1], " ", "_", -1)
				tagValue = strings.Replace(result[(i*2)+2], " ", "_", -1)
			}
			tagName, _ = opentsdb.Clean(tagName)
			tagValue, _ = opentsdb.Clean(tagValue)
			tagName = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagName, "_")
			tagValue = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagValue, "_")

			if tagName == "" || tagValue == "" {
				continue
			}
			tags = tags.Merge(opentsdb.TagSet{tagName: tagValue})
		}

		// Add metadata tag set type to configuration file?

		if query.hasTime {
			timestamp, _ := strconv.ParseInt(result[1], 10, 64)
			if timestamp < 1 {
				return nil, fmt.Errorf("invalid timestamp")
			}
			AddTS(&md, baseName+query.name, timestamp, result[0], tags, metadata.Unknown, metadata.None, query.description)
		} else {
			Add(&md, baseName+query.name, result[0], tags, metadata.Unknown, metadata.None, query.description)
		}
	}

	return md, nil
}
Example #6
0
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
	ifNamesRaw, err := snmp_subtree(host, community, ifName)
	if err != nil || len(ifNamesRaw) == 0 {
		ifNamesRaw, err = snmp_subtree(host, community, ifDescr)
		if err != nil {
			return nil, err
		}
	}
	ifAliasesRaw, err := snmp_subtree(host, community, ifAlias)
	if err != nil {
		return nil, err
	}
	ifTypesRaw, err := snmp_subtree(host, community, ifType)
	if err != nil {
		return nil, err
	}
	ifNames := make(map[interface{}]string, len(ifNamesRaw))
	ifAliases := make(map[interface{}]string, len(ifAliasesRaw))
	ifTypes := make(map[interface{}]int64, len(ifTypesRaw))
	for k, v := range ifNamesRaw {
		ifNames[k] = fmt.Sprintf("%s", v)
	}
	for k, v := range ifTypesRaw {
		val, ok := v.(int64)
		if !ok {
			return nil, fmt.Errorf("unexpected type from from MIB::ifType")
		}
		ifTypes[k] = val
	}
	for k, v := range ifAliasesRaw {
		// In case clean would come up empty, prevent the point from being removed
		// by setting our own empty case.
		ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
		if ifAliases[k] == "" {
			ifAliases[k] = "NA"
		}
	}
	var md opentsdb.MultiDataPoint
	add := func(sA snmpAdd) error {
		m, err := snmp_subtree(host, community, sA.oid)
		if err != nil {
			return err
		}
		var sum int64
		for k, v := range m {
			tags := opentsdb.TagSet{
				"host":  host,
				"iface": fmt.Sprintf("%s", k),
				"iname": ifNames[k],
			}
			if sA.dir != "" {
				tags["direction"] = sA.dir
			}
			if iVal, ok := v.(int64); ok && ifTypes[k] == 6 {
				sum += iVal
			}
			Add(&md, switchInterfaceMetric(sA.metric, ifNames[k], ifTypes[k]), v, tags, sA.rate, sA.unit, sA.desc)
			metadata.AddMeta("", tags, "alias", ifAliases[k], false)
		}
		if sA.metric == osNetBytes {
			tags := opentsdb.TagSet{"host": host, "direction": sA.dir}
			Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "The total number of bytes transfered through the network device.")
		}
		return nil
	}
	oids := []snmpAdd{
		{ifHCInBroadcastPkts, osNetBroadcast, "in", metadata.Counter, metadata.Packet, osNetBroadcastDesc},
		{ifHCInMulticastPkts, osNetMulticast, "in", metadata.Counter, metadata.Packet, osNetMulticastDesc},
		{ifHCInUcastPkts, osNetUnicast, "in", metadata.Counter, metadata.Packet, osNetUnicastDesc},
		{ifHCOutBroadcastPkts, osNetBroadcast, "out", metadata.Counter, metadata.Packet, osNetBroadcastDesc},
		{ifHCOutMulticastPkts, osNetMulticast, "out", metadata.Counter, metadata.Packet, osNetMulticastDesc},
		{ifHCOutOctets, osNetBytes, "out", metadata.Counter, metadata.Bytes, osNetBytesDesc},
		{ifHCOutUcastPkts, osNetUnicast, "out", metadata.Counter, metadata.Packet, osNetUnicastDesc},
		{ifHCinOctets, osNetBytes, "in", metadata.Counter, metadata.Bytes, osNetBytesDesc},
		{ifInDiscards, osNetDropped, "in", metadata.Counter, metadata.Packet, osNetDroppedDesc},
		{ifInErrors, osNetErrors, "in", metadata.Counter, metadata.Error, osNetErrorsDesc},
		{ifOutDiscards, osNetDropped, "out", metadata.Counter, metadata.Packet, osNetDroppedDesc},
		{ifOutErrors, osNetErrors, "out", metadata.Counter, metadata.Error, osNetErrorsDesc},
		{ifInPauseFrames, osNetPauseFrames, "in", metadata.Counter, metadata.Frame, osNetPauseFrameDesc},
		{ifOutPauseFrames, osNetPauseFrames, "out", metadata.Counter, metadata.Frame, osNetPauseFrameDesc},
		{ifMTU, osNetMTU, "", metadata.Gauge, metadata.Bytes, osNetMTUDesc},
		{ifHighSpeed, osNetifspeed, "", metadata.Gauge, metadata.Megabit, osNetIfspeedDesc},
		{ifAdminStatus, osNetAdminStatus, "", metadata.Gauge, metadata.StatusCode, osNetAdminStatusDesc},
		{ifOperStatus, osNetOperStatus, "", metadata.Gauge, metadata.StatusCode, osNetOperStatusDesc},
	}
	for _, sA := range oids {
		if err := add(sA); err != nil {
			return nil, err
		}
	}
	return md, nil
}
Example #7
0
// processAzureEADetailRow will take the actual usage data for the provided month
func processAzureEADetailRow(p *azureeabilling.DetailRow, md *opentsdb.MultiDataPoint) error {
	//Don't process todays records as they are subject to change
	nowYear, nowMonth, nowDay := time.Now().Date()
	recordMonth := int(nowMonth)
	if nowYear == p.Year && recordMonth == p.Month && nowDay == p.Day {
		return nil
	}

	resourcePaths := strings.Split(strings.ToLower(p.InstanceID), "/")
	var resourceString string

	if len(resourcePaths) < 8 {
		resourceString = strings.ToLower(p.InstanceID)
	} else {
		resourceIDs := resourcePaths[8:]
		resourceString = strings.Join(resourceIDs, "-")
	}

	if p.ResourceGroup != "" {
		resourceString = fmt.Sprintf("%s-%s", strings.ToLower(p.ResourceGroup), resourceString)
	}

	tags := opentsdb.TagSet{
		"category":    p.MeterCategory,
		"subcategory": fmt.Sprintf("%s-%s", strings.ToLower(p.MeterSubCategory), strings.ToLower(p.MeterName)),
	}

	resourceString, err := opentsdb.Clean(resourceString)
	if err != nil && resourceString != "" {
		tags["resource"] = resourceString
	}

	//Only log billing details if they are enabled in the config
	if azBillConf.LogBillingDetails {
		if p.CostCenter != "" {
			tags["costcenter"] = strings.ToLower(p.CostCenter)
		}
		cleanAccountName, _ := opentsdb.Clean(p.AccountName)
		tags["accountname"] = strings.ToLower(cleanAccountName)
		tags["subscription"] = strings.ToLower(p.SubscriptionName)
	}

	recordDate := time.Date(p.Year, time.Month(p.Month), p.Day, 0, 0, 0, 0, time.UTC)

	//Because we need to log this hourly and we only have daily data, divide the daily cost into hourly costs
	qtyPerHour := p.ConsumedQuantity / hoursInDay

	//ExtendedCost is stored only in a string, because it's a variable number of decimal places. Which means we can't reliably store it in an int, and storing in a float reduces precision.
	//This way we're choosing ourselves to drop the precision, which adds up to around 10-20c under initial testing.
	costPerDay, err := strconv.ParseFloat(p.ExtendedCostRaw, 64)
	if err != nil {
		return err
	}
	costPerHour := costPerDay / hoursInDay

	//Get 24 records for 24 hours in a day
	for i := 0; i < hoursInDay; i++ {
		recordTime := recordDate.Add(time.Duration(i) * time.Hour)
		AddTS(md, "azure.ea.usage", recordTime.Unix(), qtyPerHour, tags, metadata.Gauge, metadata.Count, usageDesc)
		AddTS(md, "azure.ea.cost", recordTime.Unix(), costPerHour, tags, metadata.Gauge, metadata.Count, costDesc)
	}

	return nil
}
Example #8
0
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
	n, err := snmp_subtree(host, community, ifName)
	if err != nil || len(n) == 0 {
		n, err = snmp_subtree(host, community, ifDescr)
		if err != nil {
			return nil, err
		}
	}
	a, err := snmp_subtree(host, community, ifAlias)
	if err != nil {
		return nil, err
	}
	names := make(map[interface{}]string, len(n))
	aliases := make(map[interface{}]string, len(a))
	for k, v := range n {
		names[k] = fmt.Sprintf("%s", v)
	}
	for k, v := range a {
		// In case clean would come up empty, prevent the point from being removed
		// by setting our own empty case.
		aliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
		if aliases[k] == "" {
			aliases[k] = "NA"
		}
	}
	var md opentsdb.MultiDataPoint
	add := func(oid, metric, dir string) error {
		m, err := snmp_subtree(host, community, oid)
		if err != nil {
			return err
		}
		for k, v := range m {
			tags := opentsdb.TagSet{
				"host":      host,
				"direction": dir,
				"iface":     fmt.Sprintf("%d", k),
				"iname":     names[k],
			}
			Add(&md, switch_bond(metric, names[k]), v, tags, metadata.Unknown, metadata.None, "")
			metadata.AddMeta("", tags, "alias", aliases[k], false)
		}
		return nil
	}
	oids := []snmpAdd{
		{ifHCInBroadcastPkts, osNetBroadcast, "in"},
		{ifHCInMulticastPkts, osNetMulticast, "in"},
		{ifHCInUcastPkts, osNetUnicast, "in"},
		{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
		{ifHCOutMulticastPkts, osNetMulticast, "out"},
		{ifHCOutOctets, osNetBytes, "out"},
		{ifHCOutUcastPkts, osNetUnicast, "out"},
		{ifHCinOctets, osNetBytes, "in"},
		{ifInDiscards, osNetDropped, "in"},
		{ifInErrors, osNetErrors, "in"},
		{ifOutDiscards, osNetDropped, "out"},
		{ifOutErrors, osNetErrors, "out"},
		{ifInPauseFrames, osNetPauseFrames, "in"},
		{ifOutPauseFrames, osNetPauseFrames, "out"},
	}
	for _, o := range oids {
		if err := add(o.oid, o.metric, o.dir); err != nil {
			return nil, err
		}
	}
	return md, nil
}
Example #9
0
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
	ifNamesRaw, err := snmp_subtree(host, community, ifName)
	if err != nil || len(ifNamesRaw) == 0 {
		ifNamesRaw, err = snmp_subtree(host, community, ifDescr)
		if err != nil {
			return nil, err
		}
	}
	ifAliasesRaw, err := snmp_subtree(host, community, ifAlias)
	if err != nil {
		return nil, err
	}
	ifTypesRaw, err := snmp_subtree(host, community, ifType)
	if err != nil {
		return nil, err
	}
	ifNames := make(map[interface{}]string, len(ifNamesRaw))
	ifAliases := make(map[interface{}]string, len(ifAliasesRaw))
	ifTypes := make(map[interface{}]int64, len(ifTypesRaw))
	for k, v := range ifNamesRaw {
		ifNames[k] = fmt.Sprintf("%s", v)
	}
	for k, v := range ifTypesRaw {
		val, ok := v.(int64)
		if !ok {
			return nil, fmt.Errorf("unexpected type from from MIB::ifType")
		}
		ifTypes[k] = val
	}
	for k, v := range ifAliasesRaw {
		// In case clean would come up empty, prevent the point from being removed
		// by setting our own empty case.
		ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
		if ifAliases[k] == "" {
			ifAliases[k] = "NA"
		}
	}
	var md opentsdb.MultiDataPoint
	add := func(oid, metric, dir string) error {
		m, err := snmp_subtree(host, community, oid)
		if err != nil {
			return err
		}
		var sum int64
		for k, v := range m {
			tags := opentsdb.TagSet{
				"host":      host,
				"direction": dir,
				"iface":     fmt.Sprintf("%d", k),
				"iname":     ifNames[k],
			}
			if iVal, ok := v.(int64); ok && ifTypes[k] == 6 {
				sum += iVal
			}
			Add(&md, switchInterfaceMetric(metric, ifNames[k], ifTypes[k]), v, tags, metadata.Unknown, metadata.None, "")
			metadata.AddMeta("", tags, "alias", ifAliases[k], false)
		}
		if metric == osNetBytes {
			tags := opentsdb.TagSet{"host": host, "direction": dir}
			Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "")
		}
		return nil
	}
	oids := []snmpAdd{
		{ifHCInBroadcastPkts, osNetBroadcast, "in"},
		{ifHCInMulticastPkts, osNetMulticast, "in"},
		{ifHCInUcastPkts, osNetUnicast, "in"},
		{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
		{ifHCOutMulticastPkts, osNetMulticast, "out"},
		{ifHCOutOctets, osNetBytes, "out"},
		{ifHCOutUcastPkts, osNetUnicast, "out"},
		{ifHCinOctets, osNetBytes, "in"},
		{ifInDiscards, osNetDropped, "in"},
		{ifInErrors, osNetErrors, "in"},
		{ifOutDiscards, osNetDropped, "out"},
		{ifOutErrors, osNetErrors, "out"},
		{ifInPauseFrames, osNetPauseFrames, "in"},
		{ifOutPauseFrames, osNetPauseFrames, "out"},
	}
	for _, o := range oids {
		if err := add(o.oid, o.metric, o.dir); err != nil {
			return nil, err
		}
	}
	return md, nil
}