func getActiveUsersByDimension(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite, dimension string, topN int) error {
call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers").Dimensions("rt:" + dimension)
data, err := call.Do()
if err != nil {
return err
}
tags := opentsdb.TagSet{"site": site.Name}
rows := make(kvList, len(data.Rows))
for i, row := range data.Rows {
// key will always be an string of the dimension we care about.
// For example, 'Chrome' would be a key for the 'browser' dimension.
key, _ := opentsdb.Clean(row[0])
if key == "" {
key = "__blank__"
}
value, err := strconv.Atoi(row[1])
if err != nil {
return fmt.Errorf("Error parsing GA data: %s", err)
}
rows[i] = kv{key: key, value: value}
}
sort.Sort(sort.Reverse(rows))
if topN != -1 && topN < len(rows) {
topRows := make(kvList, topN)
topRows = rows[:topN]
rows = topRows
}
for _, row := range rows {
Add(md, "google.analytics.realtime.activeusers.by_"+dimension, row.value, opentsdb.TagSet{dimension: row.key}.Merge(tags), metadata.Gauge, metadata.ActiveUsers, descActiveUsers)
}
return nil
}
func checkClean(s, t string) error {
if sc, err := opentsdb.Clean(s); s != sc || err != nil {
if err != nil {
return err
}
return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
}
return nil
}
func DatabaseAddCollector(c conf.Database) {
if c.Type != "mysql" {
slog.Fatalf("%v: %v", "invalid Database Type", c.Type)
}
if c.DBName != "" {
cleaned, _ := opentsdb.Clean(c.DBName)
if c.DBName != cleaned {
slog.Fatalf("%v: %v", "invalid Database DBName", c.DBName)
}
}
if c.InstId < 1 {
c.InstId = 1
}
if c.MaxOpenConns < 1 {
c.MaxOpenConns = 2
}
if c.Username == "" {
c.Username = "******"
}
if c.Protocol == "" {
c.Protocol = "tcp"
}
if c.Address == "" {
c.Address = "127.0.0.1"
}
if c.Port < 1 {
c.Port = 3306
}
var tags opentsdb.TagSet
if c.DBName == "" {
tags = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)}
} else {
tags = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)}
}
collectors = append(collectors, &ContinuousCollector{
F: func(collectorStatsChan chan<- *ContinuousCollectorStats) {
DatabaseCollect(c, collectorStatsChan)
},
name: c.Type,
tags: tags,
})
}
func DatabaseGetQueries(c conf.Database) []DatabaseQuery {
queries := make([]DatabaseQuery, 0, len(c.Query))
for _, query := range c.Query {
cleaned, _ := opentsdb.Clean(query.Name)
if query.Name != cleaned {
slog.Warningf("%v: %v: %v: %v", "Database", c.Type, "invaid metric name", query.Name)
continue
}
databaseQuery := DatabaseQuery{name: query.Name, query: query.Query, description: query.Description, hasTime: query.HasTime}
if query.Interval < 1 {
databaseQuery.interval = DefaultFreq
} else {
databaseQuery.interval = time.Duration(query.Interval) * time.Second
}
queries = append(queries, databaseQuery)
}
return queries
}
func DatabaseParseResults(c conf.Database, query *DatabaseQuery, results *[][]string) (opentsdb.MultiDataPoint, error) {
if len(*results) < 1 {
return nil, nil
}
numTagLoops := len((*results)[0]) - 1
if query.hasTime {
numTagLoops--
}
if numTagLoops < 0 {
return nil, fmt.Errorf("invalid number of columns")
}
numTagLoops = (numTagLoops - (numTagLoops % 2)) / 2
var md opentsdb.MultiDataPoint
var tagName string
var tagValue string
var tagsBase opentsdb.TagSet
baseName := c.Type + "."
if c.DBName == "" {
tagsBase = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)}
} else {
tagsBase = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)}
}
for _, result := range *results {
if result[0] == "" {
continue
}
// Check result[0] is vaild metric value?
tags := tagsBase.Copy()
for i := 0; i < numTagLoops; i++ {
if query.hasTime {
tagName = strings.Replace(result[(i*2)+2], " ", "_", -1)
tagValue = strings.Replace(result[(i*2)+3], " ", "_", -1)
} else {
tagName = strings.Replace(result[(i*2)+1], " ", "_", -1)
tagValue = strings.Replace(result[(i*2)+2], " ", "_", -1)
}
tagName, _ = opentsdb.Clean(tagName)
tagValue, _ = opentsdb.Clean(tagValue)
tagName = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagName, "_")
tagValue = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagValue, "_")
if tagName == "" || tagValue == "" {
continue
}
tags = tags.Merge(opentsdb.TagSet{tagName: tagValue})
}
// Add metadata tag set type to configuration file?
if query.hasTime {
timestamp, _ := strconv.ParseInt(result[1], 10, 64)
if timestamp < 1 {
return nil, fmt.Errorf("invalid timestamp")
}
AddTS(&md, baseName+query.name, timestamp, result[0], tags, metadata.Unknown, metadata.None, query.description)
} else {
Add(&md, baseName+query.name, result[0], tags, metadata.Unknown, metadata.None, query.description)
}
}
return md, nil
}
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
ifNamesRaw, err := snmp_subtree(host, community, ifName)
if err != nil || len(ifNamesRaw) == 0 {
ifNamesRaw, err = snmp_subtree(host, community, ifDescr)
if err != nil {
return nil, err
}
}
ifAliasesRaw, err := snmp_subtree(host, community, ifAlias)
if err != nil {
return nil, err
}
ifTypesRaw, err := snmp_subtree(host, community, ifType)
if err != nil {
return nil, err
}
ifNames := make(map[interface{}]string, len(ifNamesRaw))
ifAliases := make(map[interface{}]string, len(ifAliasesRaw))
ifTypes := make(map[interface{}]int64, len(ifTypesRaw))
for k, v := range ifNamesRaw {
ifNames[k] = fmt.Sprintf("%s", v)
}
for k, v := range ifTypesRaw {
val, ok := v.(int64)
if !ok {
return nil, fmt.Errorf("unexpected type from from MIB::ifType")
}
ifTypes[k] = val
}
for k, v := range ifAliasesRaw {
// In case clean would come up empty, prevent the point from being removed
// by setting our own empty case.
ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
if ifAliases[k] == "" {
ifAliases[k] = "NA"
}
}
var md opentsdb.MultiDataPoint
add := func(sA snmpAdd) error {
m, err := snmp_subtree(host, community, sA.oid)
if err != nil {
return err
}
var sum int64
for k, v := range m {
tags := opentsdb.TagSet{
"host": host,
"iface": fmt.Sprintf("%s", k),
"iname": ifNames[k],
}
if sA.dir != "" {
tags["direction"] = sA.dir
}
if iVal, ok := v.(int64); ok && ifTypes[k] == 6 {
sum += iVal
}
Add(&md, switchInterfaceMetric(sA.metric, ifNames[k], ifTypes[k]), v, tags, sA.rate, sA.unit, sA.desc)
metadata.AddMeta("", tags, "alias", ifAliases[k], false)
}
if sA.metric == osNetBytes {
tags := opentsdb.TagSet{"host": host, "direction": sA.dir}
Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "The total number of bytes transfered through the network device.")
}
return nil
}
oids := []snmpAdd{
{ifHCInBroadcastPkts, osNetBroadcast, "in", metadata.Counter, metadata.Packet, osNetBroadcastDesc},
{ifHCInMulticastPkts, osNetMulticast, "in", metadata.Counter, metadata.Packet, osNetMulticastDesc},
{ifHCInUcastPkts, osNetUnicast, "in", metadata.Counter, metadata.Packet, osNetUnicastDesc},
{ifHCOutBroadcastPkts, osNetBroadcast, "out", metadata.Counter, metadata.Packet, osNetBroadcastDesc},
{ifHCOutMulticastPkts, osNetMulticast, "out", metadata.Counter, metadata.Packet, osNetMulticastDesc},
{ifHCOutOctets, osNetBytes, "out", metadata.Counter, metadata.Bytes, osNetBytesDesc},
{ifHCOutUcastPkts, osNetUnicast, "out", metadata.Counter, metadata.Packet, osNetUnicastDesc},
{ifHCinOctets, osNetBytes, "in", metadata.Counter, metadata.Bytes, osNetBytesDesc},
{ifInDiscards, osNetDropped, "in", metadata.Counter, metadata.Packet, osNetDroppedDesc},
{ifInErrors, osNetErrors, "in", metadata.Counter, metadata.Error, osNetErrorsDesc},
{ifOutDiscards, osNetDropped, "out", metadata.Counter, metadata.Packet, osNetDroppedDesc},
{ifOutErrors, osNetErrors, "out", metadata.Counter, metadata.Error, osNetErrorsDesc},
{ifInPauseFrames, osNetPauseFrames, "in", metadata.Counter, metadata.Frame, osNetPauseFrameDesc},
{ifOutPauseFrames, osNetPauseFrames, "out", metadata.Counter, metadata.Frame, osNetPauseFrameDesc},
{ifMTU, osNetMTU, "", metadata.Gauge, metadata.Bytes, osNetMTUDesc},
{ifHighSpeed, osNetifspeed, "", metadata.Gauge, metadata.Megabit, osNetIfspeedDesc},
{ifAdminStatus, osNetAdminStatus, "", metadata.Gauge, metadata.StatusCode, osNetAdminStatusDesc},
{ifOperStatus, osNetOperStatus, "", metadata.Gauge, metadata.StatusCode, osNetOperStatusDesc},
}
for _, sA := range oids {
if err := add(sA); err != nil {
return nil, err
}
}
return md, nil
}
// processAzureEADetailRow will take the actual usage data for the provided month
func processAzureEADetailRow(p *azureeabilling.DetailRow, md *opentsdb.MultiDataPoint) error {
//Don't process todays records as they are subject to change
nowYear, nowMonth, nowDay := time.Now().Date()
recordMonth := int(nowMonth)
if nowYear == p.Year && recordMonth == p.Month && nowDay == p.Day {
return nil
}
resourcePaths := strings.Split(strings.ToLower(p.InstanceID), "/")
var resourceString string
if len(resourcePaths) < 8 {
resourceString = strings.ToLower(p.InstanceID)
} else {
resourceIDs := resourcePaths[8:]
resourceString = strings.Join(resourceIDs, "-")
}
if p.ResourceGroup != "" {
resourceString = fmt.Sprintf("%s-%s", strings.ToLower(p.ResourceGroup), resourceString)
}
tags := opentsdb.TagSet{
"category": p.MeterCategory,
"subcategory": fmt.Sprintf("%s-%s", strings.ToLower(p.MeterSubCategory), strings.ToLower(p.MeterName)),
}
resourceString, err := opentsdb.Clean(resourceString)
if err != nil && resourceString != "" {
tags["resource"] = resourceString
}
//Only log billing details if they are enabled in the config
if azBillConf.LogBillingDetails {
if p.CostCenter != "" {
tags["costcenter"] = strings.ToLower(p.CostCenter)
}
cleanAccountName, _ := opentsdb.Clean(p.AccountName)
tags["accountname"] = strings.ToLower(cleanAccountName)
tags["subscription"] = strings.ToLower(p.SubscriptionName)
}
recordDate := time.Date(p.Year, time.Month(p.Month), p.Day, 0, 0, 0, 0, time.UTC)
//Because we need to log this hourly and we only have daily data, divide the daily cost into hourly costs
qtyPerHour := p.ConsumedQuantity / hoursInDay
//ExtendedCost is stored only in a string, because it's a variable number of decimal places. Which means we can't reliably store it in an int, and storing in a float reduces precision.
//This way we're choosing ourselves to drop the precision, which adds up to around 10-20c under initial testing.
costPerDay, err := strconv.ParseFloat(p.ExtendedCostRaw, 64)
if err != nil {
return err
}
costPerHour := costPerDay / hoursInDay
//Get 24 records for 24 hours in a day
for i := 0; i < hoursInDay; i++ {
recordTime := recordDate.Add(time.Duration(i) * time.Hour)
AddTS(md, "azure.ea.usage", recordTime.Unix(), qtyPerHour, tags, metadata.Gauge, metadata.Count, usageDesc)
AddTS(md, "azure.ea.cost", recordTime.Unix(), costPerHour, tags, metadata.Gauge, metadata.Count, costDesc)
}
return nil
}
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
n, err := snmp_subtree(host, community, ifName)
if err != nil || len(n) == 0 {
n, err = snmp_subtree(host, community, ifDescr)
if err != nil {
return nil, err
}
}
a, err := snmp_subtree(host, community, ifAlias)
if err != nil {
return nil, err
}
names := make(map[interface{}]string, len(n))
aliases := make(map[interface{}]string, len(a))
for k, v := range n {
names[k] = fmt.Sprintf("%s", v)
}
for k, v := range a {
// In case clean would come up empty, prevent the point from being removed
// by setting our own empty case.
aliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
if aliases[k] == "" {
aliases[k] = "NA"
}
}
var md opentsdb.MultiDataPoint
add := func(oid, metric, dir string) error {
m, err := snmp_subtree(host, community, oid)
if err != nil {
return err
}
for k, v := range m {
tags := opentsdb.TagSet{
"host": host,
"direction": dir,
"iface": fmt.Sprintf("%d", k),
"iname": names[k],
}
Add(&md, switch_bond(metric, names[k]), v, tags, metadata.Unknown, metadata.None, "")
metadata.AddMeta("", tags, "alias", aliases[k], false)
}
return nil
}
oids := []snmpAdd{
{ifHCInBroadcastPkts, osNetBroadcast, "in"},
{ifHCInMulticastPkts, osNetMulticast, "in"},
{ifHCInUcastPkts, osNetUnicast, "in"},
{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
{ifHCOutMulticastPkts, osNetMulticast, "out"},
{ifHCOutOctets, osNetBytes, "out"},
{ifHCOutUcastPkts, osNetUnicast, "out"},
{ifHCinOctets, osNetBytes, "in"},
{ifInDiscards, osNetDropped, "in"},
{ifInErrors, osNetErrors, "in"},
{ifOutDiscards, osNetDropped, "out"},
{ifOutErrors, osNetErrors, "out"},
{ifInPauseFrames, osNetPauseFrames, "in"},
{ifOutPauseFrames, osNetPauseFrames, "out"},
}
for _, o := range oids {
if err := add(o.oid, o.metric, o.dir); err != nil {
return nil, err
}
}
return md, nil
}
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) {
ifNamesRaw, err := snmp_subtree(host, community, ifName)
if err != nil || len(ifNamesRaw) == 0 {
ifNamesRaw, err = snmp_subtree(host, community, ifDescr)
if err != nil {
return nil, err
}
}
ifAliasesRaw, err := snmp_subtree(host, community, ifAlias)
if err != nil {
return nil, err
}
ifTypesRaw, err := snmp_subtree(host, community, ifType)
if err != nil {
return nil, err
}
ifNames := make(map[interface{}]string, len(ifNamesRaw))
ifAliases := make(map[interface{}]string, len(ifAliasesRaw))
ifTypes := make(map[interface{}]int64, len(ifTypesRaw))
for k, v := range ifNamesRaw {
ifNames[k] = fmt.Sprintf("%s", v)
}
for k, v := range ifTypesRaw {
val, ok := v.(int64)
if !ok {
return nil, fmt.Errorf("unexpected type from from MIB::ifType")
}
ifTypes[k] = val
}
for k, v := range ifAliasesRaw {
// In case clean would come up empty, prevent the point from being removed
// by setting our own empty case.
ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v))
if ifAliases[k] == "" {
ifAliases[k] = "NA"
}
}
var md opentsdb.MultiDataPoint
add := func(oid, metric, dir string) error {
m, err := snmp_subtree(host, community, oid)
if err != nil {
return err
}
var sum int64
for k, v := range m {
tags := opentsdb.TagSet{
"host": host,
"direction": dir,
"iface": fmt.Sprintf("%d", k),
"iname": ifNames[k],
}
if iVal, ok := v.(int64); ok && ifTypes[k] == 6 {
sum += iVal
}
Add(&md, switchInterfaceMetric(metric, ifNames[k], ifTypes[k]), v, tags, metadata.Unknown, metadata.None, "")
metadata.AddMeta("", tags, "alias", ifAliases[k], false)
}
if metric == osNetBytes {
tags := opentsdb.TagSet{"host": host, "direction": dir}
Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "")
}
return nil
}
oids := []snmpAdd{
{ifHCInBroadcastPkts, osNetBroadcast, "in"},
{ifHCInMulticastPkts, osNetMulticast, "in"},
{ifHCInUcastPkts, osNetUnicast, "in"},
{ifHCOutBroadcastPkts, osNetBroadcast, "out"},
{ifHCOutMulticastPkts, osNetMulticast, "out"},
{ifHCOutOctets, osNetBytes, "out"},
{ifHCOutUcastPkts, osNetUnicast, "out"},
{ifHCinOctets, osNetBytes, "in"},
{ifInDiscards, osNetDropped, "in"},
{ifInErrors, osNetErrors, "in"},
{ifOutDiscards, osNetDropped, "out"},
{ifOutErrors, osNetErrors, "out"},
{ifInPauseFrames, osNetPauseFrames, "in"},
{ifOutPauseFrames, osNetPauseFrames, "out"},
}
for _, o := range oids {
if err := add(o.oid, o.metric, o.dir); err != nil {
return nil, err
}
}
return md, nil
}