func getActiveUsersByDimension(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf.GoogleAnalyticsSite, dimension string, topN int) error { call := svc.Data.Realtime.Get("ga:"+site.Profile, "rt:activeusers").Dimensions("rt:" + dimension) data, err := call.Do() if err != nil { return err } tags := opentsdb.TagSet{"site": site.Name} rows := make(kvList, len(data.Rows)) for i, row := range data.Rows { // key will always be an string of the dimension we care about. // For example, 'Chrome' would be a key for the 'browser' dimension. key, _ := opentsdb.Clean(row[0]) if key == "" { key = "__blank__" } value, err := strconv.Atoi(row[1]) if err != nil { return fmt.Errorf("Error parsing GA data: %s", err) } rows[i] = kv{key: key, value: value} } sort.Sort(sort.Reverse(rows)) if topN != -1 && topN < len(rows) { topRows := make(kvList, topN) topRows = rows[:topN] rows = topRows } for _, row := range rows { Add(md, "google.analytics.realtime.activeusers.by_"+dimension, row.value, opentsdb.TagSet{dimension: row.key}.Merge(tags), metadata.Gauge, metadata.ActiveUsers, descActiveUsers) } return nil }
func checkClean(s, t string) error { if sc, err := opentsdb.Clean(s); s != sc || err != nil { if err != nil { return err } return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s) } return nil }
func DatabaseAddCollector(c conf.Database) { if c.Type != "mysql" { slog.Fatalf("%v: %v", "invalid Database Type", c.Type) } if c.DBName != "" { cleaned, _ := opentsdb.Clean(c.DBName) if c.DBName != cleaned { slog.Fatalf("%v: %v", "invalid Database DBName", c.DBName) } } if c.InstId < 1 { c.InstId = 1 } if c.MaxOpenConns < 1 { c.MaxOpenConns = 2 } if c.Username == "" { c.Username = "******" } if c.Protocol == "" { c.Protocol = "tcp" } if c.Address == "" { c.Address = "127.0.0.1" } if c.Port < 1 { c.Port = 3306 } var tags opentsdb.TagSet if c.DBName == "" { tags = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)} } else { tags = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)} } collectors = append(collectors, &ContinuousCollector{ F: func(collectorStatsChan chan<- *ContinuousCollectorStats) { DatabaseCollect(c, collectorStatsChan) }, name: c.Type, tags: tags, }) }
func DatabaseGetQueries(c conf.Database) []DatabaseQuery { queries := make([]DatabaseQuery, 0, len(c.Query)) for _, query := range c.Query { cleaned, _ := opentsdb.Clean(query.Name) if query.Name != cleaned { slog.Warningf("%v: %v: %v: %v", "Database", c.Type, "invaid metric name", query.Name) continue } databaseQuery := DatabaseQuery{name: query.Name, query: query.Query, description: query.Description, hasTime: query.HasTime} if query.Interval < 1 { databaseQuery.interval = DefaultFreq } else { databaseQuery.interval = time.Duration(query.Interval) * time.Second } queries = append(queries, databaseQuery) } return queries }
func DatabaseParseResults(c conf.Database, query *DatabaseQuery, results *[][]string) (opentsdb.MultiDataPoint, error) { if len(*results) < 1 { return nil, nil } numTagLoops := len((*results)[0]) - 1 if query.hasTime { numTagLoops-- } if numTagLoops < 0 { return nil, fmt.Errorf("invalid number of columns") } numTagLoops = (numTagLoops - (numTagLoops % 2)) / 2 var md opentsdb.MultiDataPoint var tagName string var tagValue string var tagsBase opentsdb.TagSet baseName := c.Type + "." if c.DBName == "" { tagsBase = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)} } else { tagsBase = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)} } for _, result := range *results { if result[0] == "" { continue } // Check result[0] is vaild metric value? tags := tagsBase.Copy() for i := 0; i < numTagLoops; i++ { if query.hasTime { tagName = strings.Replace(result[(i*2)+2], " ", "_", -1) tagValue = strings.Replace(result[(i*2)+3], " ", "_", -1) } else { tagName = strings.Replace(result[(i*2)+1], " ", "_", -1) tagValue = strings.Replace(result[(i*2)+2], " ", "_", -1) } tagName, _ = opentsdb.Clean(tagName) tagValue, _ = opentsdb.Clean(tagValue) tagName = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagName, "_") tagValue = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagValue, "_") if tagName == "" || tagValue == "" { continue } tags = tags.Merge(opentsdb.TagSet{tagName: tagValue}) } // Add metadata tag set type to configuration file? if query.hasTime { timestamp, _ := strconv.ParseInt(result[1], 10, 64) if timestamp < 1 { return nil, fmt.Errorf("invalid timestamp") } AddTS(&md, baseName+query.name, timestamp, result[0], tags, metadata.Unknown, metadata.None, query.description) } else { Add(&md, baseName+query.name, result[0], tags, metadata.Unknown, metadata.None, query.description) } } return md, nil }
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) { ifNamesRaw, err := snmp_subtree(host, community, ifName) if err != nil || len(ifNamesRaw) == 0 { ifNamesRaw, err = snmp_subtree(host, community, ifDescr) if err != nil { return nil, err } } ifAliasesRaw, err := snmp_subtree(host, community, ifAlias) if err != nil { return nil, err } ifTypesRaw, err := snmp_subtree(host, community, ifType) if err != nil { return nil, err } ifNames := make(map[interface{}]string, len(ifNamesRaw)) ifAliases := make(map[interface{}]string, len(ifAliasesRaw)) ifTypes := make(map[interface{}]int64, len(ifTypesRaw)) for k, v := range ifNamesRaw { ifNames[k] = fmt.Sprintf("%s", v) } for k, v := range ifTypesRaw { val, ok := v.(int64) if !ok { return nil, fmt.Errorf("unexpected type from from MIB::ifType") } ifTypes[k] = val } for k, v := range ifAliasesRaw { // In case clean would come up empty, prevent the point from being removed // by setting our own empty case. ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v)) if ifAliases[k] == "" { ifAliases[k] = "NA" } } var md opentsdb.MultiDataPoint add := func(sA snmpAdd) error { m, err := snmp_subtree(host, community, sA.oid) if err != nil { return err } var sum int64 for k, v := range m { tags := opentsdb.TagSet{ "host": host, "iface": fmt.Sprintf("%s", k), "iname": ifNames[k], } if sA.dir != "" { tags["direction"] = sA.dir } if iVal, ok := v.(int64); ok && ifTypes[k] == 6 { sum += iVal } Add(&md, switchInterfaceMetric(sA.metric, ifNames[k], ifTypes[k]), v, tags, sA.rate, sA.unit, sA.desc) metadata.AddMeta("", tags, "alias", ifAliases[k], false) } if sA.metric == osNetBytes { tags := opentsdb.TagSet{"host": host, "direction": sA.dir} Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "The total number of bytes transfered through the network device.") } return nil } oids := []snmpAdd{ {ifHCInBroadcastPkts, osNetBroadcast, "in", metadata.Counter, metadata.Packet, osNetBroadcastDesc}, {ifHCInMulticastPkts, osNetMulticast, "in", metadata.Counter, metadata.Packet, osNetMulticastDesc}, {ifHCInUcastPkts, osNetUnicast, "in", metadata.Counter, metadata.Packet, osNetUnicastDesc}, {ifHCOutBroadcastPkts, osNetBroadcast, "out", metadata.Counter, metadata.Packet, osNetBroadcastDesc}, {ifHCOutMulticastPkts, osNetMulticast, "out", metadata.Counter, metadata.Packet, osNetMulticastDesc}, {ifHCOutOctets, osNetBytes, "out", metadata.Counter, metadata.Bytes, osNetBytesDesc}, {ifHCOutUcastPkts, osNetUnicast, "out", metadata.Counter, metadata.Packet, osNetUnicastDesc}, {ifHCinOctets, osNetBytes, "in", metadata.Counter, metadata.Bytes, osNetBytesDesc}, {ifInDiscards, osNetDropped, "in", metadata.Counter, metadata.Packet, osNetDroppedDesc}, {ifInErrors, osNetErrors, "in", metadata.Counter, metadata.Error, osNetErrorsDesc}, {ifOutDiscards, osNetDropped, "out", metadata.Counter, metadata.Packet, osNetDroppedDesc}, {ifOutErrors, osNetErrors, "out", metadata.Counter, metadata.Error, osNetErrorsDesc}, {ifInPauseFrames, osNetPauseFrames, "in", metadata.Counter, metadata.Frame, osNetPauseFrameDesc}, {ifOutPauseFrames, osNetPauseFrames, "out", metadata.Counter, metadata.Frame, osNetPauseFrameDesc}, {ifMTU, osNetMTU, "", metadata.Gauge, metadata.Bytes, osNetMTUDesc}, {ifHighSpeed, osNetifspeed, "", metadata.Gauge, metadata.Megabit, osNetIfspeedDesc}, {ifAdminStatus, osNetAdminStatus, "", metadata.Gauge, metadata.StatusCode, osNetAdminStatusDesc}, {ifOperStatus, osNetOperStatus, "", metadata.Gauge, metadata.StatusCode, osNetOperStatusDesc}, } for _, sA := range oids { if err := add(sA); err != nil { return nil, err } } return md, nil }
// processAzureEADetailRow will take the actual usage data for the provided month func processAzureEADetailRow(p *azureeabilling.DetailRow, md *opentsdb.MultiDataPoint) error { //Don't process todays records as they are subject to change nowYear, nowMonth, nowDay := time.Now().Date() recordMonth := int(nowMonth) if nowYear == p.Year && recordMonth == p.Month && nowDay == p.Day { return nil } resourcePaths := strings.Split(strings.ToLower(p.InstanceID), "/") var resourceString string if len(resourcePaths) < 8 { resourceString = strings.ToLower(p.InstanceID) } else { resourceIDs := resourcePaths[8:] resourceString = strings.Join(resourceIDs, "-") } if p.ResourceGroup != "" { resourceString = fmt.Sprintf("%s-%s", strings.ToLower(p.ResourceGroup), resourceString) } tags := opentsdb.TagSet{ "category": p.MeterCategory, "subcategory": fmt.Sprintf("%s-%s", strings.ToLower(p.MeterSubCategory), strings.ToLower(p.MeterName)), } resourceString, err := opentsdb.Clean(resourceString) if err != nil && resourceString != "" { tags["resource"] = resourceString } //Only log billing details if they are enabled in the config if azBillConf.LogBillingDetails { if p.CostCenter != "" { tags["costcenter"] = strings.ToLower(p.CostCenter) } cleanAccountName, _ := opentsdb.Clean(p.AccountName) tags["accountname"] = strings.ToLower(cleanAccountName) tags["subscription"] = strings.ToLower(p.SubscriptionName) } recordDate := time.Date(p.Year, time.Month(p.Month), p.Day, 0, 0, 0, 0, time.UTC) //Because we need to log this hourly and we only have daily data, divide the daily cost into hourly costs qtyPerHour := p.ConsumedQuantity / hoursInDay //ExtendedCost is stored only in a string, because it's a variable number of decimal places. Which means we can't reliably store it in an int, and storing in a float reduces precision. //This way we're choosing ourselves to drop the precision, which adds up to around 10-20c under initial testing. costPerDay, err := strconv.ParseFloat(p.ExtendedCostRaw, 64) if err != nil { return err } costPerHour := costPerDay / hoursInDay //Get 24 records for 24 hours in a day for i := 0; i < hoursInDay; i++ { recordTime := recordDate.Add(time.Duration(i) * time.Hour) AddTS(md, "azure.ea.usage", recordTime.Unix(), qtyPerHour, tags, metadata.Gauge, metadata.Count, usageDesc) AddTS(md, "azure.ea.cost", recordTime.Unix(), costPerHour, tags, metadata.Gauge, metadata.Count, costDesc) } return nil }
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) { n, err := snmp_subtree(host, community, ifName) if err != nil || len(n) == 0 { n, err = snmp_subtree(host, community, ifDescr) if err != nil { return nil, err } } a, err := snmp_subtree(host, community, ifAlias) if err != nil { return nil, err } names := make(map[interface{}]string, len(n)) aliases := make(map[interface{}]string, len(a)) for k, v := range n { names[k] = fmt.Sprintf("%s", v) } for k, v := range a { // In case clean would come up empty, prevent the point from being removed // by setting our own empty case. aliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v)) if aliases[k] == "" { aliases[k] = "NA" } } var md opentsdb.MultiDataPoint add := func(oid, metric, dir string) error { m, err := snmp_subtree(host, community, oid) if err != nil { return err } for k, v := range m { tags := opentsdb.TagSet{ "host": host, "direction": dir, "iface": fmt.Sprintf("%d", k), "iname": names[k], } Add(&md, switch_bond(metric, names[k]), v, tags, metadata.Unknown, metadata.None, "") metadata.AddMeta("", tags, "alias", aliases[k], false) } return nil } oids := []snmpAdd{ {ifHCInBroadcastPkts, osNetBroadcast, "in"}, {ifHCInMulticastPkts, osNetMulticast, "in"}, {ifHCInUcastPkts, osNetUnicast, "in"}, {ifHCOutBroadcastPkts, osNetBroadcast, "out"}, {ifHCOutMulticastPkts, osNetMulticast, "out"}, {ifHCOutOctets, osNetBytes, "out"}, {ifHCOutUcastPkts, osNetUnicast, "out"}, {ifHCinOctets, osNetBytes, "in"}, {ifInDiscards, osNetDropped, "in"}, {ifInErrors, osNetErrors, "in"}, {ifOutDiscards, osNetDropped, "out"}, {ifOutErrors, osNetErrors, "out"}, {ifInPauseFrames, osNetPauseFrames, "in"}, {ifOutPauseFrames, osNetPauseFrames, "out"}, } for _, o := range oids { if err := add(o.oid, o.metric, o.dir); err != nil { return nil, err } } return md, nil }
func c_snmp_ifaces(community, host string) (opentsdb.MultiDataPoint, error) { ifNamesRaw, err := snmp_subtree(host, community, ifName) if err != nil || len(ifNamesRaw) == 0 { ifNamesRaw, err = snmp_subtree(host, community, ifDescr) if err != nil { return nil, err } } ifAliasesRaw, err := snmp_subtree(host, community, ifAlias) if err != nil { return nil, err } ifTypesRaw, err := snmp_subtree(host, community, ifType) if err != nil { return nil, err } ifNames := make(map[interface{}]string, len(ifNamesRaw)) ifAliases := make(map[interface{}]string, len(ifAliasesRaw)) ifTypes := make(map[interface{}]int64, len(ifTypesRaw)) for k, v := range ifNamesRaw { ifNames[k] = fmt.Sprintf("%s", v) } for k, v := range ifTypesRaw { val, ok := v.(int64) if !ok { return nil, fmt.Errorf("unexpected type from from MIB::ifType") } ifTypes[k] = val } for k, v := range ifAliasesRaw { // In case clean would come up empty, prevent the point from being removed // by setting our own empty case. ifAliases[k], _ = opentsdb.Clean(fmt.Sprintf("%s", v)) if ifAliases[k] == "" { ifAliases[k] = "NA" } } var md opentsdb.MultiDataPoint add := func(oid, metric, dir string) error { m, err := snmp_subtree(host, community, oid) if err != nil { return err } var sum int64 for k, v := range m { tags := opentsdb.TagSet{ "host": host, "direction": dir, "iface": fmt.Sprintf("%d", k), "iname": ifNames[k], } if iVal, ok := v.(int64); ok && ifTypes[k] == 6 { sum += iVal } Add(&md, switchInterfaceMetric(metric, ifNames[k], ifTypes[k]), v, tags, metadata.Unknown, metadata.None, "") metadata.AddMeta("", tags, "alias", ifAliases[k], false) } if metric == osNetBytes { tags := opentsdb.TagSet{"host": host, "direction": dir} Add(&md, osNetBytes+".total", sum, tags, metadata.Counter, metadata.Bytes, "") } return nil } oids := []snmpAdd{ {ifHCInBroadcastPkts, osNetBroadcast, "in"}, {ifHCInMulticastPkts, osNetMulticast, "in"}, {ifHCInUcastPkts, osNetUnicast, "in"}, {ifHCOutBroadcastPkts, osNetBroadcast, "out"}, {ifHCOutMulticastPkts, osNetMulticast, "out"}, {ifHCOutOctets, osNetBytes, "out"}, {ifHCOutUcastPkts, osNetUnicast, "out"}, {ifHCinOctets, osNetBytes, "in"}, {ifInDiscards, osNetDropped, "in"}, {ifInErrors, osNetErrors, "in"}, {ifOutDiscards, osNetDropped, "out"}, {ifOutErrors, osNetErrors, "out"}, {ifInPauseFrames, osNetPauseFrames, "in"}, {ifOutPauseFrames, osNetPauseFrames, "out"}, } for _, o := range oids { if err := add(o.oid, o.metric, o.dir); err != nil { return nil, err } } return md, nil }