func extraHopGetAdditionalMetrics(c *gohop.Client, md *opentsdb.MultiDataPoint) error { for _, v := range extraHopAdditionalMetrics { metric, err := gohop.StoEHMetric(v) if err != nil { return err } ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to pull out. {Name: metric.MetricSpecName, CalcType: metric.MetricSpecCalcType, KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "proto", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: ehMetricNameEscape(v)}, } mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, metric.MetricCategory, metric.ObjectType, -60000, 0, ms, []int64{metric.ObjectId}) if err != nil { return err } //This is our function that is going to be executed on each data point in the extrahop dataset appendMetricPoint := func(c *gohop.Client, md *opentsdb.MultiDataPoint, a *gohop.MetricStatKeyed, b *[]gohop.MetricStatKeyedValue, d *gohop.MetricStatKeyedValue) { switch d.Vtype { case "tset": for _, e := range d.Tset { *md = append(*md, &opentsdb.DataPoint{ Metric: ehMetricNameEscape(d.Key.Str), Timestamp: a.Time, Value: e.Value, Tags: ehItemNameToTagSet(c, e.Key.Str), }) } } } processGohopStat(&mrk, c, md, appendMetricPoint) //This will loop through our datapoint structure and execute appendCountPoints on each final data piece } return nil }
func extraHopGetCertificateByCount(c *gohop.Client, md *opentsdb.MultiDataPoint) error { //These are the metrics we are populating in this part of the collector metricNameCount := "extrahop.certificates" //Metadata for the above metrics metadata.AddMeta(metricNameCount, nil, "rate", metadata.Gauge, false) metadata.AddMeta(metricNameCount, nil, "unit", metadata.Count, false) metadata.AddMeta(metricNameCount, nil, "desc", "The number of times a given certificate was seen", false) ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to pull out. {Name: "cert_subject", KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: metricNameCount}, } mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, "ssl_server_detail", "activity_group", -60000, 0, ms, []int64{int64(extraHopCertificateActivityGroup)}) if err != nil { return err } //At this time we have a keyed metric response from ExtraHop. We need to find all the stats, then the values of the stats, and then //filter out to only the records we want. //This is our function that is going to be executed on each data point in the extrahop dataset appendCountPoints := func(c *gohop.Client, md *opentsdb.MultiDataPoint, a *gohop.MetricStatKeyed, b *[]gohop.MetricStatKeyedValue, d *gohop.MetricStatKeyedValue) { thisPoint := getSSLDataPointFromSet(metricNameCount, c.APIUrl.Host, a.Time, d) if thisPoint != nil { *md = append(*md, thisPoint) } } processGohopStat(&mrk, c, md, appendCountPoints) //This will loop through our datapoint structure and execute appendCountPoints on each final data piece return nil }
/* This grabs the complex metrics of the L7 traffic from ExtraHop. It is a complex type because the data is not just a simple time series, the data needs to be tagged with vlan, protocol, etc. We can do the network and vlan tagging ourselves, but the protocol tagging comes from ExtraHop itself. */ func extraHopNetworks(c *gohop.Client, md *opentsdb.MultiDataPoint) error { nl, err := c.GetNetworkList(true) //Fetch the network list from ExtraHop, and include VLAN information if err != nil { return err } for _, net := range nl { //All found networks for _, vlan := range net.Vlans { //All vlans inside this network for l7type := range l7types { //All the types of data we want to retrieve for the vlan xhMetricName := fmt.Sprintf("extrahop.l7.%s", l7type) metricsDropped, metricsKept := 0, 0 //Counters for debugging purposes otherValues := make(map[int64]int64) //Container to put any extra time series data that we need to add, for consolidating unnamed or dropped protocols, etc. ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to grab from ExtraHop {Name: l7type, KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "proto", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: xhMetricName}, //ExtraHop breaks this by L7 protocol on its own, but we need to tell TSDB what tag to add, which is in this case "proto" } mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, "app", "vlan", int64(extraHopIntervalSeconds)*-1000, 0, ms, []int64{vlan.VlanId}) //Get the data from ExtraHop if err != nil { return err } md2, err := mrk.OpenTSDBDataPoints(ms, "vlan", map[int64]string{vlan.VlanId: fmt.Sprintf("%d", vlan.VlanId)}) //Get the OpenTSDBDataPoints from the ExtraHop data if err != nil { return err } valueCutoff := calculateDataCutoff(mrk) //Calculate what the cutoff value will be (used later on when we decide whether or not to consolidate the data) for _, dp := range md2 { //We need to manually process the TSDB datapoints that we've got dp.Tags["host"] = c.APIHost dp.Tags["network"] = net.Name switch extraHopFilterProtoBy { //These are our filter options from the the configuration file. Filter by %, named, or none case "toppercent": //Only include protocols that make up a certain % of the traffic if dp.Value.(int64) >= valueCutoff[dp.Timestamp] { //It's in the top percent so log it as-is *md = append(*md, dp) metricsKept++ } else { otherValues[dp.Timestamp] += dp.Value.(int64) metricsDropped++ } case "namedprotocols": //Only include protocols that have an actual name (SSL443 excepted) if strings.Index(dp.Tags["proto"], "tcp") != 0 && strings.Index(dp.Tags["proto"], "udp") != 0 && (strings.Index(dp.Tags["proto"], "SSL") != 0 || dp.Tags["proto"] == "SSL443") { //The first characters are not tcp or udp. *md = append(*md, dp) metricsKept++ } else { otherValues[dp.Timestamp] += dp.Value.(int64) metricsDropped++ } case "none": //Log everything. Is OK for viewing short timespans, but calculating, 2,000+ protocols over a multi-day window is bad for Bosun's performance *md = append(*md, dp) metricsKept++ } } //Take the consolidated values and add them now too for k, v := range otherValues { *md = append(*md, &opentsdb.DataPoint{ Metric: xhMetricName, Timestamp: k, Tags: opentsdb.TagSet{"vlan": fmt.Sprintf("%d", vlan.VlanId), "proto": extraHopOtherProtoName, "host": c.APIHost, "network": net.Name}, Value: v, }) } } } } return nil }