Esempio n. 1
0
func extraHopGetAdditionalMetrics(c *gohop.Client, md *opentsdb.MultiDataPoint) error {
	for _, v := range extraHopAdditionalMetrics {
		metric, err := gohop.StoEHMetric(v)
		if err != nil {
			return err
		}
		ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to pull out.
			{Name: metric.MetricSpecName, CalcType: metric.MetricSpecCalcType, KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "proto", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: ehMetricNameEscape(v)},
		}
		mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, metric.MetricCategory, metric.ObjectType, -60000, 0, ms, []int64{metric.ObjectId})
		if err != nil {
			return err
		}

		//This is our function that is going to be executed on each data point in the extrahop dataset
		appendMetricPoint := func(c *gohop.Client, md *opentsdb.MultiDataPoint, a *gohop.MetricStatKeyed, b *[]gohop.MetricStatKeyedValue, d *gohop.MetricStatKeyedValue) {
			switch d.Vtype {
			case "tset":
				for _, e := range d.Tset {
					*md = append(*md, &opentsdb.DataPoint{
						Metric:    ehMetricNameEscape(d.Key.Str),
						Timestamp: a.Time,
						Value:     e.Value,
						Tags:      ehItemNameToTagSet(c, e.Key.Str),
					})
				}
			}
		}

		processGohopStat(&mrk, c, md, appendMetricPoint) //This will loop through our datapoint structure and execute appendCountPoints on each final data piece
	}

	return nil
}
Esempio n. 2
0
func extraHopGetCertificateByCount(c *gohop.Client, md *opentsdb.MultiDataPoint) error {
	//These are the metrics we are populating in this part of the collector
	metricNameCount := "extrahop.certificates"

	//Metadata for the above metrics
	metadata.AddMeta(metricNameCount, nil, "rate", metadata.Gauge, false)
	metadata.AddMeta(metricNameCount, nil, "unit", metadata.Count, false)
	metadata.AddMeta(metricNameCount, nil, "desc", "The number of times a given certificate was seen", false)

	ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to pull out.
		{Name: "cert_subject", KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: metricNameCount},
	}
	mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, "ssl_server_detail", "activity_group", -60000, 0, ms, []int64{int64(extraHopCertificateActivityGroup)})
	if err != nil {
		return err
	}

	//At this time we have a keyed metric response from ExtraHop. We need to find all the stats, then the values of the stats, and then
	//filter out to only the records we want.

	//This is our function that is going to be executed on each data point in the extrahop dataset
	appendCountPoints := func(c *gohop.Client, md *opentsdb.MultiDataPoint, a *gohop.MetricStatKeyed, b *[]gohop.MetricStatKeyedValue, d *gohop.MetricStatKeyedValue) {
		thisPoint := getSSLDataPointFromSet(metricNameCount, c.APIUrl.Host, a.Time, d)
		if thisPoint != nil {
			*md = append(*md, thisPoint)
		}
	}

	processGohopStat(&mrk, c, md, appendCountPoints) //This will loop through our datapoint structure and execute appendCountPoints on each final data piece

	return nil
}
Esempio n. 3
0
/*
	This grabs the complex metrics of the L7 traffic from ExtraHop. It is a complex type because the data is not just a simple time series,
	the data needs to be tagged with vlan, protocol, etc. We can do the network and vlan tagging ourselves, but the protocol tagging comes
	from ExtraHop itself.
*/
func extraHopNetworks(c *gohop.Client, md *opentsdb.MultiDataPoint) error {
	nl, err := c.GetNetworkList(true) //Fetch the network list from ExtraHop, and include VLAN information
	if err != nil {
		return err
	}
	for _, net := range nl { //All found networks
		for _, vlan := range net.Vlans { //All vlans inside this network
			for l7type := range l7types { //All the types of data we want to retrieve for the vlan
				xhMetricName := fmt.Sprintf("extrahop.l7.%s", l7type)
				metricsDropped, metricsKept := 0, 0  //Counters for debugging purposes
				otherValues := make(map[int64]int64) //Container to put any extra time series data that we need to add, for consolidating unnamed or dropped protocols, etc.
				ms := []gohop.MetricSpec{            //Build a metric spec to tell ExtraHop what we want to grab from ExtraHop
					{Name: l7type, KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "proto", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: xhMetricName}, //ExtraHop breaks this by L7 protocol on its own, but we need to tell TSDB what tag to add, which is in this case "proto"
				}
				mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, "app", "vlan", int64(extraHopIntervalSeconds)*-1000, 0, ms, []int64{vlan.VlanId}) //Get the data from ExtraHop
				if err != nil {
					return err
				}
				md2, err := mrk.OpenTSDBDataPoints(ms, "vlan", map[int64]string{vlan.VlanId: fmt.Sprintf("%d", vlan.VlanId)}) //Get the OpenTSDBDataPoints from the ExtraHop data
				if err != nil {
					return err
				}
				valueCutoff := calculateDataCutoff(mrk) //Calculate what the cutoff value will be (used later on when we decide whether or not to consolidate the data)
				for _, dp := range md2 {                //We need to manually process the TSDB datapoints that we've got
					dp.Tags["host"] = c.APIHost
					dp.Tags["network"] = net.Name
					switch extraHopFilterProtoBy { //These are our filter options from the the configuration file. Filter by %, named, or none
					case "toppercent": //Only include protocols that make up a certain % of the traffic
						if dp.Value.(int64) >= valueCutoff[dp.Timestamp] { //It's in the top percent so log it as-is
							*md = append(*md, dp)
							metricsKept++
						} else {
							otherValues[dp.Timestamp] += dp.Value.(int64)
							metricsDropped++
						}
					case "namedprotocols": //Only include protocols that have an actual name (SSL443 excepted)
						if strings.Index(dp.Tags["proto"], "tcp") != 0 && strings.Index(dp.Tags["proto"], "udp") != 0 && (strings.Index(dp.Tags["proto"], "SSL") != 0 || dp.Tags["proto"] == "SSL443") { //The first characters are not tcp or udp.
							*md = append(*md, dp)
							metricsKept++
						} else {
							otherValues[dp.Timestamp] += dp.Value.(int64)
							metricsDropped++
						}
					case "none": //Log everything. Is OK for viewing short timespans, but calculating, 2,000+ protocols over a multi-day window is bad for Bosun's performance
						*md = append(*md, dp)
						metricsKept++
					}

				}
				//Take the consolidated values and add them now too
				for k, v := range otherValues {
					*md = append(*md, &opentsdb.DataPoint{
						Metric:    xhMetricName,
						Timestamp: k,
						Tags:      opentsdb.TagSet{"vlan": fmt.Sprintf("%d", vlan.VlanId), "proto": extraHopOtherProtoName, "host": c.APIHost, "network": net.Name},
						Value:     v,
					})
				}
			}
		}
	}
	return nil
}