/*
This grabs the complex metrics of the L7 traffic from ExtraHop. It is a complex type because the data is not just a simple time series,
the data needs to be tagged with vlan, protocol, etc. We can do the network and vlan tagging ourselves, but the protocol tagging comes
from ExtraHop itself.
*/
func extraHopNetworks(c *gohop.Client, md *opentsdb.MultiDataPoint) error {
nl, err := c.GetNetworkList(true) //Fetch the network list from ExtraHop, and include VLAN information
if err != nil {
return err
}
for _, net := range nl { //All found networks
for _, vlan := range net.Vlans { //All vlans inside this network
for l7type := range l7types { //All the types of data we want to retrieve for the vlan
xhMetricName := fmt.Sprintf("extrahop.l7.%s", l7type)
metricsDropped, metricsKept := 0, 0 //Counters for debugging purposes
otherValues := make(map[int64]int64) //Container to put any extra time series data that we need to add, for consolidating unnamed or dropped protocols, etc.
ms := []gohop.MetricSpec{ //Build a metric spec to tell ExtraHop what we want to grab from ExtraHop
{Name: l7type, KeyPair: gohop.KeyPair{Key1Regex: "", Key2Regex: "", OpenTSDBKey1: "proto", Key2OpenTSDBKey2: ""}, OpenTSDBMetric: xhMetricName}, //ExtraHop breaks this by L7 protocol on its own, but we need to tell TSDB what tag to add, which is in this case "proto"
}
mrk, err := c.KeyedMetricQuery(gohop.Cycle30Sec, "app", "vlan", int64(extraHopIntervalSeconds)*-1000, 0, ms, []int64{vlan.VlanId}) //Get the data from ExtraHop
if err != nil {
return err
}
md2, err := mrk.OpenTSDBDataPoints(ms, "vlan", map[int64]string{vlan.VlanId: fmt.Sprintf("%d", vlan.VlanId)}) //Get the OpenTSDBDataPoints from the ExtraHop data
if err != nil {
return err
}
valueCutoff := calculateDataCutoff(mrk) //Calculate what the cutoff value will be (used later on when we decide whether or not to consolidate the data)
for _, dp := range md2 { //We need to manually process the TSDB datapoints that we've got
dp.Tags["host"] = c.APIHost
dp.Tags["network"] = net.Name
switch extraHopFilterProtoBy { //These are our filter options from the the configuration file. Filter by %, named, or none
case "toppercent": //Only include protocols that make up a certain % of the traffic
if dp.Value.(int64) >= valueCutoff[dp.Timestamp] { //It's in the top percent so log it as-is
*md = append(*md, dp)
metricsKept++
} else {
otherValues[dp.Timestamp] += dp.Value.(int64)
metricsDropped++
}
case "namedprotocols": //Only include protocols that have an actual name (SSL443 excepted)
if strings.Index(dp.Tags["proto"], "tcp") != 0 && strings.Index(dp.Tags["proto"], "udp") != 0 && (strings.Index(dp.Tags["proto"], "SSL") != 0 || dp.Tags["proto"] == "SSL443") { //The first characters are not tcp or udp.
*md = append(*md, dp)
metricsKept++
} else {
otherValues[dp.Timestamp] += dp.Value.(int64)
metricsDropped++
}
case "none": //Log everything. Is OK for viewing short timespans, but calculating, 2,000+ protocols over a multi-day window is bad for Bosun's performance
*md = append(*md, dp)
metricsKept++
}
}
//Take the consolidated values and add them now too
for k, v := range otherValues {
*md = append(*md, &opentsdb.DataPoint{
Metric: xhMetricName,
Timestamp: k,
Tags: opentsdb.TagSet{"vlan": fmt.Sprintf("%d", vlan.VlanId), "proto": extraHopOtherProtoName, "host": c.APIHost, "network": net.Name},
Value: v,
})
}
}
}
}
return nil
}
