func CreateStats(statHistory map[enum.CacheName][]cache.Result, toData todata.TOData, crStates peer.Crstates, lastStats LastStats, now time.Time) (Stats, LastStats, error) { start := time.Now() dsStats := NewStats() for deliveryService, _ := range toData.DeliveryServiceServers { if deliveryService == "" { log.Errorf("EMPTY CreateStats deliveryService") continue } dsStats.DeliveryService[enum.DeliveryServiceName(deliveryService)] = *dsdata.NewStat() } dsStats = setStaticData(dsStats, toData.DeliveryServiceServers) var err error dsStats, err = addAvailableData(dsStats, crStates, toData.ServerCachegroups, toData.ServerDeliveryServices, toData.ServerTypes, statHistory) // TODO move after stat summarisation if err != nil { return dsStats, lastStats, fmt.Errorf("Error getting Cache availability data: %v", err) } for server, history := range statHistory { if len(history) < 1 { continue // TODO warn? } cachegroup, ok := toData.ServerCachegroups[server] if !ok { log.Warnf("server %s has no cachegroup, skipping\n", server) continue } serverType, ok := toData.ServerTypes[enum.CacheName(server)] if !ok { log.Warnf("server %s not in CRConfig, skipping\n", server) continue } result := history[len(history)-1] // TODO check result.PrecomputedData.Errors for ds, resultStat := range result.PrecomputedData.DeliveryServiceStats { if ds == "" { log.Errorf("EMPTY precomputed delivery service") continue } if _, ok := dsStats.DeliveryService[ds]; !ok { dsStats.DeliveryService[ds] = resultStat continue } httpDsStat := dsStats.DeliveryService[ds] httpDsStat.TotalStats = httpDsStat.TotalStats.Sum(resultStat.TotalStats) httpDsStat.CacheGroups[cachegroup] = httpDsStat.CacheGroups[cachegroup].Sum(resultStat.CacheGroups[cachegroup]) httpDsStat.Types[serverType] = httpDsStat.Types[serverType].Sum(resultStat.Types[serverType]) httpDsStat.Caches[server] = httpDsStat.Caches[server].Sum(resultStat.Caches[server]) httpDsStat.CachesTimeReceived[server] = resultStat.CachesTimeReceived[server] httpDsStat.CommonStats = dsStats.DeliveryService[ds].CommonStats dsStats.DeliveryService[ds] = httpDsStat // TODO determine if necessary } } perSecStats, lastStats := addPerSecStats(statHistory, dsStats, lastStats, now, toData.ServerCachegroups, toData.ServerTypes) log.Infof("CreateStats took %v\n", time.Since(start)) return perSecStats, lastStats, nil }
// TODO precompute, move to TOData; call when we get new delivery services, instead of every time we create new stats func createRegexes(dsToRegex map[string][]string) (Regexes, error) { dsRegexes := Regexes{ DirectMatches: map[string]enum.DeliveryServiceName{}, DotStartSlashDotFooSlashDotDotStar: map[string]enum.DeliveryServiceName{}, RegexMatch: map[*regexp.Regexp]enum.DeliveryServiceName{}, } for dsStr, regexStrs := range dsToRegex { ds := enum.DeliveryServiceName(dsStr) for _, regexStr := range regexStrs { prefix := `.*\.` suffix := `\..*` if strings.HasPrefix(regexStr, prefix) && strings.HasSuffix(regexStr, suffix) { matchStr := regexStr[len(prefix) : len(regexStr)-len(suffix)] if otherDs, ok := dsRegexes.DotStartSlashDotFooSlashDotDotStar[matchStr]; ok { return dsRegexes, fmt.Errorf("duplicate regex %s (%s) in %s and %s", regexStr, matchStr, ds, otherDs) } dsRegexes.DotStartSlashDotFooSlashDotDotStar[matchStr] = ds continue } if !strings.ContainsAny(regexStr, `[]^\:{}()|?+*,=%@<>!'`) { if otherDs, ok := dsRegexes.DirectMatches[regexStr]; ok { return dsRegexes, fmt.Errorf("duplicate Regex %s in %s and %s", regexStr, ds, otherDs) } dsRegexes.DirectMatches[regexStr] = ds continue } // TODO warn? regex matches are unusual r, err := regexp.Compile(regexStr) if err != nil { return dsRegexes, fmt.Errorf("regex %s failed to compile: %v", regexStr, err) } dsRegexes.RegexMatch[r] = ds } } return dsRegexes, nil }
func addAvailableData(dsStats Stats, crStates peer.Crstates, serverCachegroups map[enum.CacheName]enum.CacheGroupName, serverDs map[enum.CacheName][]enum.DeliveryServiceName, serverTypes map[enum.CacheName]enum.CacheType, statHistory map[enum.CacheName][]cache.Result) (Stats, error) { for cache, available := range crStates.Caches { cacheGroup, ok := serverCachegroups[cache] if !ok { log.Warnf("CreateStats not adding availability data for '%s': not found in Cachegroups\n", cache) continue } deliveryServices, ok := serverDs[cache] if !ok { log.Warnf("CreateStats not adding availability data for '%s': not found in DeliveryServices\n", cache) continue } cacheType, ok := serverTypes[enum.CacheName(cache)] if !ok { log.Warnf("CreateStats not adding availability data for '%s': not found in Server Types\n", cache) continue } for _, deliveryService := range deliveryServices { if deliveryService == "" { log.Errorf("EMPTY addAvailableData DS") // various bugs in other functions can cause this - this will help identify and debug them. continue } stat, ok := dsStats.DeliveryService[enum.DeliveryServiceName(deliveryService)] if !ok { log.Warnf("CreateStats not adding availability data for '%s': not found in Stats\n", cache) continue // TODO log warning? Error? } if available.IsAvailable { // c.IsAvailable.Value stat.CommonStats.IsAvailable.Value = true stat.CommonStats.CachesAvailableNum.Value++ cacheGroupStats := stat.CacheGroups[enum.CacheGroupName(cacheGroup)] cacheGroupStats.IsAvailable.Value = true stat.CacheGroups[enum.CacheGroupName(cacheGroup)] = cacheGroupStats stat.TotalStats.IsAvailable.Value = true typeStats := stat.Types[cacheType] typeStats.IsAvailable.Value = true stat.Types[cacheType] = typeStats } // TODO fix nested ifs if results, ok := statHistory[enum.CacheName(cache)]; ok { if len(results) < 1 { log.Warnf("no results %v %v\n", cache, deliveryService) } else { result := results[0] if result.PrecomputedData.Reporting { stat.CommonStats.CachesReporting[enum.CacheName(cache)] = true } else { log.Debugf("no reporting %v %v\n", cache, deliveryService) } } } else { log.Debugf("no result for %v %v\n", cache, deliveryService) } dsStats.DeliveryService[enum.DeliveryServiceName(deliveryService)] = stat // TODO Necessary? Remove? } } return dsStats, nil }
// NewDSStatFilter takes the HTTP query parameters and creates a cache.Filter, filtering according to the query parameters passed. // Query parameters used are `hc`, `stats`, `wildcard`, `type`, and `deliveryservices`. // If `hc` is 0, all history is returned. If `hc` is empty, 1 history is returned. // If `stats` is empty, all stats are returned. // If `wildcard` is empty, `stats` is considered exact. // If `type` is empty, all types are returned. func NewDSStatFilter(params url.Values, dsTypes map[enum.DeliveryServiceName]enum.DSType) (dsdata.Filter, error) { validParams := map[string]struct{}{"hc": struct{}{}, "stats": struct{}{}, "wildcard": struct{}{}, "type": struct{}{}, "deliveryservices": struct{}{}} if len(params) > len(validParams) { return nil, fmt.Errorf("invalid query parameters") } for param, _ := range params { if _, ok := validParams[param]; !ok { return nil, fmt.Errorf("invalid query parameter '%v'", param) } } historyCount := 1 if paramHc, exists := params["hc"]; exists && len(paramHc) > 0 { v, err := strconv.Atoi(paramHc[0]) if err == nil { historyCount = v } } statsToUse := map[string]struct{}{} if paramStats, exists := params["stats"]; exists && len(paramStats) > 0 { commaStats := strings.Split(paramStats[0], ",") for _, stat := range commaStats { statsToUse[stat] = struct{}{} } } wildcard := false if paramWildcard, exists := params["wildcard"]; exists && len(paramWildcard) > 0 { wildcard, _ = strconv.ParseBool(paramWildcard[0]) // ignore errors, error => false } dsType := enum.DSTypeInvalid if paramType, exists := params["type"]; exists && len(paramType) > 0 { dsType = enum.DSTypeFromString(paramType[0]) if dsType == enum.DSTypeInvalid { return nil, fmt.Errorf("invalid query parameter type '%v' - valid types are: {http, dns}", paramType[0]) } } deliveryServices := map[enum.DeliveryServiceName]struct{}{} // TODO rename 'hosts' to 'names' for consistency if paramNames, exists := params["deliveryservices"]; exists && len(paramNames) > 0 { commaNames := strings.Split(paramNames[0], ",") for _, name := range commaNames { deliveryServices[enum.DeliveryServiceName(name)] = struct{}{} } } // parameters without values are considered names, e.g. `?my-cache-0` or `?my-delivery-service` for maybeName, val := range params { if len(val) == 0 || (len(val) == 1 && val[0] == "") { deliveryServices[enum.DeliveryServiceName(maybeName)] = struct{}{} } } return &DSStatFilter{ historyCount: historyCount, statsToUse: statsToUse, wildcard: wildcard, dsType: dsType, deliveryServices: deliveryServices, dsTypes: dsTypes, }, nil }
// TODO timing, and determine if the case, or its internal `for`, should be put in a goroutine // TODO determine if subscribers take action on change, and change to mutexed objects if not. func monitorConfigListen(monitorConfigTS TrafficMonitorConfigMapThreadsafe, monitorConfigPollChan <-chan to.TrafficMonitorConfigMap, localStates peer.CRStatesThreadsafe, statUrlSubscriber chan<- poller.HttpPollerConfig, healthUrlSubscriber chan<- poller.HttpPollerConfig, peerUrlSubscriber chan<- poller.HttpPollerConfig, cfg config.Config, staticAppData StaticAppData) { for { select { case monitorConfig := <-monitorConfigPollChan: monitorConfigTS.Set(monitorConfig) healthUrls := map[string]string{} statUrls := map[string]string{} peerUrls := map[string]string{} caches := map[string]string{} for _, srv := range monitorConfig.TrafficServer { caches[srv.HostName] = srv.Status cacheName := enum.CacheName(srv.HostName) if srv.Status == "ONLINE" { localStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: true}) continue } if srv.Status == "OFFLINE" { localStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: false}) continue } // seed states with available = false until our polling cycle picks up a result if _, exists := localStates.Get().Caches[cacheName]; !exists { localStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: false}) } url := monitorConfig.Profile[srv.Profile].Parameters.HealthPollingURL r := strings.NewReplacer( "${hostname}", srv.FQDN, "${interface_name}", srv.InterfaceName, "application=system", "application=plugin.remap", "application=", "application=plugin.remap", ) url = r.Replace(url) healthUrls[srv.HostName] = url r = strings.NewReplacer("application=plugin.remap", "application=") url = r.Replace(url) statUrls[srv.HostName] = url } for _, srv := range monitorConfig.TrafficMonitor { if srv.HostName == staticAppData.Hostname { continue } if srv.Status != "ONLINE" { continue } // TODO: the URL should be config driven. -jse url := fmt.Sprintf("http://%s:%d/publish/CrStates?raw", srv.IP, srv.Port) peerUrls[srv.HostName] = url } statUrlSubscriber <- poller.HttpPollerConfig{Urls: statUrls, Interval: cfg.CacheStatPollingInterval} healthUrlSubscriber <- poller.HttpPollerConfig{Urls: healthUrls, Interval: cfg.CacheHealthPollingInterval} peerUrlSubscriber <- poller.HttpPollerConfig{Urls: peerUrls, Interval: cfg.PeerPollingInterval} for cacheName := range localStates.GetCaches() { if _, exists := monitorConfig.TrafficServer[string(cacheName)]; !exists { log.Warnf("Removing %s from localStates", cacheName) localStates.DeleteCache(cacheName) } } // TODO because there are multiple writers to localStates.DeliveryService, there is a race condition, where MonitorConfig (this func) and HealthResultManager could write at the same time, and the HealthResultManager could overwrite a delivery service addition or deletion here. Probably the simplest and most performant fix would be a lock-free algorithm using atomic compare-and-swaps. for _, ds := range monitorConfig.DeliveryService { // since caches default to unavailable, also default DS false if _, exists := localStates.Get().Deliveryservice[enum.DeliveryServiceName(ds.XMLID)]; !exists { localStates.SetDeliveryService(enum.DeliveryServiceName(ds.XMLID), peer.Deliveryservice{IsAvailable: false, DisabledLocations: []enum.CacheName{}}) // important to initialize DisabledLocations, so JSON is `[]` not `null` } } for ds, _ := range localStates.Get().Deliveryservice { if _, exists := monitorConfig.DeliveryService[string(ds)]; !exists { localStates.DeleteDeliveryService(ds) } } } } }