func (p MonitorConfigPoller) Poll() {
tick := time.NewTicker(p.Interval)
for {
select {
case opsConfig := <-p.OpsConfigChannel:
log.Infof("MonitorConfigPoller: received new opsConfig: %v\n", opsConfig)
p.OpsConfig = opsConfig
case session := <-p.SessionChannel:
log.Infof("MonitorConfigPoller: received new session: %v\n", session)
p.Session = session
case <-tick.C:
if p.Session != nil && p.OpsConfig.CdnName != "" {
monitorConfig, err := p.Session.TrafficMonitorConfigMap(p.OpsConfig.CdnName)
if err != nil {
log.Errorf("MonitorConfigPoller: %s\n %v\n", err, monitorConfig)
} else {
log.Infoln("MonitorConfigPoller: fetched monitorConfig")
p.ConfigChannel <- *monitorConfig
}
} else {
log.Warnln("MonitorConfigPoller: skipping this iteration, Session is nil")
}
}
}
}
func CreateStats(statHistory map[enum.CacheName][]cache.Result, toData todata.TOData, crStates peer.Crstates, lastStats LastStats, now time.Time) (Stats, LastStats, error) {
start := time.Now()
dsStats := NewStats()
for deliveryService, _ := range toData.DeliveryServiceServers {
if deliveryService == "" {
log.Errorf("EMPTY CreateStats deliveryService")
continue
}
dsStats.DeliveryService[enum.DeliveryServiceName(deliveryService)] = *dsdata.NewStat()
}
dsStats = setStaticData(dsStats, toData.DeliveryServiceServers)
var err error
dsStats, err = addAvailableData(dsStats, crStates, toData.ServerCachegroups, toData.ServerDeliveryServices, toData.ServerTypes, statHistory) // TODO move after stat summarisation
if err != nil {
return dsStats, lastStats, fmt.Errorf("Error getting Cache availability data: %v", err)
}
for server, history := range statHistory {
if len(history) < 1 {
continue // TODO warn?
}
cachegroup, ok := toData.ServerCachegroups[server]
if !ok {
log.Warnf("server %s has no cachegroup, skipping\n", server)
continue
}
serverType, ok := toData.ServerTypes[enum.CacheName(server)]
if !ok {
log.Warnf("server %s not in CRConfig, skipping\n", server)
continue
}
result := history[len(history)-1]
// TODO check result.PrecomputedData.Errors
for ds, resultStat := range result.PrecomputedData.DeliveryServiceStats {
if ds == "" {
log.Errorf("EMPTY precomputed delivery service")
continue
}
if _, ok := dsStats.DeliveryService[ds]; !ok {
dsStats.DeliveryService[ds] = resultStat
continue
}
httpDsStat := dsStats.DeliveryService[ds]
httpDsStat.TotalStats = httpDsStat.TotalStats.Sum(resultStat.TotalStats)
httpDsStat.CacheGroups[cachegroup] = httpDsStat.CacheGroups[cachegroup].Sum(resultStat.CacheGroups[cachegroup])
httpDsStat.Types[serverType] = httpDsStat.Types[serverType].Sum(resultStat.Types[serverType])
httpDsStat.Caches[server] = httpDsStat.Caches[server].Sum(resultStat.Caches[server])
httpDsStat.CachesTimeReceived[server] = resultStat.CachesTimeReceived[server]
httpDsStat.CommonStats = dsStats.DeliveryService[ds].CommonStats
dsStats.DeliveryService[ds] = httpDsStat // TODO determine if necessary
}
}
perSecStats, lastStats := addPerSecStats(statHistory, dsStats, lastStats, now, toData.ServerCachegroups, toData.ServerTypes)
log.Infof("CreateStats took %v\n", time.Since(start))
return perSecStats, lastStats, nil
}
// Run runs a new HTTP service at the given addr, making data requests to the given c.
// Run may be called repeatedly, and each time, will shut down any existing service first.
// Run is NOT threadsafe, and MUST NOT be called concurrently by multiple goroutines.
func (s Server) Run(f GetDataFunc, addr string) error {
// TODO make an object, which itself is not threadsafe, but which encapsulates all data so multiple
// objects can be created and Run.
if s.stoppableListener != nil {
log.Infof("Stopping Web Server\n")
s.stoppableListener.Stop()
s.stoppableListenerWaitGroup.Wait()
}
log.Infof("Starting Web Server\n")
var err error
var originalListener net.Listener
if originalListener, err = net.Listen("tcp", addr); err != nil {
return err
}
if s.stoppableListener, err = stoppableListener.New(originalListener); err != nil {
return err
}
s.getData = f
sm := http.NewServeMux()
err = s.registerEndpoints(sm)
if err != nil {
return err
}
server := &http.Server{
Addr: addr,
Handler: sm,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
s.stoppableListenerWaitGroup = sync.WaitGroup{}
s.stoppableListenerWaitGroup.Add(1)
go func() {
defer s.stoppableListenerWaitGroup.Done()
server.Serve(s.stoppableListener)
}()
log.Infof("Web server listening on %s", addr)
return nil
}
// SetPolled sets cache which have been polled. This is used to determine when the app has fully started up, and we can start serving. Serving Traffic Router with caches as 'down' which simply haven't been polled yet would be bad. Therefore, a cache is set as 'polled' if it has received different bandwidths from two different ATS ticks, OR if the cache is marked as down (and thus we won't get a bandwidth).
// This is threadsafe for one writer, along with `Set`.
// This is fast if there are no unpolled caches. Moreover, its speed is a function of the number of unpolled caches, not the number of caches total.
func (t *UnpolledCachesThreadsafe) SetPolled(results []cache.Result, lastStatsThreadsafe LastStatsThreadsafe) {
unpolledCaches := copyCaches(t.UnpolledCaches())
numUnpolledCaches := len(unpolledCaches)
if numUnpolledCaches == 0 {
return
}
lastStats := lastStatsThreadsafe.Get()
for cache, _ := range unpolledCaches {
innerLoop:
for _, result := range results {
if result.Id != cache {
continue
}
if !result.Available || len(result.Errors) > 0 {
log.Infof("polled %v\n", cache)
delete(unpolledCaches, cache)
break innerLoop
}
}
lastStat, ok := lastStats.Caches[cache]
if !ok {
continue
}
if lastStat.Bytes.PerSec != 0 {
log.Infof("polled %v\n", cache)
delete(unpolledCaches, cache)
}
}
if len(unpolledCaches) == numUnpolledCaches {
return
}
t.setUnpolledCaches(unpolledCaches)
if len(unpolledCaches) != 0 {
log.Infof("remaining unpolled %v\n", unpolledCaches)
} else {
log.Infof("all caches polled, ready to serve!\n")
}
}
// processHealthResult processes the given health results, adding their stats to the CacheAvailableStatus. Note this is NOT threadsafe, because it non-atomically gets CacheAvailableStatuses, Events, LastHealthDurations and later updates them. This MUST NOT be called from multiple threads.
func processHealthResult(cacheHealthChan <-chan cache.Result, toData todata.TODataThreadsafe, localStates peer.CRStatesThreadsafe, lastHealthDurationsThreadsafe DurationMapThreadsafe, statHistory StatHistoryThreadsafe, monitorConfig TrafficMonitorConfigMapThreadsafe, peerStates peer.CRStatesPeersThreadsafe, combinedStates peer.CRStatesThreadsafe, fetchCount UintThreadsafe, errorCount UintThreadsafe, events EventsThreadsafe, localCacheStatusThreadsafe CacheAvailableStatusThreadsafe, lastHealthEndTimes map[enum.CacheName]time.Time, healthHistory map[enum.CacheName][]cache.Result, results []cache.Result, cfg config.Config) {
if len(results) == 0 {
return
}
toDataCopy := toData.Get() // create a copy, so the same data used for all processing of this cache health result
localCacheStatus := localCacheStatusThreadsafe.Get().Copy()
monitorConfigCopy := monitorConfig.Get()
for _, healthResult := range results {
log.Debugf("poll %v %v healthresultman start\n", healthResult.PollID, time.Now())
fetchCount.Inc()
var prevResult cache.Result
healthResultHistory := healthHistory[enum.CacheName(healthResult.Id)]
// healthResultHistory := healthHistory.Get(enum.CacheName(healthResult.Id))
if len(healthResultHistory) != 0 {
prevResult = healthResultHistory[len(healthResultHistory)-1]
}
health.GetVitals(&healthResult, &prevResult, &monitorConfigCopy)
// healthHistory.Set(enum.CacheName(healthResult.Id), pruneHistory(append(healthHistory.Get(enum.CacheName(healthResult.Id)), healthResult), defaultMaxHistory))
healthHistory[enum.CacheName(healthResult.Id)] = pruneHistory(append(healthHistory[enum.CacheName(healthResult.Id)], healthResult), cfg.MaxHealthHistory)
isAvailable, whyAvailable := health.EvalCache(healthResult, &monitorConfigCopy)
if localStates.Get().Caches[healthResult.Id].IsAvailable != isAvailable {
log.Infof("Changing state for %s was: %t now: %t because %s errors: %v", healthResult.Id, prevResult.Available, isAvailable, whyAvailable, healthResult.Errors)
events.Add(Event{Time: time.Now().Unix(), Description: whyAvailable, Name: healthResult.Id, Hostname: healthResult.Id, Type: toDataCopy.ServerTypes[healthResult.Id].String(), Available: isAvailable})
}
localCacheStatus[healthResult.Id] = CacheAvailableStatus{Available: isAvailable, Status: monitorConfigCopy.TrafficServer[string(healthResult.Id)].Status} // TODO move within localStates?
localStates.SetCache(healthResult.Id, peer.IsAvailable{IsAvailable: isAvailable})
log.Debugf("poll %v %v calculateDeliveryServiceState start\n", healthResult.PollID, time.Now())
calculateDeliveryServiceState(toDataCopy.DeliveryServiceServers, localStates)
log.Debugf("poll %v %v calculateDeliveryServiceState end\n", healthResult.PollID, time.Now())
}
localCacheStatusThreadsafe.Set(localCacheStatus)
// TODO determine if we should combineCrStates() here
lastHealthDurations := lastHealthDurationsThreadsafe.Get().Copy()
for _, healthResult := range results {
if lastHealthStart, ok := lastHealthEndTimes[enum.CacheName(healthResult.Id)]; ok {
d := time.Since(lastHealthStart)
lastHealthDurations[enum.CacheName(healthResult.Id)] = d
}
lastHealthEndTimes[enum.CacheName(healthResult.Id)] = time.Now()
log.Debugf("poll %v %v finish\n", healthResult.PollID, time.Now())
healthResult.PollFinished <- healthResult.PollID
}
lastHealthDurationsThreadsafe.Set(lastHealthDurations)
}
// TODO iterationCount and/or p.TickChan?
func pollHttp(interval time.Duration, id string, url string, fetcher fetcher.Fetcher, die <-chan struct{}) {
tick := time.NewTicker(interval)
lastTime := time.Now()
for {
select {
case now := <-tick.C:
realInterval := now.Sub(lastTime)
if realInterval > interval+(time.Millisecond*100) {
instr.TimerFail.Inc()
log.Infof("Intended Duration: %v Actual Duration: %v\n", interval, realInterval)
}
lastTime = time.Now()
pollId := atomic.AddUint64(&debugPollNum, 1)
pollFinishedChan := make(chan uint64)
log.Debugf("poll %v %v start\n", pollId, time.Now())
go fetcher.Fetch(id, url, pollId, pollFinishedChan) // TODO persist fetcher, with its own die chan?
<-pollFinishedChan
case <-die:
return
}
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
staticData, err := getStaticAppData()
if err != nil {
fmt.Printf("Error starting service: failed to get static app data: %v\n", err)
os.Exit(1)
}
opsConfigFile := flag.String("opsCfg", "", "The traffic ops config file")
configFileName := flag.String("config", "", "The Traffic Monitor config file path")
flag.Parse()
if *opsConfigFile == "" {
fmt.Println("Error starting service: The --opsCfg argument is required")
os.Exit(1)
}
// TODO add hot reloading (like opsConfigFile)?
cfg, err := config.Load(*configFileName)
if err != nil {
fmt.Printf("Error starting service: failed to load config: %v\n", err)
os.Exit(1)
}
errW, warnW, infoW, debugW, err := getLogWriters(cfg.LogLocationError, cfg.LogLocationWarning, cfg.LogLocationInfo, cfg.LogLocationDebug)
if err != nil {
fmt.Printf("Error starting service: failed to create log writers: %v\n", err)
os.Exit(1)
}
log.Init(errW, warnW, infoW, debugW)
log.Infof("Starting with config %+v\n", cfg)
manager.Start(*opsConfigFile, cfg, staticData)
}