// newAWsMgr returns a new AWS mgr instance
// configPath determines where the awsAccounts config is stored in the config service in respect to your service
// For example hailo/service/foo/awsAccounts.
func newAwsMgr(configPath ...string) *AwsMgr {
m := &AwsMgr{
Accounts: loadAccConfig(configPath...),
sts: sts.NewSTSConnectionManager(),
}
ch := config.SubscribeChanges()
hash, _ := config.LastLoaded()
// Launch our config updater
go func() {
for _ = range ch {
newHash, _ := config.LastLoaded()
if hash != newHash {
hash = newHash
accs := loadAccConfig(configPath...)
m.Lock()
if len(accs) > 0 {
m.Accounts = accs
log.Debugf("[AWS Manager] Updating AWS Accounts:%v", m.Accounts)
}
m.Unlock()
}
}
}()
log.Debugf("[AWS Manager] Accounts: %v", m.Accounts)
return m
}
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")
// hash and test
hash, _ := config.LastLoaded()
if p.configHash == hash {
return nil
}
p.configHash = hash
// lock now and then update everything
p.Lock()
defer p.Unlock()
canonicalHosts := make(map[string]bool)
for _, host := range pubHosts {
canonicalHosts[host] = true
// do we have a producer for this host?
if _, ok := p.producers[host]; !ok {
cfg := nsqlib.NewConfig()
cfg.HeartbeatInterval = hbInterval
prod, err := nsqlib.NewProducer(host, cfg)
if err != nil {
return err
}
prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
p.producers[host] = prod
}
}
// now remove any removed ones
for host, prod := range p.producers {
if !canonicalHosts[host] {
delete(p.producers, host)
prod.Stop()
}
}
// add hosts to hostpool
p.hostpool.SetHosts(pubHosts)
log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))
// setup the other meta data
p.count = len(p.producers)
switch cl {
case "TWO":
p.cl = cl_TWO
p.n = 2
case "QUORUM":
p.cl = cl_QUORUM
p.n = p.count/2 + 1
default:
p.cl = cl_ONE // our default
p.n = 1
}
p.configHash = hash
return nil
}
func initHealthChecks() {
// add default healthcheck (to check we have config loaded - since we've just tried to load it)
HealthCheck("com.hailocab.kernel.configloaded", func() (map[string]string, error) {
ret := make(map[string]string)
h, t := config.LastLoaded()
ret["hash"] = h
ret["lastLoaded"] = "[never]"
if !t.IsZero() {
ret["lastLoaded"] = fmt.Sprintf("%v", t.Format("2006-01-02 15:04:05"))
}
if ret["hash"] == "" || ret["lastLoaded"] == "[never]" {
return ret, fmt.Errorf("Config not loaded")
}
return ret, nil
})
// add default healthcheck (to check the service to service errors)
HealthCheck("com.hailocab.kernel.servicetoservice.auth.badrole", func() (map[string]string, error) {
ret := make(map[string]string)
var failing []string
// Get the error counts
counters := errors.Get("com.hailocab.kernel.auth.badrole")
failed := 0
for name, count := range counters {
if count == 0 {
continue
}
failed += count
if split := strings.Split(name, ":"); len(split) > 1 {
name = strings.Join(split[1:], ".")
}
ret[name] = fmt.Sprintf("%d", count)
failing = append(failing, fmt.Sprintf("%s: %d", name, count))
}
if cleared := errors.Cleared(); time.Since(cleared).Seconds() > 60 {
// Clear the errors counts
errors.Clear("com.hailocab.kernel.auth.badrole")
}
if len(failing) > 0 {
return ret, fmt.Errorf("%d failed calls in last minute to %d services: %s", failed, len(failing), strings.Join(failing, ", "))
}
return ret, nil
})
// add default healthcheck (to check the platform capacity)
HealthCheck("com.hailocab.kernel.resource.capacity", func() (map[string]string, error) {
capacity := 0
offendingCallers := []string{}
tokensMtx.RLock()
for caller, tokC := range tokens {
capacity += cap(tokC)
if len(tokC) == 0 {
offendingCallers = append(offendingCallers, caller)
}
}
tokensMtx.RUnlock()
var err error
if len(offendingCallers) > 0 {
err = fmt.Errorf("Callers exceeding capacity: %s", strings.Join(offendingCallers, ", "))
}
return map[string]string{
"capacity": fmt.Sprintf("%d", capacity),
"inflight": fmt.Sprintf("%d", atomic.LoadUint64(&inFlightRequests)),
}, err
})
HealthCheck("com.hailocab.kernel.client.circuit", circuitbreaker.CircuitHealthCheck)
}
