예제 #1
0
// newAWsMgr returns a new AWS mgr instance
// configPath determines where the awsAccounts config is stored in the config service in respect to your service
// For example hailo/service/foo/awsAccounts.
func newAwsMgr(configPath ...string) *AwsMgr {

	m := &AwsMgr{
		Accounts: loadAccConfig(configPath...),
		sts:      sts.NewSTSConnectionManager(),
	}

	ch := config.SubscribeChanges()
	hash, _ := config.LastLoaded()
	// Launch our config updater
	go func() {
		for _ = range ch {
			newHash, _ := config.LastLoaded()
			if hash != newHash {
				hash = newHash
				accs := loadAccConfig(configPath...)
				m.Lock()
				if len(accs) > 0 {
					m.Accounts = accs
					log.Debugf("[AWS Manager] Updating AWS Accounts:%v", m.Accounts)
				}
				m.Unlock()
			}
		}
	}()
	log.Debugf("[AWS Manager] Accounts: %v", m.Accounts)
	return m

}
예제 #2
0
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
	cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
	pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
	hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")

	// hash and test
	hash, _ := config.LastLoaded()
	if p.configHash == hash {
		return nil
	}
	p.configHash = hash

	// lock now and then update everything
	p.Lock()
	defer p.Unlock()

	canonicalHosts := make(map[string]bool)
	for _, host := range pubHosts {
		canonicalHosts[host] = true
		// do we have a producer for this host?
		if _, ok := p.producers[host]; !ok {
			cfg := nsqlib.NewConfig()
			cfg.HeartbeatInterval = hbInterval
			prod, err := nsqlib.NewProducer(host, cfg)
			if err != nil {
				return err
			}
			prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
			p.producers[host] = prod
		}
	}

	// now remove any removed ones
	for host, prod := range p.producers {
		if !canonicalHosts[host] {
			delete(p.producers, host)
			prod.Stop()
		}
	}

	// add hosts to hostpool
	p.hostpool.SetHosts(pubHosts)

	log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))

	// setup the other meta data
	p.count = len(p.producers)
	switch cl {
	case "TWO":
		p.cl = cl_TWO
		p.n = 2
	case "QUORUM":
		p.cl = cl_QUORUM
		p.n = p.count/2 + 1
	default:
		p.cl = cl_ONE // our default
		p.n = 1
	}

	p.configHash = hash

	return nil
}
예제 #3
0
func initHealthChecks() {
	// add default healthcheck (to check we have config loaded - since we've just tried to load it)
	HealthCheck("com.hailocab.kernel.configloaded", func() (map[string]string, error) {
		ret := make(map[string]string)
		h, t := config.LastLoaded()
		ret["hash"] = h
		ret["lastLoaded"] = "[never]"
		if !t.IsZero() {
			ret["lastLoaded"] = fmt.Sprintf("%v", t.Format("2006-01-02 15:04:05"))
		}
		if ret["hash"] == "" || ret["lastLoaded"] == "[never]" {
			return ret, fmt.Errorf("Config not loaded")
		}
		return ret, nil
	})

	// add default healthcheck (to check the service to service errors)
	HealthCheck("com.hailocab.kernel.servicetoservice.auth.badrole", func() (map[string]string, error) {
		ret := make(map[string]string)

		var failing []string

		// Get the error counts
		counters := errors.Get("com.hailocab.kernel.auth.badrole")

		failed := 0
		for name, count := range counters {
			if count == 0 {
				continue
			}

			failed += count

			if split := strings.Split(name, ":"); len(split) > 1 {
				name = strings.Join(split[1:], ".")
			}

			ret[name] = fmt.Sprintf("%d", count)
			failing = append(failing, fmt.Sprintf("%s: %d", name, count))
		}

		if cleared := errors.Cleared(); time.Since(cleared).Seconds() > 60 {
			// Clear the errors counts
			errors.Clear("com.hailocab.kernel.auth.badrole")
		}

		if len(failing) > 0 {
			return ret, fmt.Errorf("%d failed calls in last minute to %d services: %s", failed, len(failing), strings.Join(failing, ", "))
		}

		return ret, nil
	})

	// add default healthcheck (to check the platform capacity)
	HealthCheck("com.hailocab.kernel.resource.capacity", func() (map[string]string, error) {
		capacity := 0
		offendingCallers := []string{}

		tokensMtx.RLock()
		for caller, tokC := range tokens {
			capacity += cap(tokC)
			if len(tokC) == 0 {
				offendingCallers = append(offendingCallers, caller)
			}
		}
		tokensMtx.RUnlock()

		var err error
		if len(offendingCallers) > 0 {
			err = fmt.Errorf("Callers exceeding capacity: %s", strings.Join(offendingCallers, ", "))
		}

		return map[string]string{
			"capacity": fmt.Sprintf("%d", capacity),
			"inflight": fmt.Sprintf("%d", atomic.LoadUint64(&inFlightRequests)),
		}, err
	})

	HealthCheck("com.hailocab.kernel.client.circuit", circuitbreaker.CircuitHealthCheck)
}