Example #1
0
func getHosts() []string {
	port := config.AtPath("hailo", "service", "cassandra", "defaults", "cqlPort").AsInt(defaultPort)
	hosts := config.AtPath("hailo", "service", "cassandra", hostsCfgKey()).AsHostnameArray(port)
	if len(hosts) > 0 {
		return hosts
	}

	// No hosts returned: try DNS
	tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("premium")
	hosts, err := dns.Hosts("cassandra-" + tier)
	if err != nil {
		log.Errorf("[Cassandra] Failed to load hosts from DNS: %s", err.Error())
		return defaultHosts
	}

	if len(hosts) == 0 {
		return defaultHosts
	}
	// We need to append the port to hosts coming from DNS
	for i, host := range hosts {
		hosts[i] = host + fmt.Sprintf(":%d", port)
	}

	return hosts
}
Example #2
0
func getHosts(port int, path ...string) []string {
	if hosts := config.AtPath(path...).AsHostnameArray(port); len(hosts) > 0 {
		return hosts
	}

	// should we lookup dns?
	if config.AtPath("hailo", "service", "nsq", "disableDnsLookup").AsBool() {
		return []string{}
	}

	// try dns lookup
	cluster := config.AtPath("hailo", "service", "nsq", "cluster").AsString("general")
	hosts, err := dns.Hosts("nsq-" + cluster)
	if err != nil {
		log.Errorf("Failed to load NSQ hosts from dns: %v", err)
		return []string{}
	}

	// append port
	for i, host := range hosts {
		hosts[i] = fmt.Sprintf("%s:%d", host, port)
	}

	return hosts
}
Example #3
0
func getHosts() []string {
	cassandraHostKey := getCassandraHostConfigKey()
	config.WaitUntilLoaded(5 * time.Second)
	port := config.AtPath("hailo", "service", "cassandra", "defaults", "thriftPort").AsInt(defaultPort)
	if hosts := config.AtPath("hailo", "service", "cassandra", cassandraHostKey).AsHostnameArray(port); len(hosts) > 0 {
		return hosts
	}

	// No hosts returned: try DNS
	tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("premium")
	hosts, err := dns.Hosts("cassandra-" + tier)
	if err != nil {
		log.Errorf("Failed to load Cassandra hosts from dns: %v", err)
		return defaultHosts
	}

	if len(hosts) == 0 {
		return defaultHosts
	}
	// We need to append the port to hosts coming from DNS
	for i, host := range hosts {
		hosts[i] = host + fmt.Sprintf(":%d", port)
	}

	return hosts
}
Example #4
0
func getHosts() []string {
	hostsConfigPath := []string{"hailo", "service", "zookeeper", "hosts"}
	tier := config.AtPath("hailo", "service", "zookeeper", "tier").AsString("general")
	if tier != "general" {
		hostsConfigPath = append(hostsConfigPath, tier)
	}

	if hosts := config.AtPath(hostsConfigPath...).AsHostnameArray(2181); len(hosts) > 0 {
		return hosts
	}

	// no hosts returned so try dns
	hosts, err := dns.Hosts("zookeeper-" + tier)
	if err != nil {
		log.Errorf("Failed to load ZK hosts from dns: %v", err)
		return []string{"localhost:2181"}
	}

	// for safety fall back to localhost
	if len(hosts) == 0 {
		return []string{"localhost:2181"}
	}

	// append port
	for i, host := range hosts {
		hosts[i] = host + ":2181"
	}

	return hosts
}
Example #5
0
func getHosts() []string {
	hostConfigPath := []string{"hailo", "service", "memcache", "servers"}
	host := "memcached"

	// check if tier is specified and act accordingly
	tier := config.AtPath("hailo", "service", "memcache", "tier").AsString("")
	if tier != "" {
		hostConfigPath = append(hostConfigPath, tier)
		host = fmt.Sprintf("%s-%s", host, tier)
	}

	if hosts := config.AtPath(hostConfigPath...).AsHostnameArray(11211); len(hosts) > 0 {
		return hosts
	}

	// no hosts returned so try dns
	hosts, err := dns.Hosts(host)
	if err != nil {
		log.Errorf("[Memcache] Failed to load hosts from dns, returning empty list: %v", err)
		return []string{}
	}

	// append port
	for i, host := range hosts {
		hosts[i] = host + ":11211"
	}

	return hosts
}
Example #6
0
func newdefaultClient() MemcacheClient {
	serverSelector := new(memcache.ServerList)
	client := memcache.NewFromSelector(serverSelector)

	// Listen for config changes
	ch := config.SubscribeChanges()
	go func() {
		for _ = range ch {
			loadFromConfig(serverSelector, client)
		}
	}()

	loadFromConfig(serverSelector, client)

	// Log on init
	hosts := config.AtPath("hailo", "service", "memcache", "servers").AsHostnameArray(11211)
	operationTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
		AsDuration(defaultDialTimeout)
	dialTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
		AsDuration(defaultOperationTimeout)

	log.Infof("[Memcache] Initialising Memcache client to hosts %v: dial timeout %v, op timeout: %v", hosts,
		dialTimeout, operationTimeout)

	return client
}
func createCircuit(service, endpoint string) Circuit {
	options := defaultOptions
	config.AtPath("hailo", "platform", "circuitbreaker").AsStruct(&options)
	config.AtPath("hailo", "platform", "circuitbreaker", "endpoints", service, endpoint).AsStruct(&options)

	log.Debugf("Circuitbreaker config for %s.%s: %#v", service, endpoint, options)
	return NewDefaultCircuit(options)
}
Example #8
0
func (rs *RedisDedupeClient) connect() (*redis.Pool, error) {
	host := config.AtPath("hailo", "service", "deduper", "redis", "hostname").AsString(":16379")
	// var password string
	log.Debugf("Setting redis server from config: %v", host)
	pool := &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", host)
			if err != nil {
				return nil, err
			}
			if _, err := c.Do("PING"); err != nil {
				c.Close()
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}

	return pool, nil
}
func (s *DefaultSubscriber) AddHandlers(handler nsqlib.Handler) {
	subHandlers := config.AtPath("hailo", "service", "nsq", "subHandlers").AsInt(6)
	log.Infof("Adding %d handlers", subHandlers)
	for i := 0; i < subHandlers; i++ {
		s.AddHandler(handler)
	}
}
Example #10
0
func loadFromConfig(sl *memcache.ServerList, client *memcache.Client) {
	hosts := getHosts()
	log.Tracef("[Memcache] Setting memcache servers from config: %v", hosts)
	err := sl.SetServers(hosts...)
	if err != nil {
		log.Errorf("[Memcache] Error setting memcache servers: %v", err)
	}

	// Technically we have a race here since the timeouts are not protected by a mutex, however it isn't really a
	// problem if the timeout is stale for a short period.
	client.Timeout = config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
		AsDuration(defaultOperationTimeout)
	log.Tracef("[Memcache] Set Memcache operation timeout from config: %v", client.Timeout)
	client.DialTimeout = config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
		AsDuration(defaultDialTimeout)
	log.Tracef("[Memcache] Set Memcache dial timeout from config: %v", client.DialTimeout)
}
Example #11
0
// loadFromConfig will grab the configurable settings from config service
func (t *Timeout) loadFromConfig() {
	min := config.AtPath("hailo", "platform", "timeout", "min").AsDuration(defaultMin)
	max := config.AtPath("hailo", "platform", "timeout", "max").AsDuration(defaultMax)
	multiplier := config.AtPath("hailo", "platform", "timeout", "multiplier").AsFloat64(defaultMultiplier)

	// any difference?
	if hashTimeouts(min, max, multiplier) == t.hashTimeouts() {
		return
	}

	t.Lock()
	defer t.Unlock()
	t.min = min
	t.max = max
	t.multiplier = multiplier

	log.Infof("[Client] Loaded timeout configuration from config service [min=%v, max=%v, multiplier=%v]", min, max, multiplier)
}
// NewGlobalLocker returns a global leader which is basically just a region leader pinned to one region based on
// config.
func NewGlobalLeader(id string) Leader {
	for {
		if config.AtPath("leaders", "isLeader").AsBool() {
			break
		}
		<-config.SubscribeChanges()
	}
	return RegionLeader(id)
}
Example #13
0
func loadAccConfig(path ...string) []*AWSAccount {
	bytes := config.AtPath(path...).AsJson()
	accs := make([]*AWSAccount, 0)
	err := json.Unmarshal(bytes, &accs)
	if err != nil {
		log.Warnf("[AWS Manager] Failed to unmarshal AWS credential pairs from config: %s", err)
		return nil
	}
	return accs
}
func TestPub(t *testing.T) {
	config.LoadFromService("testservice")
	s := config.AtPath("configService", "hash").AsString("default")
	if s == "default" {
		t.Fatal("Failed to load config from config service")
	}
	err := Publish("testtopic", []byte("This is my payload"))
	if err != nil {
		t.Error(fmt.Sprintf("Failed to PUB: %v", err))
	}
}
func loadEndpointConfig() {
	log.Info("Loading ElasticSearch config")

	port := config.AtPath("hailo", "service", "elasticsearch", "port").AsInt(9200)
	hosts := config.AtPath("hailo", "service", "elasticsearch", "hosts").AsHostnameArray(port)

	if len(hosts) == 0 {
		hosts = append(hosts, "localhost:19200")
	}

	// Set these hosts in the Elasticsearch library
	// This will initialise a host pool which uses an Epsilon Greedy algorithm to find healthy hosts
	// and send to requests to them, and not unhealthy or slow hosts
	eapi.Port = strconv.Itoa(port)
	if port == 443 {
		eapi.Protocol = "https"
	}
	eapi.SetHosts(hosts)

	log.Infof("ElasticSearch hosts loaded: %v", eapi.Hosts)
}
Example #16
0
// ksAuth returns the username and password for the given keyspace
func ksAuth(ks string) (string, string, error) {
	if !config.AtPath("hailo", "service", "cassandra", "authentication", "enabled").AsBool() {
		return "", "", nil
	}

	confJson := config.AtPath("hailo", "service", "cassandra", "authentication", "keyspaces").AsJson()
	rawConf := make(map[string]map[string]string, 5)
	if err := json.Unmarshal(confJson, &rawConf); err != nil {
		// Don't log raw data in an attempt to not log our passwords
		log.Warnf("[Cassandra] Failed to unmarshal authentication configuration: %s", err.Error())
		return "", "", err
	}

	for candidateKs, v := range rawConf {
		if candidateKs == ks {
			return v["username"], v["password"], nil
		}
	}

	return "", "", nil
}
Example #17
0
func getKsConfig(ks string) (ksConfig, error) {
	if !config.WaitUntilLoaded(5 * time.Second) {
		return ksConfig{}, fmt.Errorf("Config not loaded")
	}

	username, password, err := ksAuth(ks)
	if err != nil {
		return ksConfig{}, err
	}

	c := ksConfig{
		ks:       ks,
		hosts:    getHosts(),
		username: username,
		password: password,
		retries:  config.AtPath("hailo", "service", "cassandra", "defaults", "maxRetries").AsInt(5),
		cl:       clFromString(config.AtPath("hailo", "service", "cassandra", "defaults", "consistencyLevel").AsString("")),
		timeout:  config.AtPath("hailo", "service", "cassandra", "defaults", "recvTimeout").AsDuration("1s"),
	}
	cc := gocql.NewCluster(c.hosts...)
	cc.ProtoVersion = config.AtPath("hailo", "service", "cassandra", "defaults", "protoVersion").AsInt(2)
	cc.Consistency = c.cl
	cc.Compressor = gocql.SnappyCompressor{}
	cc.DiscoverHosts = false
	cc.NumConns = config.AtPath("hailo", "service", "cassandra", "defaults", "maxHostConns").AsInt(2)
	cc.Authenticator = gocql.PasswordAuthenticator{
		Username: c.username,
		Password: c.password,
	}
	cc.Timeout = c.timeout
	cc.Keyspace = c.ks
	cc.RetryPolicy = &gocql.SimpleRetryPolicy{
		NumRetries: c.retries,
	}
	cc.PoolConfig.HostSelectionPolicy = gocql.HostPoolHostPolicy(
		hostpool.NewEpsilonGreedy(c.hosts, 5*time.Minute, &hostpool.LinearEpsilonValueCalculator{}),
	)
	c.cc = cc
	return c, nil
}
func loadStatsd(addr string) g2s.Statter {
	disabled := config.AtPath("hailo", "service", "instrumentation", "statsd", "disabled").AsBool()
	if disabled {
		return g2s.Noop()
	}

	s, err := g2s.Dial("udp", addr)
	if err != nil {
		log.Warnf("Error initialising statsd connection to %v", addr)
		return nil
	}

	return s
}
Example #19
0
// hostsCfgKey gets the config key that should be used to load cassandra hosts. This is to support multiple 'tiered'
// Cassandra clusters.
func hostsCfgKey() string {
	// Check what cluster are we supposed to contact, revert to the default config if not specified
	tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("")
	if tier == "" {
		tier = defaultTier
	}

	switch tier {
	case "general":
		return "hosts"
	default:
		return fmt.Sprintf("%sHosts", tier)
	}
}
Example #20
0
// run is our main healthcheck loop
func (r *results) run() {
	for {
		select {
		// Listen for config changes and update the healthcheck when needed
		case <-config.SubscribeChanges():
			// Allow healthcheck parameters to be overridden in config
			config.AtPath("hailo", "platform", "healthcheck", r.hc.Id).AsStruct(r.hc)
		case <-runNow:
			r.collect()
		case <-time.After(r.hc.Interval):
			r.collect()
		}
	}
}
Example #21
0
// PriorityHealthCheck is a healthcheck with a configurable priority for this server
func PriorityHealthCheck(id string, checker slhc.Checker, priority slhc.Priority) {
	hc := &healthcheck.HealthCheck{
		Id:             id,
		ServiceName:    Name,
		ServiceVersion: Version,
		Hostname:       hostname,
		InstanceId:     InstanceID,
		Interval:       healthcheck.StandardInterval,
		Checker:        checker,
		Priority:       priority,
	}

	// Allow healthcheck parameters to be overridden in config
	config.AtPath("hailo", "platform", "healthcheck", id).AsStruct(hc)

	healthcheck.Register(hc)
}
Example #22
0
// getCassandraHostConfigPath gets the config key that should be used to load cassandra hosts
// this is to support multiple 'tiered' cassandra clusters
func getCassandraHostConfigKey() string {

	// Check what cluster are we supposed to contact, revert to the default config if not specified
	tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString(defaultTier)
	if tier == "" {
		tier = defaultTier
	}
	log.Debugf("Attempting to connect to the %v Cassandra cluster", tier)

	var cassandraHosts string
	switch tier {
	case "general":
		cassandraHosts = "hosts"
	default:
		cassandraHosts = fmt.Sprintf("%sHosts", tier)
	}

	return cassandraHosts
}
Example #23
0
// shouldTrace determiens if we should trace this request, when sending
func (r *Request) shouldTrace() bool {
	if r.traceID != "" {
		return true
	}

	pcChance := config.AtPath("hailo", "service", "trace", "pcChance").AsFloat64(0)
	if pcChance <= 0 {
		return false
	}

	if rand.Float64() < pcChance {
		u4, err := uuid.NewV4()
		if err != nil {
			return false
		}

		r.SetTraceID(u4.String())
		return true
	}

	return false
}
Example #24
0
func reconnectDefault() {
	hosts := getHosts()
	recvTimeout := config.AtPath("hailo", "service", "zookeeper", "recvTimeout").AsDuration("100ms")
	if !hasConfigChanged(hosts, recvTimeout) {
		log.Infof("ZooKeeper config has not changed")
		return
	}

	if defaultClient != nil {
		if recvTimeout != defaultTimeout {
			// cannot gracefully set timeout so close it
			defaultTimeout = recvTimeout
			defaultClient.Close()
		} else {
			// update the hosts only
			log.Tracef("Setting ZK hosts to %v", hosts)
			defaultClient.UpdateAddrs(hosts)
			return
		}
	}

	connectDefault(hosts, recvTimeout)
}
Example #25
0
// load will load public key location from config service and switch pub key
func (v *validatorImpl) load() error {
	v.Lock()
	defer v.Unlock()

	fn := config.AtPath("hailo", "service", "authentication", "publicKey").AsString("")
	log.Tracef("[Auth] Loading auth library public key from: %s", fn)
	if fn == "" {
		return fmt.Errorf("public key filename undefined in config")
	}

	// load key from file
	f, err := os.Open(fn)
	if err != nil {
		return fmt.Errorf("Failed to open public key %s (%v)", fn, err)
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
	}

	if bytes.Equal(b, v.lastRead) {
		// no change
		return nil
	}

	// turn bytes into an actual key instance
	k, err := bytesToKey(b)
	if err != nil {
		return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
	}

	log.Infof("[Auth] Loaded public key: %v", k)

	v.pub = k
	v.lastRead = b
	return nil
}
Example #26
0
// loadConfig gets host, scheme, port and timeouts from config service
func (conn *Connection) loadConfig() {
	// reload host/scheme/port and see if differnet
	host := config.AtPath("hailo", "service", "graphite", "host").AsString("graphite-internal-test.elasticride.com")
	scheme := config.AtPath("hailo", "service", "graphite", "scheme").AsString("http")
	port := config.AtPath("hailo", "service", "graphite", "port").AsInt(-1)
	connTimeout := config.AtPath("hailo", "service", "graphite", "connectTimeout").AsDuration("300ms")
	reqTimeout := config.AtPath("hailo", "service", "graphite", "requestTimeout").AsDuration("2s")
	rspHdrTimeout := config.AtPath("hailo", "service", "graphite", "responseHeaderTimeout").AsDuration("1s")

	// hash all this and compare
	h := md5.New()
	io.WriteString(h, host)
	io.WriteString(h, scheme)
	io.WriteString(h, fmt.Sprintf("%v", port))
	io.WriteString(h, fmt.Sprintf("%v", connTimeout))
	io.WriteString(h, fmt.Sprintf("%v", reqTimeout))
	io.WriteString(h, fmt.Sprintf("%v", rspHdrTimeout))
	reloadHash := fmt.Sprintf("%x", h.Sum(nil))

	if conn.currentHash() == reloadHash {
		// no change
		return
	}

	conn.Lock()
	defer conn.Unlock()

	conn.host, conn.scheme, conn.port = host, scheme, port
	conn.connTimeout, conn.reqTimeout, conn.rspHdrTimeout = connTimeout, reqTimeout, rspHdrTimeout

	// init transport
	if conn.transport != nil {
		conn.transport.Close()
	}
	conn.transport = conn.newTimeoutTransport()
	conn.client = &http.Client{Transport: conn.transport}
}
Example #27
0
func reconnectDefault() {
	mtx.Lock()
	defer mtx.Unlock()

	log.Infof("Reloading cassandra configuration")

	retries = config.AtPath("hailo", "service", "cassandra", "defaults", "maxRetries").AsInt(5)
	readCl = config.AtPath("hailo", "service", "cassandra", "defaults", "readConsistencyLevel").AsString("ONE")
	writeCl = config.AtPath("hailo", "service", "cassandra", "defaults", "writeConsistencyLevel").AsString("ONE")
	timeout = config.AtPath("hailo", "service", "cassandra", "defaults", "recvTimeout").AsDuration("1s")

	log.Debugf("Setting Cassandra defaults retries:%v, readCl: %v, writeCl: %v, timeout: %v from config", retries, readCl, writeCl, timeout)

	nodes = getHosts()
	log.Debugf("Setting Cassandra nodes %v from config", nodes)

	// Set up authentication if enabled
	if authEnabled := config.AtPath("hailo", "service", "cassandra", "authentication", "enabled").AsBool(); authEnabled {

		// Get config as json as its effectively a map[string]map[string]string
		authconfig := config.AtPath("hailo", "service", "cassandra", "authentication", "keyspaces").AsJson()

		// Parse and set if successful
		a, err := parseAuth(authconfig)
		if err == nil {
			auth = a
			log.Debugf("Setting Cassandra authentication from config: %v", a)
		} else {
			log.Warnf("Failed to set Cassandra authentication from config: %v", err)
		}

	}

	// Reset the pools map
	pools = make(map[string]gossie.ConnectionPool)
}
Example #28
0
func (s *DefaultSubscriber) doLoad() error {
	subHosts := config.AtPath("hailo", "service", "nsq", "subHosts").AsHostnameArray(4150)
	disableLookupd := config.AtPath("hailo", "service", "nsq", "disableLookupd").AsBool()
	lookupdHosts := getHosts(4161, "hailo", "service", "nsq", "nsqlookupdSeeds")

	h := md5.New()
	io.WriteString(h, strings.Join(subHosts, ","))
	io.WriteString(h, fmt.Sprintf("%v", disableLookupd))
	io.WriteString(h, strings.Join(lookupdHosts, ","))
	hash := fmt.Sprintf("%x", h.Sum(nil))

	if s.configHash == hash {
		return nil // don't bother as nothing interesting has changed
	}

	if disableLookupd {
		log.Infof("Connecting to NSQ directly: %v", subHosts)
		var hostList []string
		for _, addr := range subHosts {
			// support comma separated host lists too
			hostList = append(hostList, strings.Split(addr, ",")...)
		}
		err := s.consumer.ConnectToNSQDs(hostList)
		if err != nil && err != nsqlib.ErrAlreadyConnected {
			return fmt.Errorf("Error connecting to nsqd(s): %v", err)
		}
	} else {
		if len(lookupdHosts) > 0 {
			log.Infof("Connecting to NSQ via lookupd hosts: %v", lookupdHosts)
			err := s.consumer.ConnectToNSQLookupds(lookupdHosts)
			if err != nil {
				return fmt.Errorf("Error connecting to nsqlookupd(s): %v", err)
			}
		}
	}

	// Disconnect from old hosts
	if hosts := diffHosts(s.subHosts, subHosts); len(hosts) > 0 {
		log.Infof("Disconnecting from NSQ hosts: %v", hosts)
		for _, host := range hosts {
			err := s.consumer.DisconnectFromNSQD(host)
			if err != nil && err != nsqlib.ErrNotConnected {
				log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
			}
		}
	}

	// Disconnect from old lookupds
	if hosts := diffHosts(s.lookupdHosts, lookupdHosts); len(hosts) > 0 {
		log.Infof("Disconnecting from NSQ lookupds: %v", hosts)
		for _, host := range hosts {
			err := s.consumer.DisconnectFromNSQLookupd(host)
			if err != nil && err != nsqlib.ErrNotConnected {
				log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
			}
		}
	}

	// save state on success
	s.configHash = hash
	s.subHosts = subHosts
	s.lookupdHosts = lookupdHosts

	return nil
}
Example #29
0
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
	cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
	pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
	hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")

	// hash and test
	hash, _ := config.LastLoaded()
	if p.configHash == hash {
		return nil
	}
	p.configHash = hash

	// lock now and then update everything
	p.Lock()
	defer p.Unlock()

	canonicalHosts := make(map[string]bool)
	for _, host := range pubHosts {
		canonicalHosts[host] = true
		// do we have a producer for this host?
		if _, ok := p.producers[host]; !ok {
			cfg := nsqlib.NewConfig()
			cfg.HeartbeatInterval = hbInterval
			prod, err := nsqlib.NewProducer(host, cfg)
			if err != nil {
				return err
			}
			prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
			p.producers[host] = prod
		}
	}

	// now remove any removed ones
	for host, prod := range p.producers {
		if !canonicalHosts[host] {
			delete(p.producers, host)
			prod.Stop()
		}
	}

	// add hosts to hostpool
	p.hostpool.SetHosts(pubHosts)

	log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))

	// setup the other meta data
	p.count = len(p.producers)
	switch cl {
	case "TWO":
		p.cl = cl_TWO
		p.n = 2
	case "QUORUM":
		p.cl = cl_QUORUM
		p.n = p.count/2 + 1
	default:
		p.cl = cl_ONE // our default
		p.n = 1
	}

	p.configHash = hash

	return nil
}
// MaxConnHealthCheck asserts that the total number of established connections to all zookeeper nodes
// falls below a given max threshold.
func MaxConnHealthCheck(maxconns int) healthcheck.Checker {
	return func() (map[string]string, error) {
		nodes := config.AtPath("hailo", "service", "zookeeper", "hosts").AsHostnameArray(2181)
		return connhealthcheck.MaxTcpConnections(nodes, maxconns)()
	}
}