func getHosts() []string {
port := config.AtPath("hailo", "service", "cassandra", "defaults", "cqlPort").AsInt(defaultPort)
hosts := config.AtPath("hailo", "service", "cassandra", hostsCfgKey()).AsHostnameArray(port)
if len(hosts) > 0 {
return hosts
}
// No hosts returned: try DNS
tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("premium")
hosts, err := dns.Hosts("cassandra-" + tier)
if err != nil {
log.Errorf("[Cassandra] Failed to load hosts from DNS: %s", err.Error())
return defaultHosts
}
if len(hosts) == 0 {
return defaultHosts
}
// We need to append the port to hosts coming from DNS
for i, host := range hosts {
hosts[i] = host + fmt.Sprintf(":%d", port)
}
return hosts
}
func getHosts(port int, path ...string) []string {
if hosts := config.AtPath(path...).AsHostnameArray(port); len(hosts) > 0 {
return hosts
}
// should we lookup dns?
if config.AtPath("hailo", "service", "nsq", "disableDnsLookup").AsBool() {
return []string{}
}
// try dns lookup
cluster := config.AtPath("hailo", "service", "nsq", "cluster").AsString("general")
hosts, err := dns.Hosts("nsq-" + cluster)
if err != nil {
log.Errorf("Failed to load NSQ hosts from dns: %v", err)
return []string{}
}
// append port
for i, host := range hosts {
hosts[i] = fmt.Sprintf("%s:%d", host, port)
}
return hosts
}
func getHosts() []string {
cassandraHostKey := getCassandraHostConfigKey()
config.WaitUntilLoaded(5 * time.Second)
port := config.AtPath("hailo", "service", "cassandra", "defaults", "thriftPort").AsInt(defaultPort)
if hosts := config.AtPath("hailo", "service", "cassandra", cassandraHostKey).AsHostnameArray(port); len(hosts) > 0 {
return hosts
}
// No hosts returned: try DNS
tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("premium")
hosts, err := dns.Hosts("cassandra-" + tier)
if err != nil {
log.Errorf("Failed to load Cassandra hosts from dns: %v", err)
return defaultHosts
}
if len(hosts) == 0 {
return defaultHosts
}
// We need to append the port to hosts coming from DNS
for i, host := range hosts {
hosts[i] = host + fmt.Sprintf(":%d", port)
}
return hosts
}
func getHosts() []string {
hostsConfigPath := []string{"hailo", "service", "zookeeper", "hosts"}
tier := config.AtPath("hailo", "service", "zookeeper", "tier").AsString("general")
if tier != "general" {
hostsConfigPath = append(hostsConfigPath, tier)
}
if hosts := config.AtPath(hostsConfigPath...).AsHostnameArray(2181); len(hosts) > 0 {
return hosts
}
// no hosts returned so try dns
hosts, err := dns.Hosts("zookeeper-" + tier)
if err != nil {
log.Errorf("Failed to load ZK hosts from dns: %v", err)
return []string{"localhost:2181"}
}
// for safety fall back to localhost
if len(hosts) == 0 {
return []string{"localhost:2181"}
}
// append port
for i, host := range hosts {
hosts[i] = host + ":2181"
}
return hosts
}
func getHosts() []string {
hostConfigPath := []string{"hailo", "service", "memcache", "servers"}
host := "memcached"
// check if tier is specified and act accordingly
tier := config.AtPath("hailo", "service", "memcache", "tier").AsString("")
if tier != "" {
hostConfigPath = append(hostConfigPath, tier)
host = fmt.Sprintf("%s-%s", host, tier)
}
if hosts := config.AtPath(hostConfigPath...).AsHostnameArray(11211); len(hosts) > 0 {
return hosts
}
// no hosts returned so try dns
hosts, err := dns.Hosts(host)
if err != nil {
log.Errorf("[Memcache] Failed to load hosts from dns, returning empty list: %v", err)
return []string{}
}
// append port
for i, host := range hosts {
hosts[i] = host + ":11211"
}
return hosts
}
func newdefaultClient() MemcacheClient {
serverSelector := new(memcache.ServerList)
client := memcache.NewFromSelector(serverSelector)
// Listen for config changes
ch := config.SubscribeChanges()
go func() {
for _ = range ch {
loadFromConfig(serverSelector, client)
}
}()
loadFromConfig(serverSelector, client)
// Log on init
hosts := config.AtPath("hailo", "service", "memcache", "servers").AsHostnameArray(11211)
operationTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
AsDuration(defaultDialTimeout)
dialTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
AsDuration(defaultOperationTimeout)
log.Infof("[Memcache] Initialising Memcache client to hosts %v: dial timeout %v, op timeout: %v", hosts,
dialTimeout, operationTimeout)
return client
}
func createCircuit(service, endpoint string) Circuit {
options := defaultOptions
config.AtPath("hailo", "platform", "circuitbreaker").AsStruct(&options)
config.AtPath("hailo", "platform", "circuitbreaker", "endpoints", service, endpoint).AsStruct(&options)
log.Debugf("Circuitbreaker config for %s.%s: %#v", service, endpoint, options)
return NewDefaultCircuit(options)
}
func (rs *RedisDedupeClient) connect() (*redis.Pool, error) {
host := config.AtPath("hailo", "service", "deduper", "redis", "hostname").AsString(":16379")
// var password string
log.Debugf("Setting redis server from config: %v", host)
pool := &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", host)
if err != nil {
return nil, err
}
if _, err := c.Do("PING"); err != nil {
c.Close()
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return pool, nil
}
func (s *DefaultSubscriber) AddHandlers(handler nsqlib.Handler) {
subHandlers := config.AtPath("hailo", "service", "nsq", "subHandlers").AsInt(6)
log.Infof("Adding %d handlers", subHandlers)
for i := 0; i < subHandlers; i++ {
s.AddHandler(handler)
}
}
func loadFromConfig(sl *memcache.ServerList, client *memcache.Client) {
hosts := getHosts()
log.Tracef("[Memcache] Setting memcache servers from config: %v", hosts)
err := sl.SetServers(hosts...)
if err != nil {
log.Errorf("[Memcache] Error setting memcache servers: %v", err)
}
// Technically we have a race here since the timeouts are not protected by a mutex, however it isn't really a
// problem if the timeout is stale for a short period.
client.Timeout = config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
AsDuration(defaultOperationTimeout)
log.Tracef("[Memcache] Set Memcache operation timeout from config: %v", client.Timeout)
client.DialTimeout = config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
AsDuration(defaultDialTimeout)
log.Tracef("[Memcache] Set Memcache dial timeout from config: %v", client.DialTimeout)
}
// loadFromConfig will grab the configurable settings from config service
func (t *Timeout) loadFromConfig() {
min := config.AtPath("hailo", "platform", "timeout", "min").AsDuration(defaultMin)
max := config.AtPath("hailo", "platform", "timeout", "max").AsDuration(defaultMax)
multiplier := config.AtPath("hailo", "platform", "timeout", "multiplier").AsFloat64(defaultMultiplier)
// any difference?
if hashTimeouts(min, max, multiplier) == t.hashTimeouts() {
return
}
t.Lock()
defer t.Unlock()
t.min = min
t.max = max
t.multiplier = multiplier
log.Infof("[Client] Loaded timeout configuration from config service [min=%v, max=%v, multiplier=%v]", min, max, multiplier)
}
// NewGlobalLocker returns a global leader which is basically just a region leader pinned to one region based on
// config.
func NewGlobalLeader(id string) Leader {
for {
if config.AtPath("leaders", "isLeader").AsBool() {
break
}
<-config.SubscribeChanges()
}
return RegionLeader(id)
}
func loadAccConfig(path ...string) []*AWSAccount {
bytes := config.AtPath(path...).AsJson()
accs := make([]*AWSAccount, 0)
err := json.Unmarshal(bytes, &accs)
if err != nil {
log.Warnf("[AWS Manager] Failed to unmarshal AWS credential pairs from config: %s", err)
return nil
}
return accs
}
func TestPub(t *testing.T) {
config.LoadFromService("testservice")
s := config.AtPath("configService", "hash").AsString("default")
if s == "default" {
t.Fatal("Failed to load config from config service")
}
err := Publish("testtopic", []byte("This is my payload"))
if err != nil {
t.Error(fmt.Sprintf("Failed to PUB: %v", err))
}
}
func loadEndpointConfig() {
log.Info("Loading ElasticSearch config")
port := config.AtPath("hailo", "service", "elasticsearch", "port").AsInt(9200)
hosts := config.AtPath("hailo", "service", "elasticsearch", "hosts").AsHostnameArray(port)
if len(hosts) == 0 {
hosts = append(hosts, "localhost:19200")
}
// Set these hosts in the Elasticsearch library
// This will initialise a host pool which uses an Epsilon Greedy algorithm to find healthy hosts
// and send to requests to them, and not unhealthy or slow hosts
eapi.Port = strconv.Itoa(port)
if port == 443 {
eapi.Protocol = "https"
}
eapi.SetHosts(hosts)
log.Infof("ElasticSearch hosts loaded: %v", eapi.Hosts)
}
// ksAuth returns the username and password for the given keyspace
func ksAuth(ks string) (string, string, error) {
if !config.AtPath("hailo", "service", "cassandra", "authentication", "enabled").AsBool() {
return "", "", nil
}
confJson := config.AtPath("hailo", "service", "cassandra", "authentication", "keyspaces").AsJson()
rawConf := make(map[string]map[string]string, 5)
if err := json.Unmarshal(confJson, &rawConf); err != nil {
// Don't log raw data in an attempt to not log our passwords
log.Warnf("[Cassandra] Failed to unmarshal authentication configuration: %s", err.Error())
return "", "", err
}
for candidateKs, v := range rawConf {
if candidateKs == ks {
return v["username"], v["password"], nil
}
}
return "", "", nil
}
func getKsConfig(ks string) (ksConfig, error) {
if !config.WaitUntilLoaded(5 * time.Second) {
return ksConfig{}, fmt.Errorf("Config not loaded")
}
username, password, err := ksAuth(ks)
if err != nil {
return ksConfig{}, err
}
c := ksConfig{
ks: ks,
hosts: getHosts(),
username: username,
password: password,
retries: config.AtPath("hailo", "service", "cassandra", "defaults", "maxRetries").AsInt(5),
cl: clFromString(config.AtPath("hailo", "service", "cassandra", "defaults", "consistencyLevel").AsString("")),
timeout: config.AtPath("hailo", "service", "cassandra", "defaults", "recvTimeout").AsDuration("1s"),
}
cc := gocql.NewCluster(c.hosts...)
cc.ProtoVersion = config.AtPath("hailo", "service", "cassandra", "defaults", "protoVersion").AsInt(2)
cc.Consistency = c.cl
cc.Compressor = gocql.SnappyCompressor{}
cc.DiscoverHosts = false
cc.NumConns = config.AtPath("hailo", "service", "cassandra", "defaults", "maxHostConns").AsInt(2)
cc.Authenticator = gocql.PasswordAuthenticator{
Username: c.username,
Password: c.password,
}
cc.Timeout = c.timeout
cc.Keyspace = c.ks
cc.RetryPolicy = &gocql.SimpleRetryPolicy{
NumRetries: c.retries,
}
cc.PoolConfig.HostSelectionPolicy = gocql.HostPoolHostPolicy(
hostpool.NewEpsilonGreedy(c.hosts, 5*time.Minute, &hostpool.LinearEpsilonValueCalculator{}),
)
c.cc = cc
return c, nil
}
func loadStatsd(addr string) g2s.Statter {
disabled := config.AtPath("hailo", "service", "instrumentation", "statsd", "disabled").AsBool()
if disabled {
return g2s.Noop()
}
s, err := g2s.Dial("udp", addr)
if err != nil {
log.Warnf("Error initialising statsd connection to %v", addr)
return nil
}
return s
}
// hostsCfgKey gets the config key that should be used to load cassandra hosts. This is to support multiple 'tiered'
// Cassandra clusters.
func hostsCfgKey() string {
// Check what cluster are we supposed to contact, revert to the default config if not specified
tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString("")
if tier == "" {
tier = defaultTier
}
switch tier {
case "general":
return "hosts"
default:
return fmt.Sprintf("%sHosts", tier)
}
}
// run is our main healthcheck loop
func (r *results) run() {
for {
select {
// Listen for config changes and update the healthcheck when needed
case <-config.SubscribeChanges():
// Allow healthcheck parameters to be overridden in config
config.AtPath("hailo", "platform", "healthcheck", r.hc.Id).AsStruct(r.hc)
case <-runNow:
r.collect()
case <-time.After(r.hc.Interval):
r.collect()
}
}
}
// PriorityHealthCheck is a healthcheck with a configurable priority for this server
func PriorityHealthCheck(id string, checker slhc.Checker, priority slhc.Priority) {
hc := &healthcheck.HealthCheck{
Id: id,
ServiceName: Name,
ServiceVersion: Version,
Hostname: hostname,
InstanceId: InstanceID,
Interval: healthcheck.StandardInterval,
Checker: checker,
Priority: priority,
}
// Allow healthcheck parameters to be overridden in config
config.AtPath("hailo", "platform", "healthcheck", id).AsStruct(hc)
healthcheck.Register(hc)
}
// getCassandraHostConfigPath gets the config key that should be used to load cassandra hosts
// this is to support multiple 'tiered' cassandra clusters
func getCassandraHostConfigKey() string {
// Check what cluster are we supposed to contact, revert to the default config if not specified
tier := config.AtPath("hailo", "service", "cassandra", "tier").AsString(defaultTier)
if tier == "" {
tier = defaultTier
}
log.Debugf("Attempting to connect to the %v Cassandra cluster", tier)
var cassandraHosts string
switch tier {
case "general":
cassandraHosts = "hosts"
default:
cassandraHosts = fmt.Sprintf("%sHosts", tier)
}
return cassandraHosts
}
// shouldTrace determiens if we should trace this request, when sending
func (r *Request) shouldTrace() bool {
if r.traceID != "" {
return true
}
pcChance := config.AtPath("hailo", "service", "trace", "pcChance").AsFloat64(0)
if pcChance <= 0 {
return false
}
if rand.Float64() < pcChance {
u4, err := uuid.NewV4()
if err != nil {
return false
}
r.SetTraceID(u4.String())
return true
}
return false
}
func reconnectDefault() {
hosts := getHosts()
recvTimeout := config.AtPath("hailo", "service", "zookeeper", "recvTimeout").AsDuration("100ms")
if !hasConfigChanged(hosts, recvTimeout) {
log.Infof("ZooKeeper config has not changed")
return
}
if defaultClient != nil {
if recvTimeout != defaultTimeout {
// cannot gracefully set timeout so close it
defaultTimeout = recvTimeout
defaultClient.Close()
} else {
// update the hosts only
log.Tracef("Setting ZK hosts to %v", hosts)
defaultClient.UpdateAddrs(hosts)
return
}
}
connectDefault(hosts, recvTimeout)
}
// load will load public key location from config service and switch pub key
func (v *validatorImpl) load() error {
v.Lock()
defer v.Unlock()
fn := config.AtPath("hailo", "service", "authentication", "publicKey").AsString("")
log.Tracef("[Auth] Loading auth library public key from: %s", fn)
if fn == "" {
return fmt.Errorf("public key filename undefined in config")
}
// load key from file
f, err := os.Open(fn)
if err != nil {
return fmt.Errorf("Failed to open public key %s (%v)", fn, err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
}
if bytes.Equal(b, v.lastRead) {
// no change
return nil
}
// turn bytes into an actual key instance
k, err := bytesToKey(b)
if err != nil {
return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
}
log.Infof("[Auth] Loaded public key: %v", k)
v.pub = k
v.lastRead = b
return nil
}
// loadConfig gets host, scheme, port and timeouts from config service
func (conn *Connection) loadConfig() {
// reload host/scheme/port and see if differnet
host := config.AtPath("hailo", "service", "graphite", "host").AsString("graphite-internal-test.elasticride.com")
scheme := config.AtPath("hailo", "service", "graphite", "scheme").AsString("http")
port := config.AtPath("hailo", "service", "graphite", "port").AsInt(-1)
connTimeout := config.AtPath("hailo", "service", "graphite", "connectTimeout").AsDuration("300ms")
reqTimeout := config.AtPath("hailo", "service", "graphite", "requestTimeout").AsDuration("2s")
rspHdrTimeout := config.AtPath("hailo", "service", "graphite", "responseHeaderTimeout").AsDuration("1s")
// hash all this and compare
h := md5.New()
io.WriteString(h, host)
io.WriteString(h, scheme)
io.WriteString(h, fmt.Sprintf("%v", port))
io.WriteString(h, fmt.Sprintf("%v", connTimeout))
io.WriteString(h, fmt.Sprintf("%v", reqTimeout))
io.WriteString(h, fmt.Sprintf("%v", rspHdrTimeout))
reloadHash := fmt.Sprintf("%x", h.Sum(nil))
if conn.currentHash() == reloadHash {
// no change
return
}
conn.Lock()
defer conn.Unlock()
conn.host, conn.scheme, conn.port = host, scheme, port
conn.connTimeout, conn.reqTimeout, conn.rspHdrTimeout = connTimeout, reqTimeout, rspHdrTimeout
// init transport
if conn.transport != nil {
conn.transport.Close()
}
conn.transport = conn.newTimeoutTransport()
conn.client = &http.Client{Transport: conn.transport}
}
func reconnectDefault() {
mtx.Lock()
defer mtx.Unlock()
log.Infof("Reloading cassandra configuration")
retries = config.AtPath("hailo", "service", "cassandra", "defaults", "maxRetries").AsInt(5)
readCl = config.AtPath("hailo", "service", "cassandra", "defaults", "readConsistencyLevel").AsString("ONE")
writeCl = config.AtPath("hailo", "service", "cassandra", "defaults", "writeConsistencyLevel").AsString("ONE")
timeout = config.AtPath("hailo", "service", "cassandra", "defaults", "recvTimeout").AsDuration("1s")
log.Debugf("Setting Cassandra defaults retries:%v, readCl: %v, writeCl: %v, timeout: %v from config", retries, readCl, writeCl, timeout)
nodes = getHosts()
log.Debugf("Setting Cassandra nodes %v from config", nodes)
// Set up authentication if enabled
if authEnabled := config.AtPath("hailo", "service", "cassandra", "authentication", "enabled").AsBool(); authEnabled {
// Get config as json as its effectively a map[string]map[string]string
authconfig := config.AtPath("hailo", "service", "cassandra", "authentication", "keyspaces").AsJson()
// Parse and set if successful
a, err := parseAuth(authconfig)
if err == nil {
auth = a
log.Debugf("Setting Cassandra authentication from config: %v", a)
} else {
log.Warnf("Failed to set Cassandra authentication from config: %v", err)
}
}
// Reset the pools map
pools = make(map[string]gossie.ConnectionPool)
}
func (s *DefaultSubscriber) doLoad() error {
subHosts := config.AtPath("hailo", "service", "nsq", "subHosts").AsHostnameArray(4150)
disableLookupd := config.AtPath("hailo", "service", "nsq", "disableLookupd").AsBool()
lookupdHosts := getHosts(4161, "hailo", "service", "nsq", "nsqlookupdSeeds")
h := md5.New()
io.WriteString(h, strings.Join(subHosts, ","))
io.WriteString(h, fmt.Sprintf("%v", disableLookupd))
io.WriteString(h, strings.Join(lookupdHosts, ","))
hash := fmt.Sprintf("%x", h.Sum(nil))
if s.configHash == hash {
return nil // don't bother as nothing interesting has changed
}
if disableLookupd {
log.Infof("Connecting to NSQ directly: %v", subHosts)
var hostList []string
for _, addr := range subHosts {
// support comma separated host lists too
hostList = append(hostList, strings.Split(addr, ",")...)
}
err := s.consumer.ConnectToNSQDs(hostList)
if err != nil && err != nsqlib.ErrAlreadyConnected {
return fmt.Errorf("Error connecting to nsqd(s): %v", err)
}
} else {
if len(lookupdHosts) > 0 {
log.Infof("Connecting to NSQ via lookupd hosts: %v", lookupdHosts)
err := s.consumer.ConnectToNSQLookupds(lookupdHosts)
if err != nil {
return fmt.Errorf("Error connecting to nsqlookupd(s): %v", err)
}
}
}
// Disconnect from old hosts
if hosts := diffHosts(s.subHosts, subHosts); len(hosts) > 0 {
log.Infof("Disconnecting from NSQ hosts: %v", hosts)
for _, host := range hosts {
err := s.consumer.DisconnectFromNSQD(host)
if err != nil && err != nsqlib.ErrNotConnected {
log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
}
}
}
// Disconnect from old lookupds
if hosts := diffHosts(s.lookupdHosts, lookupdHosts); len(hosts) > 0 {
log.Infof("Disconnecting from NSQ lookupds: %v", hosts)
for _, host := range hosts {
err := s.consumer.DisconnectFromNSQLookupd(host)
if err != nil && err != nsqlib.ErrNotConnected {
log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
}
}
}
// save state on success
s.configHash = hash
s.subHosts = subHosts
s.lookupdHosts = lookupdHosts
return nil
}
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")
// hash and test
hash, _ := config.LastLoaded()
if p.configHash == hash {
return nil
}
p.configHash = hash
// lock now and then update everything
p.Lock()
defer p.Unlock()
canonicalHosts := make(map[string]bool)
for _, host := range pubHosts {
canonicalHosts[host] = true
// do we have a producer for this host?
if _, ok := p.producers[host]; !ok {
cfg := nsqlib.NewConfig()
cfg.HeartbeatInterval = hbInterval
prod, err := nsqlib.NewProducer(host, cfg)
if err != nil {
return err
}
prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
p.producers[host] = prod
}
}
// now remove any removed ones
for host, prod := range p.producers {
if !canonicalHosts[host] {
delete(p.producers, host)
prod.Stop()
}
}
// add hosts to hostpool
p.hostpool.SetHosts(pubHosts)
log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))
// setup the other meta data
p.count = len(p.producers)
switch cl {
case "TWO":
p.cl = cl_TWO
p.n = 2
case "QUORUM":
p.cl = cl_QUORUM
p.n = p.count/2 + 1
default:
p.cl = cl_ONE // our default
p.n = 1
}
p.configHash = hash
return nil
}
// MaxConnHealthCheck asserts that the total number of established connections to all zookeeper nodes
// falls below a given max threshold.
func MaxConnHealthCheck(maxconns int) healthcheck.Checker {
return func() (map[string]string, error) {
nodes := config.AtPath("hailo", "service", "zookeeper", "hosts").AsHostnameArray(2181)
return connhealthcheck.MaxTcpConnections(nodes, maxconns)()
}
}