func getHosts() []string {
hostsConfigPath := []string{"hailo", "service", "zookeeper", "hosts"}
tier := config.AtPath("hailo", "service", "zookeeper", "tier").AsString("general")
if tier != "general" {
hostsConfigPath = append(hostsConfigPath, tier)
}
if hosts := config.AtPath(hostsConfigPath...).AsHostnameArray(2181); len(hosts) > 0 {
return hosts
}
// no hosts returned so try dns
hosts, err := dns.Hosts("zookeeper-" + tier)
if err != nil {
log.Errorf("Failed to load ZK hosts from dns: %v", err)
return []string{"localhost:2181"}
}
// for safety fall back to localhost
if len(hosts) == 0 {
return []string{"localhost:2181"}
}
// append port
for i, host := range hosts {
hosts[i] = host + ":2181"
}
return hosts
}
func newdefaultClient() MemcacheClient {
serverSelector := new(memcache.ServerList)
client := memcache.NewFromSelector(serverSelector)
// Listen for config changes
ch := config.SubscribeChanges()
go func() {
for _ = range ch {
loadFromConfig(serverSelector, client)
}
}()
loadFromConfig(serverSelector, client)
// Log on init
hosts := config.AtPath("hailo", "service", "memcache", "servers").AsHostnameArray(11211)
operationTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
AsDuration(defaultDialTimeout)
dialTimeout := config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
AsDuration(defaultOperationTimeout)
log.Infof("[Memcache] Initialising Memcache client to hosts %v: dial timeout %v, op timeout: %v", hosts,
dialTimeout, operationTimeout)
return client
}
func getHosts() []string {
hostConfigPath := []string{"hailo", "service", "memcache", "servers"}
host := "memcached"
// check if tier is specified and act accordingly
tier := config.AtPath("hailo", "service", "memcache", "tier").AsString("")
if tier != "" {
hostConfigPath = append(hostConfigPath, tier)
host = fmt.Sprintf("%s-%s", host, tier)
}
if hosts := config.AtPath(hostConfigPath...).AsHostnameArray(11211); len(hosts) > 0 {
return hosts
}
// no hosts returned so try dns
hosts, err := dns.Hosts(host)
if err != nil {
log.Errorf("[Memcache] Failed to load hosts from dns, returning empty list: %v", err)
return []string{}
}
// append port
for i, host := range hosts {
hosts[i] = host + ":11211"
}
return hosts
}
func getHosts(port int, path ...string) []string {
if hosts := config.AtPath(path...).AsHostnameArray(port); len(hosts) > 0 {
return hosts
}
// should we lookup dns?
if config.AtPath("hailo", "service", "nsq", "disableDnsLookup").AsBool() {
return []string{}
}
// try dns lookup
cluster := config.AtPath("hailo", "service", "nsq", "cluster").AsString("general")
hosts, err := dns.Hosts("nsq-" + cluster)
if err != nil {
log.Errorf("Failed to load NSQ hosts from dns: %v", err)
return []string{}
}
// append port
for i, host := range hosts {
hosts[i] = fmt.Sprintf("%s:%d", host, port)
}
return hosts
}
func createCircuit(service, endpoint string) Circuit {
options := defaultOptions
config.AtPath("hailo", "platform", "circuitbreaker").AsStruct(&options)
config.AtPath("hailo", "platform", "circuitbreaker", "endpoints", service, endpoint).AsStruct(&options)
log.Debugf("Circuitbreaker config for %s.%s: %#v", service, endpoint, options)
return NewDefaultCircuit(options)
}
func loadFromConfig(sl *memcache.ServerList, client *memcache.Client) {
hosts := getHosts()
log.Tracef("[Memcache] Setting memcache servers from config: %v", hosts)
err := sl.SetServers(hosts...)
if err != nil {
log.Errorf("[Memcache] Error setting memcache servers: %v", err)
}
// Technically we have a race here since the timeouts are not protected by a mutex, however it isn't really a
// problem if the timeout is stale for a short period.
client.Timeout = config.AtPath("hailo", "service", "memcache", "timeouts", "operationTimeout").
AsDuration(defaultOperationTimeout)
log.Tracef("[Memcache] Set Memcache operation timeout from config: %v", client.Timeout)
client.DialTimeout = config.AtPath("hailo", "service", "memcache", "timeouts", "dialTimeout").
AsDuration(defaultDialTimeout)
log.Tracef("[Memcache] Set Memcache dial timeout from config: %v", client.DialTimeout)
}
func TestPub(t *testing.T) {
config.LoadFromService("testservice")
s := config.AtPath("configService", "hash").AsString("default")
if s == "default" {
t.Fatal("Failed to load config from config service")
}
err := Publish("testtopic", []byte("This is my payload"))
if err != nil {
t.Error(fmt.Sprintf("Failed to PUB: %v", err))
}
}
// run is our main healthcheck loop
func (r *results) run() {
for {
select {
// Listen for config changes and update the healthcheck when needed
case <-config.SubscribeChanges():
// Allow healthcheck parameters to be overridden in config
config.AtPath("hailo", "platform", "healthcheck", r.hc.Id).AsStruct(r.hc)
case <-runNow:
r.collect()
case <-time.After(r.hc.Interval):
r.collect()
}
}
}
// PriorityHealthCheck is a healthcheck with a configurable priority for this server
func PriorityHealthCheck(id string, checker slhc.Checker, priority slhc.Priority) {
hc := &healthcheck.HealthCheck{
Id: id,
ServiceName: Name,
ServiceVersion: Version,
Hostname: hostname,
InstanceId: InstanceID,
Interval: healthcheck.StandardInterval,
Checker: checker,
Priority: priority,
}
// Allow healthcheck parameters to be overridden in config
config.AtPath("hailo", "platform", "healthcheck", id).AsStruct(hc)
healthcheck.Register(hc)
}
// shouldTrace determiens if we should trace this request, when sending
func (r *Request) shouldTrace() bool {
if r.traceID != "" {
return true
}
pcChance := config.AtPath("hailo", "service", "trace", "pcChance").AsFloat64(0)
if pcChance <= 0 {
return false
}
if rand.Float64() < pcChance {
u4, err := uuid.NewV4()
if err != nil {
return false
}
r.SetTraceID(u4.String())
return true
}
return false
}
func reconnectDefault() {
hosts := getHosts()
recvTimeout := config.AtPath("hailo", "service", "zookeeper", "recvTimeout").AsDuration("100ms")
if !hasConfigChanged(hosts, recvTimeout) {
log.Infof("ZooKeeper config has not changed")
return
}
if defaultClient != nil {
if recvTimeout != defaultTimeout {
// cannot gracefully set timeout so close it
defaultTimeout = recvTimeout
defaultClient.Close()
} else {
// update the hosts only
log.Tracef("Setting ZK hosts to %v", hosts)
defaultClient.UpdateAddrs(hosts)
return
}
}
connectDefault(hosts, recvTimeout)
}
// load will load public key location from config service and switch pub key
func (v *validatorImpl) load() error {
v.Lock()
defer v.Unlock()
fn := config.AtPath("hailo", "service", "authentication", "publicKey").AsString("")
log.Tracef("[Auth] Loading auth library public key from: %s", fn)
if fn == "" {
return fmt.Errorf("public key filename undefined in config")
}
// load key from file
f, err := os.Open(fn)
if err != nil {
return fmt.Errorf("Failed to open public key %s (%v)", fn, err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
}
if bytes.Equal(b, v.lastRead) {
// no change
return nil
}
// turn bytes into an actual key instance
k, err := bytesToKey(b)
if err != nil {
return fmt.Errorf("Failed to read public key from %s (%v)", fn, err)
}
log.Infof("[Auth] Loaded public key: %v", k)
v.pub = k
v.lastRead = b
return nil
}
func (s *DefaultSubscriber) doLoad() error {
subHosts := config.AtPath("hailo", "service", "nsq", "subHosts").AsHostnameArray(4150)
disableLookupd := config.AtPath("hailo", "service", "nsq", "disableLookupd").AsBool()
lookupdHosts := getHosts(4161, "hailo", "service", "nsq", "nsqlookupdSeeds")
h := md5.New()
io.WriteString(h, strings.Join(subHosts, ","))
io.WriteString(h, fmt.Sprintf("%v", disableLookupd))
io.WriteString(h, strings.Join(lookupdHosts, ","))
hash := fmt.Sprintf("%x", h.Sum(nil))
if s.configHash == hash {
return nil // don't bother as nothing interesting has changed
}
if disableLookupd {
log.Infof("Connecting to NSQ directly: %v", subHosts)
var hostList []string
for _, addr := range subHosts {
// support comma separated host lists too
hostList = append(hostList, strings.Split(addr, ",")...)
}
err := s.consumer.ConnectToNSQDs(hostList)
if err != nil && err != nsqlib.ErrAlreadyConnected {
return fmt.Errorf("Error connecting to nsqd(s): %v", err)
}
} else {
if len(lookupdHosts) > 0 {
log.Infof("Connecting to NSQ via lookupd hosts: %v", lookupdHosts)
err := s.consumer.ConnectToNSQLookupds(lookupdHosts)
if err != nil {
return fmt.Errorf("Error connecting to nsqlookupd(s): %v", err)
}
}
}
// Disconnect from old hosts
if hosts := diffHosts(s.subHosts, subHosts); len(hosts) > 0 {
log.Infof("Disconnecting from NSQ hosts: %v", hosts)
for _, host := range hosts {
err := s.consumer.DisconnectFromNSQD(host)
if err != nil && err != nsqlib.ErrNotConnected {
log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
}
}
}
// Disconnect from old lookupds
if hosts := diffHosts(s.lookupdHosts, lookupdHosts); len(hosts) > 0 {
log.Infof("Disconnecting from NSQ lookupds: %v", hosts)
for _, host := range hosts {
err := s.consumer.DisconnectFromNSQLookupd(host)
if err != nil && err != nsqlib.ErrNotConnected {
log.Warnf("Error disconnecting from NSQ host %s: %v", host, err)
}
}
}
// save state on success
s.configHash = hash
s.subHosts = subHosts
s.lookupdHosts = lookupdHosts
return nil
}
func loadPhosphorAddr() string {
host := config.AtPath("hailo", "service", "trace", "phosphor", "host").AsString("localhost")
port := config.AtPath("hailo", "service", "trace", "phosphor", "port").AsInt(8130)
return fmt.Sprintf("%s:%v", host, port)
}
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")
// hash and test
hash, _ := config.LastLoaded()
if p.configHash == hash {
return nil
}
p.configHash = hash
// lock now and then update everything
p.Lock()
defer p.Unlock()
canonicalHosts := make(map[string]bool)
for _, host := range pubHosts {
canonicalHosts[host] = true
// do we have a producer for this host?
if _, ok := p.producers[host]; !ok {
cfg := nsqlib.NewConfig()
cfg.HeartbeatInterval = hbInterval
prod, err := nsqlib.NewProducer(host, cfg)
if err != nil {
return err
}
prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
p.producers[host] = prod
}
}
// now remove any removed ones
for host, prod := range p.producers {
if !canonicalHosts[host] {
delete(p.producers, host)
prod.Stop()
}
}
// add hosts to hostpool
p.hostpool.SetHosts(pubHosts)
log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))
// setup the other meta data
p.count = len(p.producers)
switch cl {
case "TWO":
p.cl = cl_TWO
p.n = 2
case "QUORUM":
p.cl = cl_QUORUM
p.n = p.count/2 + 1
default:
p.cl = cl_ONE // our default
p.n = 1
}
p.configHash = hash
return nil
}
func loadStatsdAddr() string {
host := config.AtPath("hailo", "service", "instrumentation", "statsd", "host").AsString("localhost")
port := config.AtPath("hailo", "service", "instrumentation", "statsd", "port").AsInt(8125)
return fmt.Sprintf("%s:%v", host, port)
}
// MaxConnHealthCheck asserts that the total number of established connections to all zookeeper nodes
// falls below a given max threshold.
func MaxConnHealthCheck(maxconns int) healthcheck.Checker {
return func() (map[string]string, error) {
nodes := config.AtPath("hailo", "service", "zookeeper", "hosts").AsHostnameArray(2181)
return connhealthcheck.MaxTcpConnections(nodes, maxconns)()
}
}