// NewDefaultSubscriber yields a DefaultSubscriber that automatically connects to // the configured (via config service) nsqlookupds to find nodes hosting the // messages for the given topic func NewDefaultSubscriber(topic string, channel string) (Subscriber, error) { return &DefaultSubscriber{ cfg: nsqlib.NewConfig(), topic: topic, channel: channel, stop: make(chan struct{}), }, nil }
// loadFromConfig grabs latest config, then diffs against currently loaded func (p *HostpoolPublisher) loadFromConfig() error { cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE") pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts") hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s") // hash and test hash, _ := config.LastLoaded() if p.configHash == hash { return nil } p.configHash = hash // lock now and then update everything p.Lock() defer p.Unlock() canonicalHosts := make(map[string]bool) for _, host := range pubHosts { canonicalHosts[host] = true // do we have a producer for this host? if _, ok := p.producers[host]; !ok { cfg := nsqlib.NewConfig() cfg.HeartbeatInterval = hbInterval prod, err := nsqlib.NewProducer(host, cfg) if err != nil { return err } prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug) p.producers[host] = prod } } // now remove any removed ones for host, prod := range p.producers { if !canonicalHosts[host] { delete(p.producers, host) prod.Stop() } } // add hosts to hostpool p.hostpool.SetHosts(pubHosts) log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", ")) // setup the other meta data p.count = len(p.producers) switch cl { case "TWO": p.cl = cl_TWO p.n = 2 case "QUORUM": p.cl = cl_QUORUM p.n = p.count/2 + 1 default: p.cl = cl_ONE // our default p.n = 1 } p.configHash = hash return nil }
// NewServiceLoader returns a loader that reads config from config service func NewServiceLoader(c *Config, addr, service, region, env string) (*Loader, error) { // define our hierarchy: // H2:BASE // H2:BASE:<service-name> // H2:REGION:<aws region> // H2:REGION:<aws region>:<service-name> // H2:ENV:<env> // H2:ENV:<env>:<service-name> hierarchy := []string{ "H2:BASE", fmt.Sprintf("H2:BASE:%s", service), fmt.Sprintf("H2:REGION:%s", region), fmt.Sprintf("H2:REGION:%s:%s", region, service), fmt.Sprintf("H2:ENV:%s", env), fmt.Sprintf("H2:ENV:%s:%s", env, service), } // construct URL if !strings.Contains(addr, "://") { addr = "https://" + addr } addr = strings.TrimRight(addr, "/") + "/compile" u, err := url.Parse(addr) if err != nil { return nil, fmt.Errorf("Failed to parse config service address: %v", err) } q := u.Query() q.Set("ids", strings.Join(hierarchy, ",")) u.RawQuery = q.Encode() configUrl := u.String() log.Infof("[Config] Initialising service loader for service '%s' in region '%s' in '%s' environment via URL %s", service, region, env, configUrl) rdr := func() (io.ReadCloser, error) { rsp, err := http.Get(configUrl) if err != nil { log.Errorf("[Config] Failed to load config via %s: %v", configUrl, err) return nil, fmt.Errorf("Failed to load config via %s: %v", configUrl, err) } defer rsp.Body.Close() if rsp.StatusCode != 200 { log.Errorf("[Config] Failed to load config via %s - status code %v", configUrl, rsp.StatusCode) return nil, fmt.Errorf("Failed to load config via %s - status code %v", configUrl, rsp.StatusCode) } b, _ := ioutil.ReadAll(rsp.Body) loaded := make(map[string]interface{}) err = json.Unmarshal(b, &loaded) if err != nil { log.Errorf("[Config] Unable to unmarshal loaded config: %v", err) return nil, fmt.Errorf("Unable to unmarshal loaded config: %v", err) } b, err = json.Marshal(loaded["config"]) if err != nil { log.Errorf("[Config] Unable to unmarshal loaded config: %v", err) return nil, fmt.Errorf("Unable to unmarshal loaded config: %v", err) } rdr := ioutil.NopCloser(bytes.NewReader(b)) return rdr, nil } changesChan := make(chan bool) l := NewLoader(c, changesChan, rdr) go func() { // wait until loaded l.reload() // look out for config changes PUBbed via NSQ -- subscribe via a random ephemeral channel channel := fmt.Sprintf("g%v#ephemeral", rand.Uint32()) consumer, err := nsqlib.NewConsumer("config.reload", channel, nsqlib.NewConfig()) if err != nil { log.Warnf("[Config] Failed to create NSQ reader to pickup config changes (fast reload disabled): ch=%v %v", channel, err) return } consumer.AddHandler(nsqlib.HandlerFunc(func(m *nsqlib.Message) error { changesChan <- true return nil })) // now configure it -- NOT via lookupd! There is a bug we think! subHosts := AtPath("hailo", "service", "nsq", "subHosts").AsHostnameArray(4150) if len(subHosts) == 0 { log.Warnf("[Config] No subHosts defined for config.reload topic (fast reload disabled)") return } log.Infof("[Config Load] Subscribing to config.reload (for fast config reloads) via NSQ hosts: %v", subHosts) if err := consumer.ConnectToNSQDs(subHosts); err != nil { log.Warnf("[Config Load] Failed to connect to NSQ for config changes (fast reload disabled): %v", err) return } // Wait for the Loader to be killed, and then stop the NSQ reader l.Wait() consumer.Stop() }() return l, nil }