// NewDefaultSubscriber yields a DefaultSubscriber that automatically connects to
// the configured (via config service) nsqlookupds to find nodes hosting the
// messages for the given topic
func NewDefaultSubscriber(topic string, channel string) (Subscriber, error) {
	return &DefaultSubscriber{
		cfg:     nsqlib.NewConfig(),
		topic:   topic,
		channel: channel,
		stop:    make(chan struct{}),
	}, nil
}
Example #2
0
// loadFromConfig grabs latest config, then diffs against currently loaded
func (p *HostpoolPublisher) loadFromConfig() error {
	cl := config.AtPath("hailo", "service", "nsq", "writeCl").AsString("ONE")
	pubHosts := getHosts(4150, "hailo", "service", "nsq", "pubHosts")
	hbInterval := config.AtPath("hailo", "service", "nsq", "pubHeartbeatInterval").AsDuration("30s")

	// hash and test
	hash, _ := config.LastLoaded()
	if p.configHash == hash {
		return nil
	}
	p.configHash = hash

	// lock now and then update everything
	p.Lock()
	defer p.Unlock()

	canonicalHosts := make(map[string]bool)
	for _, host := range pubHosts {
		canonicalHosts[host] = true
		// do we have a producer for this host?
		if _, ok := p.producers[host]; !ok {
			cfg := nsqlib.NewConfig()
			cfg.HeartbeatInterval = hbInterval
			prod, err := nsqlib.NewProducer(host, cfg)
			if err != nil {
				return err
			}
			prod.SetLogger(&logBridge{}, nsqlib.LogLevelDebug)
			p.producers[host] = prod
		}
	}

	// now remove any removed ones
	for host, prod := range p.producers {
		if !canonicalHosts[host] {
			delete(p.producers, host)
			prod.Stop()
		}
	}

	// add hosts to hostpool
	p.hostpool.SetHosts(pubHosts)

	log.Infof("Initialized NSQ publisher with hosts %v", strings.Join(pubHosts, ", "))

	// setup the other meta data
	p.count = len(p.producers)
	switch cl {
	case "TWO":
		p.cl = cl_TWO
		p.n = 2
	case "QUORUM":
		p.cl = cl_QUORUM
		p.n = p.count/2 + 1
	default:
		p.cl = cl_ONE // our default
		p.n = 1
	}

	p.configHash = hash

	return nil
}
Example #3
0
// NewServiceLoader returns a loader that reads config from config service
func NewServiceLoader(c *Config, addr, service, region, env string) (*Loader, error) {
	// define our hierarchy:
	// H2:BASE
	// H2:BASE:<service-name>
	// H2:REGION:<aws region>
	// H2:REGION:<aws region>:<service-name>
	// H2:ENV:<env>
	// H2:ENV:<env>:<service-name>

	hierarchy := []string{
		"H2:BASE",
		fmt.Sprintf("H2:BASE:%s", service),
		fmt.Sprintf("H2:REGION:%s", region),
		fmt.Sprintf("H2:REGION:%s:%s", region, service),
		fmt.Sprintf("H2:ENV:%s", env),
		fmt.Sprintf("H2:ENV:%s:%s", env, service),
	}

	// construct URL
	if !strings.Contains(addr, "://") {
		addr = "https://" + addr
	}
	addr = strings.TrimRight(addr, "/") + "/compile"
	u, err := url.Parse(addr)
	if err != nil {
		return nil, fmt.Errorf("Failed to parse config service address: %v", err)
	}
	q := u.Query()
	q.Set("ids", strings.Join(hierarchy, ","))
	u.RawQuery = q.Encode()

	configUrl := u.String()

	log.Infof("[Config] Initialising service loader for service '%s' in region '%s' in '%s' environment via URL %s", service, region, env, configUrl)

	rdr := func() (io.ReadCloser, error) {
		rsp, err := http.Get(configUrl)
		if err != nil {
			log.Errorf("[Config] Failed to load config via %s: %v", configUrl, err)
			return nil, fmt.Errorf("Failed to load config via %s: %v", configUrl, err)
		}
		defer rsp.Body.Close()
		if rsp.StatusCode != 200 {
			log.Errorf("[Config] Failed to load config via %s - status code %v", configUrl, rsp.StatusCode)
			return nil, fmt.Errorf("Failed to load config via %s - status code %v", configUrl, rsp.StatusCode)
		}
		b, _ := ioutil.ReadAll(rsp.Body)

		loaded := make(map[string]interface{})
		err = json.Unmarshal(b, &loaded)
		if err != nil {
			log.Errorf("[Config] Unable to unmarshal loaded config: %v", err)
			return nil, fmt.Errorf("Unable to unmarshal loaded config: %v", err)
		}

		b, err = json.Marshal(loaded["config"])
		if err != nil {
			log.Errorf("[Config] Unable to unmarshal loaded config: %v", err)
			return nil, fmt.Errorf("Unable to unmarshal loaded config: %v", err)
		}
		rdr := ioutil.NopCloser(bytes.NewReader(b))
		return rdr, nil
	}

	changesChan := make(chan bool)
	l := NewLoader(c, changesChan, rdr)

	go func() {
		// wait until loaded
		l.reload()

		// look out for config changes PUBbed via NSQ -- subscribe via a random ephemeral channel
		channel := fmt.Sprintf("g%v#ephemeral", rand.Uint32())
		consumer, err := nsqlib.NewConsumer("config.reload", channel, nsqlib.NewConfig())
		if err != nil {
			log.Warnf("[Config] Failed to create NSQ reader to pickup config changes (fast reload disabled): ch=%v %v", channel, err)
			return
		}
		consumer.AddHandler(nsqlib.HandlerFunc(func(m *nsqlib.Message) error {
			changesChan <- true
			return nil
		}))

		// now configure it -- NOT via lookupd! There is a bug we think!
		subHosts := AtPath("hailo", "service", "nsq", "subHosts").AsHostnameArray(4150)
		if len(subHosts) == 0 {
			log.Warnf("[Config] No subHosts defined for config.reload topic (fast reload disabled)")
			return
		}

		log.Infof("[Config Load] Subscribing to config.reload (for fast config reloads) via NSQ hosts: %v", subHosts)
		if err := consumer.ConnectToNSQDs(subHosts); err != nil {
			log.Warnf("[Config Load] Failed to connect to NSQ for config changes (fast reload disabled): %v", err)
			return
		}

		// Wait for the Loader to be killed, and then stop the NSQ reader
		l.Wait()
		consumer.Stop()
	}()

	return l, nil
}