Beispiel #1
0
func Run(nsqLookupdHTTPAddrs []string, st store.Store) {

	cfg := nsq.NewConfig()
	cfg.UserAgent = fmt.Sprintf("phosphor go-nsq/%s", nsq.VERSION)
	cfg.MaxInFlight = maxInFlight

	consumer, err := nsq.NewConsumer(topic, channel, cfg)
	if err != nil {
		log.Critical(err)
		os.Exit(1)
	}

	consumer.AddHandler(&IngestionHandler{
		store: st,
	})

	err = consumer.ConnectToNSQLookupds(nsqLookupdHTTPAddrs)
	if err != nil {
		log.Critical(err)
		os.Exit(1)
	}

	// Block until exit
	<-consumer.StopChan
}
Beispiel #2
0
func Push(topic string, data interface{}) error {
	config := nsqq.NewConfig()
	w, err := nsqq.NewProducer(getDaemonHosts(), config)
	if w == nil && err == nil {
		return fmt.Errorf("%s", "failed to connect to producer")
	}
	w.SetLogger(&nopLogger{}, nsqq.LogLevelError)
	defer w.Stop()

	if err != nil {
		return err
	}

	js, err := json.Marshal(data)
	if err != nil {
		return nil
	}

	err = w.Publish(topic, js)
	if err != nil {
		return err
	}

	return nil
}
Beispiel #3
0
func (p *NsqSrcProvider) Init(ctx *sj.Json) error {
	var err error
	p.nsqMsgChan = make(chan nsq.Message) // not buffered

	p.timeOutSec = ctx.Get("timeout_sec").MustInt()
	nsqLookupdAddrs := ctx.Get("nsq_lookupd_addrs").MustStringArray()
	if len(nsqLookupdAddrs) <= 0 {
		return errors.New("no nsq_lookupd_addr in provided")
	}
	topic := ctx.Get("topic").MustString()
	channel := ctx.Get("channel").MustString()

	p.config = nsq.NewConfig()
	p.consumer, err = nsq.NewConsumer(topic, channel, p.config)
	if p.consumer == nil || err != nil {
		return err
	}

	p.consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
		p.nsqMsgChan <- *message
		return nil
	}))

	err = p.consumer.ConnectToNSQLookupds(nsqLookupdAddrs) // ip:port
	if err != nil {
		return errors.New("Could not connect to nsqLookupd")
	}

	return nil
}
Beispiel #4
0
func (arch *Archiver) init() {
	arch.pending = make(chan []byte)
	arch.stop = make(chan bool)
	cfg := nsq.NewConfig()
	consumer, err := nsq.NewConsumer(TOPIC, CHANNEL, cfg)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}

	// message process
	consumer.AddHandler(nsq.HandlerFunc(func(msg *nsq.Message) error {
		arch.pending <- msg.Body
		return nil
	}))

	// read environtment variable
	addresses := []string{DEFAULT_NSQLOOKUPD}
	if env := os.Getenv(ENV_NSQLOOKUPD); env != "" {
		addresses = strings.Split(env, ";")
	}

	// connect to nsqlookupd
	log.Trace("connect to nsqlookupds ip:", addresses)
	if err := consumer.ConnectToNSQLookupds(addresses); err != nil {
		log.Critical(err)
		return
	}
	log.Info("nsqlookupd connected")

	go arch.archive_task()
}
Beispiel #5
0
func productor() {
	cfg := nsq.NewConfig()
	q, _ := nsq.NewProducer(host+":5150", cfg)
	for i := 0; i < 100; i++ {
		q.Publish("test", []byte("msg"+strconv.Itoa(i)))
		time.Sleep(1 * time.Second)
	}
	q.Stop()
}
Beispiel #6
0
func initPool() {
	configs, err := goconfig.ReadConfigFile(configFileName)

	if err != nil {
		logger.Criticalf("Can not read nsq configs from %s. Error: %s", configFileName, err)
		panic(err)
	}

	options, err := configs.GetOptions(nsqdConfigSection)

	if err != nil {
		logger.Criticalf("Can not read nsqd config in %s. Error: $s", configFileName, err)
		panic(err)
	}

	addrs := make([]string, 0, len(options))

	for _, option := range options {
		value, err := configs.GetString(nsqdConfigSection, option)

		if err != nil {
			logger.Errorf("Get error when reading section %s option %s in %s. Error: %s", nsqdConfigSection, option, configFileName, err)
			continue
		}

		addrs = append(addrs, value)
	}

	if len(addrs) <= 0 {
		logger.Criticalf("Read 0 configs for nsqd address in %s.", configFileName)
		panic("Read 0 configs for nsqd address in config file " + configFileName)
	}

	pool = make(map[string]*gonsq.Producer)
	lostConns = make([]string, 0)

	for _, addr := range addrs {
		config := gonsq.NewConfig()
		producer, err := gonsq.NewProducer(addr, config)

		if err != nil {
			logger.Errorf("Can not create nsq producer for address: %s. Error: %s", addr, err)
			continue
		}

		err = producer.Ping()

		if err != nil {
			logger.Errorf("Can not connect to address %s. Error: %s", addr, err)
			lostConns = append(lostConns, addr)
		}

		pool[addr] = producer
	}

	go autoReconnect()
}
Beispiel #7
0
func consummer() {
	cfg := nsq.NewConfig()
	c, _ := nsq.NewConsumer("test", "c1", cfg)
	c.AddHandler(nsq.HandlerFunc(func(msg *nsq.Message) error { // 这个比定义一个实现handler interface的struct要快捷的多
		fmt.Println(msg.Body)
		return nil
	}))
	c.ConnectToNSQLookupd(host + ":4161")
	<-c.StopChan

}
Beispiel #8
0
func init() {

	//初始化配置
	var configError error
	config, configError = goconfig.ReadConfigFile("conf.ini")
	if configError != nil {
		log.Panic("can not read the config file")
	}

	//初始化主机名
	var errHost error
	hostname, errHost = os.Hostname()
	if errHost != nil {
		log.Fatal("Hostname can not obtain!")
	}
	AppCommUnitInit()

	//端口
	Port, _ = config.GetString("server", "port")

	//初始化指令接收者
	configConsumer := nsq.NewConfig()
	consumerTopic, _ := config.GetString("message", "consumerTopic")
	log.Println("aaaaaaa")
	log.Println(consumerTopic)
	q, _ = nsq.NewConsumer(consumerTopic, "ch", configConsumer)

	//初始化指令发送者
	configProducer := nsq.NewConfig()
	nsq_url, _ := config.GetString("message", "nsq_url")
	w, _ = nsq.NewProducer(nsq_url, configProducer)

	delay, _ = config.GetInt64("message", "delay_time")

	//指令回复处理handler
	Receive(cs)
}
Beispiel #9
0
// NewPeer creates and returns a new Peer for communicating with NSQ.
func NewPeer(host string) (*Peer, error) {
	producer, err := nsq.NewProducer(host, nsq.NewConfig())
	if err != nil {
		return nil, err
	}

	return &Peer{
		host:     host,
		producer: producer,
		messages: make(chan []byte, 10000),
		send:     make(chan []byte),
		errors:   make(chan error, 1),
		done:     make(chan bool),
		flush:    make(chan bool),
	}, nil
}
Beispiel #10
0
func connectNsqd(addr string) (*gonsq.Producer, error) {
	config := gonsq.NewConfig()
	producer, err := gonsq.NewProducer(addr, config)

	if err != nil {
		return nil, err
	}

	err = producer.Ping()

	if err != nil {
		return nil, err
	}

	return producer, nil
}
Beispiel #11
0
// Subscribe prepares the peer to consume messages.
func (n *Peer) Subscribe() error {
	consumer, err := nsq.NewConsumer(topic, broker.GenerateName(), nsq.NewConfig())
	if err != nil {
		return err
	}

	consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
		n.messages <- message.Body
		return nil
	}))

	if err := consumer.ConnectToNSQD(n.host); err != nil {
		return err
	}

	n.consumer = consumer
	return nil
}