Exemple #1
0
func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, *NSQLookupd) {
	lgr := test.NewTestLogger(t)

	nsqlookupdOpts := NewOptions()
	nsqlookupdOpts.TCPAddress = "127.0.0.1:0"
	nsqlookupdOpts.HTTPAddress = "127.0.0.1:0"
	nsqlookupdOpts.BroadcastAddress = "127.0.0.1"
	nsqlookupdOpts.Logger = lgr
	nsqlookupd1 := New(nsqlookupdOpts)
	go nsqlookupd1.Main()

	time.Sleep(100 * time.Millisecond)

	nsqdOpts := nsqd.NewOptions()
	nsqdOpts.TCPAddress = "127.0.0.1:0"
	nsqdOpts.HTTPAddress = "127.0.0.1:0"
	nsqdOpts.BroadcastAddress = "127.0.0.1"
	nsqdOpts.NSQLookupdTCPAddresses = []string{nsqlookupd1.RealTCPAddr().String()}
	nsqdOpts.Logger = lgr
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	nsqdOpts.DataPath = tmpDir
	nsqd1 := nsqd.New(nsqdOpts)
	go nsqd1.Main()

	time.Sleep(100 * time.Millisecond)

	return tmpDir, []*nsqd.NSQD{nsqd1}, nsqlookupd1
}
Exemple #2
0
func TestTLSHTTPClient(t *testing.T) {
	nsqdOpts := nsqd.NewOptions()
	nsqdOpts.Verbose = true
	nsqdOpts.TLSCert = "./test/server.pem"
	nsqdOpts.TLSKey = "./test/server-key.pem"
	nsqdOpts.TLSRootCAFile = "./test/ca.pem"
	nsqdOpts.TLSClientAuthPolicy = "require-verify"
	_, nsqdHTTPAddr, nsqd := mustStartNSQD(nsqdOpts)
	defer os.RemoveAll(nsqdOpts.DataPath)
	defer nsqd.Exit()

	opts := NewOptions()
	opts.HTTPAddress = "127.0.0.1:0"
	opts.NSQDHTTPAddresses = []string{nsqdHTTPAddr.String()}
	opts.HTTPClientTLSRootCAFile = "./test/ca.pem"
	opts.HTTPClientTLSCert = "./test/client.pem"
	opts.HTTPClientTLSKey = "./test/client-key.pem"
	nsqadmin := New(opts)
	nsqadmin.Main()
	defer nsqadmin.Exit()

	httpAddr := nsqadmin.RealHTTPAddr()
	u := url.URL{
		Scheme: "http",
		Host:   httpAddr.String(),
		Path:   "/api/nodes/" + nsqdHTTPAddr.String(),
	}

	resp, err := http.Get(u.String())
	defer resp.Body.Close()

	test.Equal(t, nil, err)
	test.Equal(t, resp.StatusCode < 500, true)
}
Exemple #3
0
func (p *program) Start() error {
	flagSet := nsqFlagset()
	flagSet.Parse(os.Args[1:])

	rand.Seed(time.Now().UTC().UnixNano())

	if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
		fmt.Println(version.String("nsqd"))
		os.Exit(0)
	}

	var cfg config
	configFile := flagSet.Lookup("config").Value.String()
	if configFile != "" {
		_, err := toml.DecodeFile(configFile, &cfg)
		if err != nil {
			log.Fatalf("ERROR: failed to load config file %s - %s", configFile, err.Error())
		}
	}
	cfg.Validate()

	opts := nsqd.NewOptions()
	options.Resolve(opts, flagSet, cfg)
	nsqd := nsqd.New(opts)

	nsqd.LoadMetadata()
	err := nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()

	p.nsqd = nsqd
	return nil
}
func (b *Broker) StartBroker() {
	b.logger.Info("start broker service")
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		opt := nsqd.NewOptions()
		opt.HTTPAddress = b.BrokerHttpAddr + ":" + b.BrokerHttpPort
		opt.TCPAddress = b.BrokerAddr + ":" + b.BrokerPort
		opt.BroadcastAddress = b.BrokerBroadCastAddr
		opt.NSQLookupdTCPAddresses = []string{b.LookUpTcpAddrr + ":" + b.LookUpTcpPort}
		b.broker = nsqd.New(opt)
		b.broker.Main()
		wg.Done()
		b.logger.Info("broker service started")
	}()
	wg.Wait()
}
Exemple #5
0
func (p *program) Start() error {
	//解析用户输入的命令
	flagSet := nsqFlagset()
	flagSet.Parse(os.Args[1:])

	rand.Seed(time.Now().UTC().UnixNano())

	//输出版本号并结束
	if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
		fmt.Println(version.String("nsqd"))
		os.Exit(0)
	}

	//读取配置文件
	var cfg config
	configFile := flagSet.Lookup("config").Value.String()
	if configFile != "" {
		_, err := toml.DecodeFile(configFile, &cfg)
		if err != nil {
			log.Fatalf("ERROR: failed to load config file %s - %s", configFile, err.Error())
		}
	}
	//验证配置文件
	cfg.Validate()

	//载入默认配置选项
	opts := nsqd.NewOptions()
	//整合配置选项(优先级:--???--)
	options.Resolve(opts, flagSet, cfg)
	//建立服务(载入基本配置)
	nsqd := nsqd.New(opts)

	//启动服务前 恢复数据(从硬盘载入数据)
	nsqd.LoadMetadata()
	//将内存中的数据写入到硬盘文件中 (创建新文件 保留旧文件--???--)
	err := nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()

	p.nsqd = nsqd
	return nil
}
Exemple #6
0
func TestConfigFlagParsing(t *testing.T) {
	flagSet := nsqFlagset()
	flagSet.Parse([]string{})

	var cfg config
	f, err := os.Open("../../contrib/nsqd.cfg.example")
	if err != nil {
		t.Fatalf("%s", err)
	}
	toml.DecodeReader(f, &cfg)
	cfg.Validate()

	opts := nsqd.NewOptions()
	options.Resolve(opts, flagSet, cfg)
	nsqd.New(opts)

	if opts.TLSMinVersion != tls.VersionTLS10 {
		t.Errorf("min %#v not expected %#v", opts.TLSMinVersion, tls.VersionTLS10)
	}
}
Exemple #7
0
func main() {
	flagSet := nsqFlagset()
	flagSet.Parse(os.Args[1:])

	rand.Seed(time.Now().UTC().UnixNano())

	if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
		fmt.Println(version.String("nsqd"))
		return
	}

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	var cfg config
	configFile := flagSet.Lookup("config").Value.String()
	if configFile != "" {
		_, err := toml.DecodeFile(configFile, &cfg)
		if err != nil {
			log.Fatalf("ERROR: failed to load config file %s - %s", configFile, err.Error())
		}
	}
	cfg.Validate()

	opts := nsqd.NewOptions()
	options.Resolve(opts, flagSet, cfg)
	nsqd := nsqd.New(opts)

	nsqd.LoadMetadata()
	err := nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()
	<-signalChan
	nsqd.Exit()
}
Exemple #8
0
// MAIN
func main() {
	var err error
	app := cli.NewApp()
	app.Name = "tmail"
	app.Usage = "SMTP server"
	app.Author = "Stéphane Depierrepont aka toorop"
	app.Email = "*****@*****.**"
	app.Version = TmailVersion
	app.Commands = tcli.CliCommands
	// no know command ? Launch server
	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 0 {
			cli.ShowAppHelp(c)
		} else {
			// if there is nothing to do then... do nothing
			if !core.Cfg.GetLaunchDeliverd() && !core.Cfg.GetLaunchSmtpd() {
				log.Fatalln("I have nothing to do, so i do nothing. Bye.")
			}

			// Init Bolt (used as cache)
			if err = core.InitBolt(); err != nil {
				log.Fatalln("Init bolt failed", err)
			}

			// Loop
			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

			// TODO
			// Chanel to comunicate between all elements
			//daChan := make(chan string)

			// init and launch nsqd
			opts := nsqd.NewOptions()
			opts.Logger = log.New(ioutil.Discard, "", 0)
			if core.Cfg.GetDebugEnabled() {
				opts.Logger = core.Log
			}
			opts.Verbose = core.Cfg.GetDebugEnabled()
			opts.DataPath = core.GetBasePath() + "/nsq"
			// if cluster get lookupd addresses
			if core.Cfg.GetClusterModeEnabled() {
				opts.NSQLookupdTCPAddresses = core.Cfg.GetNSQLookupdTcpAddresses()
			}

			// deflate (compression)
			opts.DeflateEnabled = true

			// if a message timeout it returns to the queue: https://groups.google.com/d/msg/nsq-users/xBQF1q4srUM/kX22TIoIs-QJ
			// msg timeout : base time to wait from consummer before requeuing a message
			// note: deliverd consumer return immediatly (message is handled in a go routine)
			// Ce qui est au dessus est faux malgres la go routine il attends toujours a la réponse
			// et c'est normal car le message est toujours "in flight"
			// En fait ce timeout c'est le temps durant lequel le message peut rester dans le state "in flight"
			// autrement dit c'est le temps maxi que peu prendre deliverd.processMsg
			opts.MsgTimeout = 10 * time.Minute

			// maximum duration before a message will timeout
			opts.MaxMsgTimeout = 15 * time.Hour

			// maximum requeuing timeout for a message
			// si le client ne demande pas de requeue dans ce delais alors
			// le message et considéré comme traité
			opts.MaxReqTimeout = 1 * time.Hour

			// Number of message in RAM before synching to disk
			opts.MemQueueSize = 0

			nsqd := nsqd.New(opts)
			nsqd.LoadMetadata()
			if err = nsqd.PersistMetadata(); err != nil {
				log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
			}
			nsqd.Main()

			// smtpd
			if core.Cfg.GetLaunchSmtpd() {
				// clamav ?
				if core.Cfg.GetSmtpdClamavEnabled() {
					if err = core.NewClamav().Ping(); err != nil {
						log.Fatalln("Unable to connect to clamd -", err)
					}
				}
				smtpdDsns, err := core.GetDsnsFromString(core.Cfg.GetSmtpdDsns())
				if err != nil {
					log.Fatalln("unable to parse smtpd dsn -", err)
				}
				for _, dsn := range smtpdDsns {
					go core.NewSmtpd(dsn).ListenAndServe()
					// TODO at this point we don't know if serveur is launched
					core.Log.Info("smtpd " + dsn.String() + " launched.")
				}
			}

			// deliverd
			go core.LaunchDeliverd()

			// HTTP REST server
			if core.Cfg.GetRestServerLaunch() {
				go rest.LaunchServer()
			}

			<-sigChan
			core.Log.Info("Exiting...")

			// close NsqQueueProducer if exists
			core.NsqQueueProducer.Stop()

			// flush nsqd memory to disk
			nsqd.Exit()

			// exit
			os.Exit(0)
		}
	}
	app.Run(os.Args)

}