Esempio n. 1
0
func main() {
	foreGround := flag.Bool("foreground", true, "run as foreground")
	configFile := flag.String("config", "config.toml", "the config file")

	flag.Parse()

	//	f, _ := os.Create("profiler")
	//	pprof.StartCPUProfile(f)
	//	defer func() {
	//		pprof.StopCPUProfile()
	//		os.Exit(0)
	//	}()

	if !*foreGround {
		err := util.Daemonize(0, 0)
		if err != 0 {
			log.Info("daemonize failed")
			os.Exit(-1)
		}
	}

	pid := os.Getpid()
	log.Info("Server pid: %d started", pid)

	confpath, _ := filepath.Abs(*configFile)
	runtime.GOMAXPROCS(runtime.NumCPU())
	app := server.NewApplication(confpath)
	app.Start()
	app.Loop()

	log.Info("Server pid: %d finished", pid)
}
Esempio n. 2
0
func (self *HttpServer) Serve(l net.Listener) error {
	log.Info("momonga_http: started http server: %s", l.Addr().String())

	// TODO: how do I stop this?
	go self.Server.Serve(l)
	return nil
}
Esempio n. 3
0
func (self *MmuxConnection) Attach(conn Connection) {
	self.Mutex.Lock()
	defer self.Mutex.Unlock()

	var container Connection
	container = conn
	old := atomic.SwapPointer((*unsafe.Pointer)((unsafe.Pointer)(&self.Connection)), unsafe.Pointer(&container))
	if old != nil {
		// 1.If the ClientId represents a Client already connected to the Server
		// then the Server MUST disconnect the existing Client [MQTT-3.1.4-2].
		(*(*Connection)(old)).Close()
		log.Debug("close existing connection")
	}

	self.CleanSession = conn.ShouldCleanSession()
	if conn.ShouldCleanSession() {
		self.OfflineQueue = self.OfflineQueue[:0]
		self.SubscribeMap = make(map[string]bool)
		self.SubscribedTopics = make(map[string]*SubscribeSet)

		// Should I remove remaining QoS1, QoS2 message at this time?
		self.OutGoingTable.Clean()
	} else {
		if len(self.OfflineQueue) > 0 {
			log.Info("Process Offline Queue: Playback: %d", len(self.OfflineQueue))
			for i := 0; i < len(self.OfflineQueue); i++ {
				self.writeMessageQueue(self.OfflineQueue[i])
			}
			self.OfflineQueue = self.OfflineQueue[:0]
		}
	}
}
Esempio n. 4
0
func (self *TcpServer) ListenAndServe() error {
	if self.inherit {
		file := os.NewFile(uintptr(3), "sock")
		tmp, err := net.FileListener(file)
		file.Close()
		if err != nil {
			log.Error("Error: %s", err)
			return nil
		}

		listener := tmp.(*net.TCPListener)
		self.listener = &MyListener{Listener: listener}
	} else {
		addr, err := net.ResolveTCPAddr("tcp4", self.ListenAddress)
		listener, err := net.ListenTCP("tcp", addr)

		if err != nil {
			panic(fmt.Sprintf("Error: %s", err))
			return err
		}

		self.listener = &MyListener{Listener: listener}
	}

	log.Info("momonga_tcp: started tcp server: %s", self.listener.Addr().String())
	for i := 0; i < self.config.GetAcceptorCount(); i++ {
		go self.Serve(self.listener)
	}

	return nil
}
Esempio n. 5
0
func (self *Momonga) Doom() {
	for _, x := range self.Connections {
		for _, v := range x {
			wait := 5 + rand.Intn(30)
			log.Info("DOOM in %d seconds: %s\n", wait, v.GetId())
			go func(x *MmuxConnection, wait int) {
				time.AfterFunc(time.Second*time.Duration(wait), func() {
					x.Close()
				})
			}(v, wait)
		}
	}
	time.AfterFunc(time.Second*60, func() {
		log.Info("Force Exit")
		os.Exit(0)
	})
}
Esempio n. 6
0
func (self *HttpServer) Stop() {
	close(self.stop)

	self.once.Do(func() {
		self.listener.(*HttpListener).wg.Wait()
		log.Info("Finished HTTP Listener")
		self.wg.Done()
	})
}
Esempio n. 7
0
func (self *HttpListener) File() (f *os.File, err error) {
	if tl, ok := self.Listener.(*net.TCPListener); ok {
		file, _ := tl.File()
		return file, nil
	}
	log.Info("HttpListener Failed to convert file: %T", self.Listener)

	return nil, nil
}
Esempio n. 8
0
func (self *TlsServer) ListenAndServe() error {
	if self.inherit {
		file := os.NewFile(uintptr(3), "sock")
		tmp, err := net.FileListener(file)
		file.Close()
		if err != nil {
			log.Error("Error: %s", err)
			return nil
		}

		cert, err := tls.LoadX509KeyPair(self.config.Server.Certfile, self.config.Server.Keyfile)
		if err != nil {
			panic(fmt.Sprintf("LoadX509KeyPair error: %s", err))
		}

		config := &tls.Config{Certificates: []tls.Certificate{cert}}
		self.tlsConfig = config
		// TODO: IS THIS CORRECT?
		listener := tmp.(net.Listener)
		self.listener = &MyListener{Listener: listener}
	} else {
		cert, err := tls.LoadX509KeyPair(self.config.Server.Certfile, self.config.Server.Keyfile)
		if err != nil {
			panic(fmt.Sprintf("LoadX509KeyPair error: %s", err))
		}

		config := &tls.Config{Certificates: []tls.Certificate{cert}}
		listener, err := tls.Listen("tcp", self.ListenAddress, config)
		self.tlsConfig = config

		if err != nil {
			panic(fmt.Sprintf("Error: %s", err))
		}

		self.listener = &MyListener{Listener: listener}
	}

	log.Info("momonga_tls: started tls server: %s", self.listener.Addr().String())
	for i := 0; i < self.config.GetAcceptorCount(); i++ {
		go self.Serve(self.listener)
	}

	return nil
}
Esempio n. 9
0
func (self *TlsServer) Serve(l net.Listener) error {
	defer func() {
		l.Close()
	}()

	var tempDelay time.Duration // how long to sleep on accept failure

Accept:
	for {
		select {
		case <-self.stop:
			break Accept
		default:
			client, err := l.Accept()
			if err != nil {
				if v, ok := client.(*net.TCPConn); ok {
					v.SetNoDelay(true)
					v.SetKeepAlive(true)
				}

				if ne, ok := err.(net.Error); ok && ne.Temporary() {
					if tempDelay == 0 {
						tempDelay = 5 * time.Millisecond
					} else {
						tempDelay *= 2
					}

					if max := 1 * time.Second; tempDelay > max {
						tempDelay = max
					}

					log.Info("momonga: Accept error: %v; retrying in %v", err, tempDelay)
					time.Sleep(tempDelay)
					continue
				}

				if strings.Contains(err.Error(), "use of closed network connection") {
					log.Error("Accept Failed: %s", err)
					continue
				}

				log.Error("Accept Error: %s", err)
				return err
			}
			tempDelay = 0

			myconf := GetDefaultMyConfig()
			myconf.MaxMessageSize = self.Engine.Config().Server.MessageSizeLimit
			conn := NewMyConnection(myconf)
			conn.SetMyConnection(client)
			conn.SetId(client.RemoteAddr().String())

			log.Debug("Accepted: %s", conn.GetId())
			go self.Engine.HandleConnection(conn)
		}
	}

	self.listener.(*MyListener).wg.Wait()
	self.once.Do(func() {
		self.wg.Done()
	})
	return nil
}
Esempio n. 10
0
func (self *TlsServer) Listener() Listener {
	log.Info("LIS: %#v\n", self.listener)
	return self.listener
}
Esempio n. 11
0
func (self *TlsServer) Graceful() {
	log.Info("stop new accepting")
	close(self.stop)
	self.listener.Close()

}
Esempio n. 12
0
func (self *Momonga) Handshake(p *codec.ConnectMessage, conn *MyConnection) *MmuxConnection {
	log.Debug("handshaking: %s", p.Identifier)

	if conn.Connected == true {
		log.Error("wrong sequence. (connect twice)")
		conn.Close()
		return nil
	}

	if ok := self.checkVersion(p); ok != nil {
		log.Error("magic is not expected: %s  %+v\n", string(p.Magic), p)
		conn.Close()
		return nil
	}

	// TODO: implement authenticator
	if ok, _ := self.Authenticate(p.UserName, p.Password); !ok {
		log.Error("authenticate failed:")
		conn.Close()
		return nil
	}

	// preserve messagen when will flag set
	if (p.Flag & 0x4) > 0 {
		conn.SetWillMessage(*p.Will)
	}

	if (p.Version & 0x80) > 0 {
		conn.Bridged = true
	}

	if !p.CleanSession {
		conn.DisableCleanSession()
	}

	var mux *MmuxConnection
	var err error

	reply := codec.NewConnackMessage()
	if mux, err = self.GetConnectionByClientId(p.Identifier); err == nil {
		// [MQTT-3.2.2-2] If the Server accepts a connection with CleanSession set to 0,
		// the value set in Session Present depends on whether the Server already has stored Session state
		// for the supplied client ID. If the Server has stored Session state,
		// it MUST set Session Present to 1 in the CONNACK packet.
		reply.Reserved |= 0x01
	}

	// CONNACK MUST BE FIRST RESPONSE
	// clean周りはAttachでぜんぶやるべきでは
	conn.WriteMessageQueue(reply)
	if mux != nil {
		log.Info("Attach to mux[%s]", mux.GetId())

		conn.SetId(p.Identifier)
		conn.SetKeepaliveInterval(int(p.KeepAlive))
		mux.SetKeepaliveInterval(int(p.KeepAlive))
		mux.SetState(STATE_CONNECTED)
		mux.DisableCleanSession()

		if conn.ShouldCleanSession() {
			self.CleanSubscription(mux)
		}
		mux.Attach(conn)
	} else {
		mux = NewMmuxConnection()
		mux.SetId(p.Identifier)

		mux.Attach(conn)
		self.SetConnectionByClientId(mux.GetId(), mux)

		conn.SetId(p.Identifier)
		conn.SetKeepaliveInterval(int(p.KeepAlive))
		mux.SetKeepaliveInterval(int(p.KeepAlive))
		mux.SetState(STATE_CONNECTED)

		log.Debug("Starting new mux[%s]", mux.GetId())
	}

	if p.CleanSession {
		// これは正直どうでもいい
		delete(self.RetryMap, mux.GetId())
	} else {
		// Okay, attach to existing session.
		tbl := mux.GetOutGoingTable()

		// これは途中の再送処理
		for _, c := range tbl.Hash {
			msgs := make([]codec.Message, 0)
			//check qos1, 2 message and resend to this client.
			if v, ok := c.Message.(*codec.PublishMessage); ok {
				if v.QosLevel > 0 {
					//mux.WriteMessageQueue(c.Message)
					msgs = append(msgs, c.Message)
				}
			}
			tbl.Clean()

			// TODO: improve this
			for _, v := range msgs {
				mux.WriteMessageQueue(v)
			}
		}
	}

	conn.Connected = true
	log.Debug("handshake Successful: %s", p.Identifier)
	Metrics.System.Broker.Clients.Connected.Add(1)
	return mux
}
Esempio n. 13
0
func (self *Momonga) SendPublishMessage(msg *codec.PublishMessage, client_id string, is_bridged bool) {
	// Don't pass wrong message here. user should validate the message be ore using this API.
	if len(msg.TopicName) < 1 {
		return
	}

	// TODO: Have to persist retain message.
	if msg.Retain > 0 {
		if len(msg.Payload) == 0 {
			log.Debug("[DELETE RETAIN: %s]\n%s", msg.TopicName, hex.Dump([]byte(msg.TopicName)))

			err := self.DataStore.Del([]byte(msg.TopicName), []byte(msg.TopicName))
			if err != nil {
				log.Error("Error: %s\n", err)
			}
			// これ配送したらおかしいべ
			log.Debug("Deleted retain: %s", msg.TopicName)
			// あれ、ackとかかえすんだっけ?
			Metrics.System.Broker.Messages.RetainedCount.Add(-1)
			return
		} else {
			buffer := bytes.NewBuffer(nil)
			codec.WriteMessageTo(msg, buffer)
			self.DataStore.Put([]byte(msg.TopicName), buffer.Bytes())
			Metrics.System.Broker.Messages.RetainedCount.Add(1)
		}
	}

	if Mflags["experimental.qos1"] {
		if msg.QosLevel == 1 {
			targets := self.TopicMatcher.Match(msg.TopicName)

			go func(msg *codec.PublishMessage, set []interface{}) {
				p := make(chan string, 1000)
				wg := sync.WaitGroup{}
				wg.Add(3) // bulk sernder, retry kun, receive kun

				mchan := make(chan string, 256)
				term := make(chan bool, 1)
				cnt := len(set)
				mng := make(map[string]*codec.PublishMessage)

				// retry kun。こういう実装だととても楽
				go func(term chan bool, mchan chan string, mng map[string]*codec.PublishMessage) {
					for {
						select {
						case m := <-mchan:
							if msg, ok := mng[m]; ok {
								conn, err := self.GetConnectionByClientId(m)
								if err != nil {
									fmt.Printf("something wrong: %s %s", m, err)
									continue
								}

								if err == nil {
									log.Debug("sending a retry msg: %s", msg)
									conn.WriteMessageQueue(msg)
								} else {
									log.Debug("connection not exist. next retry")
								}
							}
						case <-term:
							log.Debug("  retry finished.")

							wg.Done()
							return
						}
					}
				}(term, mchan, mng)

				// reader
				go func(p chan string, term chan bool, cnt int, mng map[string]*codec.PublishMessage, mchan chan string) {
					limit := time.After(time.Second * 60)
					for {
						select {
						case id := <-p:
							cnt--
							delete(mng, id)
							// これはcallbackでやってもいいようなきもするけど、wait groupとかもろもろ渡すの面倒くさい

							if cnt < 1 {
								log.Debug("  all delivery finished.")
								term <- true

								wg.Done()
								return
							}
						case <-time.After(time.Second * 20):
							// 終わってないやつをなめてリトライさせる
							for cid, m := range mng {
								m.Dupe = true
								mchan <- cid
							}
						case <-limit:
							log.Debug("  gave up retry.")
							term <- true
							wg.Done()
							return
						}
					}
				}(p, term, cnt, mng, mchan)

				// sender. これは勝手に終わる
				go func(msg *codec.PublishMessage, set []interface{}, p chan string, mng map[string]*codec.PublishMessage) {
					dp := make(map[string]bool)
					for i := range targets {
						var tbl *util.MessageTable
						var ok bool

						myset := targets[i].(*SubscribeSet)
						fmt.Printf("myset: %s", myset)

						// NOTE (from interoperability/client_test.py):
						//
						//   overlapping subscriptions. When there is more than one matching subscription for the same client for a topic,
						//   the server may send back one message with the highest QoS of any matching subscription, or one message for
						//   each subscription with a matching QoS.
						//
						// Currently, We choose one message for each subscription with a matching QoS.
						//
						if _, ok := dp[myset.ClientId]; ok {
							continue
						}
						dp[myset.ClientId] = true

						x, _ := codec.CopyPublishMessage(msg)
						x.QosLevel = myset.QoS
						conn, err := self.GetConnectionByClientId(myset.ClientId)
						// これは面倒臭い。clean sessionがtrueで再接続した時はもはや別人として扱わなければならない

						if x.QosLevel == 0 {
							// QoS 0にダウングレードしたらそのまま終わらせる
							conn.WriteMessageQueue(x)
							p <- myset.ClientId
							continue
						}

						if tbl, ok = self.InflightTable[myset.ClientId]; !ok {
							self.InflightTable[myset.ClientId] = util.NewMessageTable()
							// callback仕込めるんだよなー。QoS1なら使わなくてもいいかなー。とかおもったり

							tbl = self.InflightTable[myset.ClientId]
						}

						id := tbl.NewId()
						x.PacketIdentifier = id
						x.Opaque = p
						tbl.Register2(id, x, 1, x)

						if err != nil {
							continue
						}
						mng[myset.ClientId] = x
						conn.WriteMessageQueue(x)
					}
					log.Debug("  all fisrt delivery finished.")

					wg.Done()
				}(msg, targets, p, mng)

				wg.Wait()
				close(p)
				close(mchan)
				close(term)
				mng = nil
				log.Debug("  okay, cleanup qos1 sending thread.")
			}(msg, targets)
			return
		}
	}

	// Publishで受け取ったMessageIdのやつのCountをとっておく
	// で、Pubackが帰ってきたらrefcountを下げて0になったらMessageを消す
	//log.Debug("TopicName: %s %s", m.TopicName, m.Payload)
	targets := self.TopicMatcher.Match(msg.TopicName)
	if msg.TopicName[0:1] == "#" {
		// TODO:  [MQTT-4.7.2-1] The Server MUST NOT match Topic Filters starting with a wildcard character
		// (# or +) with Topic Names beginning with a $ character
	}

	// list つくってからとって、だとタイミング的に居ない奴も出てくるんだよな。マジカオス
	// ここで必要なのは, connection(clientId), subscribed qosがわかればあとは投げるだけ
	// なんで、Qlobberがかえすのはであるべきなんだけど。これすっげー消しづらいのよね・・・
	// {
	//    Connection: Connection or client id
	//    QoS:
	// }
	// いやまぁエラーハンドリングちゃんとやってれば問題ない。
	// client idのほうがベターだな。Connectionを無駄に参照つけると後が辛い
	dp := make(map[string]bool)
	count := 0

	for i := range targets {
		var cn Connection
		var ok error

		myset := targets[i].(*SubscribeSet)
		clientId := myset.ClientId
		//clientId := targets[i].(string)

		// NOTE (from interoperability/client_test.py):
		//
		//   overlapping subscriptions. When there is more than one matching subscription for the same client for a topic,
		//   the server may send back one message with the highest QoS of any matching subscription, or one message for
		//   each subscription with a matching QoS.
		//
		// Currently, We choose one message for each subscription with a matching QoS.
		//
		if _, ok := dp[clientId]; ok {
			continue
		}
		dp[clientId] = true

		cn, ok = self.GetConnectionByClientId(clientId)
		if ok != nil {
			// どちらかというとClientが悪いと思うよ!
			// リスト拾った瞬間にはいたけど、その後いなくなったんだから配送リストからこれらは無視すべき
			log.Info("(can't fetch %s. already disconnected, or unsubscribed?)", clientId)
			continue
		}

		if cn.IsBridge() && clientId == client_id {
			// Don't send message to same bridge
			continue
		}

		var x *codec.PublishMessage

		if msg.QosLevel == 0 {
			// we don't need to copy message on QoS 0
			x = msg
		} else {
			x = codec.MustCopyPublishMessage(msg)
		}

		subscriberQos := myset.QoS
		// Downgrade QoS
		if subscriberQos < x.QosLevel {
			x.QosLevel = subscriberQos
		}

		if x.QosLevel > 0 {
			// TODO: ClientごとにInflightTableを持つ
			// engineのOutGoingTableなのはとりあえず、ということ
			id := self.OutGoingTable.NewId()
			x.PacketIdentifier = id
			if sender, ok := x.Opaque.(Connection); ok {
				// TODO: ここ(でなにをするつもりだったのか・・w)
				self.OutGoingTable.Register2(x.PacketIdentifier, x, len(targets), sender)
			}
		}

		cn.WriteMessageQueue(x)
		count++
	}

	Metrics.System.Broker.Messages.Sent.Add(int64(count))
}
Esempio n. 14
0
func NewApplication(configPath string) *Application {
	conf, err := configuration.LoadConfiguration(configPath)
	if err != nil {
		log.Error("Can't read config.toml. use default setting.: %s", err)
	}
	log.SetupLogging(conf.Server.LogLevel, conf.Server.LogFile)
	pid := strconv.Itoa(os.Getpid())
	if conf.Server.PidFile != "" {
		if err := ioutil.WriteFile(conf.Server.PidFile, []byte(pid), 0644); err != nil {
			panic(err)
		}
		util.WritePid(conf.Server.PidFile)
	}

	// NOTE: INHERIT=TRUE means the process invoked from os.StartProess
	inherit := false
	if os.Getenv("INHERIT") == "TRUE" {
		inherit = true
	}

	log.Info("Momonga started pid: %s (inherit:%t)", pid, inherit)
	engine := NewMomonga(conf)
	app := &Application{
		Engine:     engine,
		Servers:    []Server{},
		configPath: configPath,
		config:     conf,
	}

	// TODO: improve this block
	if conf.Server.Port > 0 {
		t := NewTcpServer(engine, conf, inherit)
		t.wg = &app.wg
		app.RegisterServer(t)
	}
	if conf.Server.Socket != "" {
		u := NewUnixServer(engine, conf, inherit)
		u.wg = &app.wg
		app.RegisterServer(u)
	}
	if conf.Server.HttpPort > 0 {
		h := NewHttpServer(engine, conf, inherit)
		h.wg = &app.wg
		app.RegisterServer(h)
	}

	if conf.Server.EnableTls && conf.Server.TlsPort > 0 {
		h := NewTlsServer(engine, conf, inherit)
		h.wg = &app.wg
		app.RegisterServer(h)
	}

	// memonized application path
	app.execPath, err = exec.LookPath(os.Args[0])
	if err != nil {
		log.Error("Error: %s", err)
		return app
	}
	app.workingDir, err = os.Getwd()
	if err != nil {
		log.Error("Error: %s", err)
		return app
	}

	if conf.Bridge.Address != "" {
		//TODO: Bridgeは複数指定できる
		//TODO: Bridgeは途中で制御できる
		// /api/bridge/list
		// /api/bridge/connection/stop
		// /api/bridge/connection/status
		// /api/bridge/connection/start
		// /api/bridge/connection/delete
		// /api/bridge/connection/new?address=&port=&type=both&topic[]=
		// /api/bridge/config
		go func() {
			flag := 0
			switch conf.Bridge.Type {
			case "both":
				flag = 3
			case "out":
				flag = 1
			case "in":
				flag = 2
			default:
				panic(fmt.Sprintf("%s does not support.", conf.Bridge.Type))
			}

			// in
			addr := fmt.Sprintf("%s:%d", conf.Bridge.Address, conf.Bridge.Port)
			c := client.NewClient(client.Option{
				TransporterCallback: func() (net.Conn, error) {
					conn, err := net.Dial("tcp", addr)
					return conn, err
				},
				Identifier: fmt.Sprintf(conf.Bridge.ClientId),
				Magic:      []byte("MQTT"),
				Version:    4 | 0x80,
				Keepalive:  0,
			})

			c.Connect()
			c.WaitConnection()
			if flag == 1 || flag == 3 {
				c.Subscribe("#", 2)
				c.SetRequestPerSecondLimit(-1)
				c.On("publish", func(msg *codec.PublishMessage) {
					engine.SendPublishMessage(msg, conf.Bridge.Connection, true)
				})
			}

			//out
			if flag == 2 || flag == 3 {
				addr2 := fmt.Sprintf("%s:%d", conf.Server.BindAddress, conf.Server.Port)
				c2 := client.NewClient(client.Option{
					TransporterCallback: func() (net.Conn, error) {
						conn, err := net.Dial("tcp", addr2)
						return conn, err
					},
					Identifier: fmt.Sprintf(conf.Bridge.ClientId),
					Magic:      []byte("MQTT"),
					Version:    4 | 0x80,
					Keepalive:  0,
				})

				c2.Connect()
				c2.WaitConnection()
				c2.Subscribe("#", 2)
				c2.SetRequestPerSecondLimit(-1)
				c2.On("publish", func(msg *codec.PublishMessage) {
					c.Publish(msg.TopicName, msg.Payload, msg.QosLevel)
				})
			}
			select {}
		}()
	}

	return app
}
Esempio n. 15
0
func (self *Application) Start() {
	self.wg.Add(1)

	ch := make(chan os.Signal, 8)
	// TODO: windows can't use signal. split this block into another file.
	signals := []os.Signal{syscall.SIGINT, syscall.SIGHUP, syscall.SIGUSR2, syscall.SIGQUIT}
	signal.Notify(ch, signals...)

	go func(ch chan os.Signal) {
		for {
			select {
			case x := <-ch:
				switch x {
				case syscall.SIGINT:
					self.Stop()
				case syscall.SIGQUIT:
					// TODO: like sigdump feature. should change file descriptor
					pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
					pprof.Lookup("heap").WriteTo(os.Stdout, 1)
					pprof.Lookup("block").WriteTo(os.Stdout, 1)

				case syscall.SIGHUP:
					// reload config
					log.Info("reload configuration from %s", self.configPath)
					configuration.LoadConfigurationTo(self.configPath, self.config)

				case syscall.SIGUSR2:
					// graceful restart
					self.mu.Lock()
					var env []string
					for _, v := range os.Environ() {
						env = append(env, v)
					}

					discriptors := append([]*os.File{
						os.Stdin,
						os.Stdout,
						os.Stderr,
					})

					for i := 0; i < len(self.Servers); i++ {
						svr := self.Servers[i]
						f, e := svr.Listener().File()

						if e != nil {
							log.Error("Error: %s", e)
							self.mu.Unlock()
							continue
						}

						fd, _ := syscall.Dup(int(f.Fd()))
						fx := os.NewFile(uintptr(fd), "sock")
						discriptors = append(discriptors, []*os.File{fx}...)
					}

					env = append(env, "INHERIT=TRUE")
					p, err := os.StartProcess(self.execPath, os.Args, &os.ProcAttr{
						Dir:   self.workingDir,
						Env:   env,
						Files: discriptors,
					})

					if err != nil {
						log.Error("Error: %s, stop gracefull restart.", err)
						p.Kill()
						self.mu.Unlock()
						continue
					}

					// maybe, new server is alive in 3 seconds
					time.Sleep(time.Second * 3)
					for i := 0; i < len(self.Servers); i++ {
						svr := self.Servers[i]
						svr.Graceful()
					}
					self.wg.Done()

					// Kill current connection in N seconds.
					self.Engine.Doom()
					self.mu.Unlock()
					return
				}
			}
		}
	}(ch)

	go self.Engine.Run()
	for i := 0; i < len(self.Servers); i++ {
		svr := self.Servers[i]
		self.wg.Add(1)
		go svr.ListenAndServe()
	}
}