Exemple #1
0
func main() {
	if err := initialize(); err != nil {
		panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
	}

	//start server...
	server := network.NewServer()
	if server == nil {
		panic("new tcp server failed.")
	}
	go server.Start()

	//start rule updater
	updater := task.NewUpdater()
	if updater == nil {
		panic("new updater failed.")
	}
	go updater.Start()

	//start worker
	filters := make([]*task.Filter, common.WorkerNum)
	loggerf := common.NewLogger("filter")
	backends := make([]*network.Backend, common.WorkerNum)
	loggerb := common.NewLogger("backend")
	for i := 0; i < common.WorkerNum; i++ {
		filters[i] = task.NewFilter(i, common.PacketChans[i], common.FilterChans[i], loggerf)
		if filters[i] == nil {
			panic("new filter failed.")
		}
		go filters[i].Start()

		backends[i] = network.NewBackend(i, common.FilterChans[i], loggerb)
		if backends[i] == nil {
			panic("new backend failed.")
		}
		go backends[i].Start()
	}

	fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
	//wait for signal
	sig_chan := make(chan os.Signal)
	signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
	<-sig_chan

	//clear works...
	server.Stop()
	updater.Stop()
	for i := 0; i < common.WorkerNum; i++ {
		filters[i].Stop()
		backends[i].Stop()
	}

	common.Alarm(VERSION + "-exit")
	time.Sleep(time.Second)
	fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
Exemple #2
0
func NewServer() *Server {
	s := &Server{
		port:          common.GetConfInt("server", "port", 8080),
		maxClients:    common.GetConfInt("server", "max_clients", 100000),
		clientNum:     0,
		acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
		connTimeout:   common.GetConfSecond("server", "connection_timeout", 60*3),
		headerLen:     common.GetConfInt("server", "header_length", 10),
		maxBodyLen:    common.GetConfInt("server", "max_body_length", 102400),
		running:       false,
		heartbeat:     time.Now().Unix(),
		workerNum:     common.WorkerNum,
		currentWorker: -1,
		logger:        common.NewLogger("server"),
	}

	if s == nil {
		fmt.Println("new server failed")
		return nil
	}

	if s.logger == nil {
		fmt.Println("new server logger failed")
		return nil
	}

	ip, err := common.Conf.String("server", "bind")
	if err != nil {
		ip = ""
	}
	s.ip = ip

	return s
}
Exemple #3
0
func NewUpdater() *Updater {
	u := &Updater{
		running:   false,
		heartbeat: time.Now().Unix(),
		interval:  common.GetConfSecond("updater", "interval", 20*60),
		logger:    common.NewLogger("updater"),
	}

	if u == nil {
		fmt.Println("new updater failed")
		return nil
	}

	redisAddr, err := common.Conf.String("updater", "redis_addr")
	if err != nil {
		fmt.Printf("updater get redis_addr failed, err:%s\n", err.Error())
		return nil
	}
	u.redisCli.Addr = redisAddr
	u.redisCli.Db = common.GetConfInt("updater", "redis_db", 0)

	if u.logger == nil {
		fmt.Println("updater new logger failed")
		return nil
	}

	return u
}
Exemple #4
0
func NewServer() *Server {
	s := &Server{
		port:          common.GetConfInt("server", "port", 8080),
		maxClients:    common.GetConfInt("server", "max_clients", 100000),
		clientNum:     0,
		acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
		connTimeout:   common.GetConfSecond("server", "connection_timeout", 60*3),
		headerLen:     common.GetConfInt("server", "header_length", 10),
		maxBodyLen:    common.GetConfInt("server", "max_body_length", 102400),
		running:       false,
		heartbeat:     time.Now().Unix(),
		workerNum:     common.WorkerNum,
		currentWorker: -1,
		slowread:      common.GetConfSecond("server", "slow_read", 0),
		logger:        common.NewLogger("server"),
	}

	if s == nil {
		fmt.Println("new server failed")
		return nil
	}

	if s.logger == nil {
		fmt.Println("new server logger failed")
		return nil
	}

	ip, err := common.Conf.String("server", "bind")
	if err != nil {
		ip = ""
	}
	s.ip = ip

	max_clients := uint64(s.maxClients)
	if max_clients > 1024 {
		var rlim syscall.Rlimit
		err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
		if err != nil {
			fmt.Println("server get rlimit error: " + err.Error())
			return nil
		}
		rlim.Cur = max_clients
		rlim.Max = max_clients
		err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
		if err != nil {
			fmt.Println("server set rlimit error: " + err.Error())
			return nil
		}
		s.logger.Info("set fd limit to %d", s.maxClients)
	}

	return s
}
Exemple #5
0
func main() {
	if err := initialize(); err != nil {
		panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
	}

	//start server...
	server := network.NewServer()
	if server == nil {
		panic("new tcp server failed.")
	}
	go server.Start()

	//start worker
	savers := make([]*task.KafkaSaver, common.WorkerNum)
	logger := common.NewLogger("kafka")
	for i := 0; i < common.WorkerNum; i++ {
		savers[i] = task.NewKafkaSaver(i, common.PacketChans[i], logger)
		if savers[i] == nil {
			panic("new filter failed.")
		}
		go savers[i].Start()
	}

	fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
	//wait for signal
	sig_chan := make(chan os.Signal)
	signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
	<-sig_chan

	//clear works...
	server.Stop()
	for i := 0; i < common.WorkerNum; i++ {
		savers[i].Stop()
	}

	common.Alarm(VERSION + "-exit")
	time.Sleep(time.Second)
	fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
Exemple #6
0
func NewBackend(idx int, inputChan chan []byte) *Backend {
	b := &Backend{
		index:      idx,
		backendNum: 0,
		heartbeat:  time.Now().Unix(),
		logger:     common.NewLogger("backend"),
		inChan:     inputChan,
		running:    false,
	}

	if b == nil {
		fmt.Println("new backend failed")
		return nil
	}

	if b.logger == nil {
		fmt.Println("backend new logger failed")
		return nil
	}

	if b.inChan == nil {
		fmt.Println("backend must has an input channel to get bytes for back point")
		return nil
	}

	options, err := common.Conf.Options("backend")
	if err != nil {
		fmt.Printf("backend get all options failed:%s\n", err.Error())
		return nil
	}

	b.backendList = make([]*Backpoint, 0, 4)
	for _, option := range options {
		if !strings.HasPrefix(option, "backend_list_") {
			continue
		}

		back, err := common.Conf.String("backend", option)
		if err != nil {
			fmt.Printf("Read conf %s failed, error: %s, get backend total number:%d\n", option, err.Error(), b.backendNum)
			break
		}

		backend_name := strings.TrimPrefix(option, "backend_list_")
		if backend_name == "" || backend_name == option {
			fmt.Printf("get backend name failed")
			break
		}

		addrs := strings.Split(back, ";")
		num := len(addrs)

		if num < 1 {
			fmt.Printf("one backend:%s must at least has one address", backend_name)
			continue
		}

		point := &Backpoint{
			name:           backend_name,
			backNum:        num,
			backAddrs:      addrs,
			conn:           nil,
			connTimeout:    common.GetConfSecond("backend", "connection_timeout", 180),
			sendTimeout:    common.GetConfSecond("backend", "send_timeout", 180),
			currentBackIdx: -1,
			retryTimes:     common.GetConfInt("backend", "retry_times", 5),
			retryInterval:  common.GetConfSecond("backend", "retry_interval", 5),
			recvBuf:        make([]byte, common.GetConfInt("backend", "receive_buffer_size", 10)),
			sendingChan:    make(chan []byte, common.GetConfInt("backend", "sending_buffer_size", 10000)),
		}
		if point == nil {
			fmt.Println("new back point failed")
			return nil
		}

		b.backendList = append(b.backendList, point)
		b.backendNum += 1
		b.logger.Debug("Backend%d get a back %s: %s, %d points", idx, backend_name, back, num)
	}

	if b.backendNum < 1 {
		fmt.Println("no backends")
		return nil
	}

	return b
}
Exemple #7
0
func NewServer() *Server {
	s := &Server{
		port:          common.GetConfInt("server", "port", 8080),
		maxClients:    common.GetConfInt("server", "max_clients", 100000),
		headerLen:     common.GetConfInt("server", "header_length", 10),
		maxBodyLen:    common.GetConfInt("server", "max_body_length", 102400),
		acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
		connTimeout:   common.GetConfSecond("server", "connection_timeout", 60*3),
		clientNum:     0,
		currentWorker: -1,
		running:       false,
		heartbeat:     time.Now().Unix(),
		workerNum:     common.WorkerNum,
		logger:        common.NewLogger("server"),
		ipWhitelist:   nil,
		itemWhitelist: nil,
		itemBlacklist: nil,
	}

	if s == nil {
		fmt.Println("new server failed")
		return nil
	}

	if s.logger == nil {
		fmt.Println("new server logger failed")
		return nil
	}

	ip, err := common.Conf.String("server", "bind")
	if err != nil {
		//not set, bind all interface
		ip = ""
	}
	s.ip = ip

	white_list, err := common.Conf.String("server", "ip_whitelist")
	if err == nil {
		s.ipWhitelist = make(map[string]bool)
		for _, ip := range strings.Split(white_list, ";") {
			s.ipWhitelist[ip] = true
		}
	}
	s.logger.Debug("connection ip white list:%v", s.ipWhitelist)

	options, err := common.Conf.Options("item_blacklist")
	if err == nil && len(options) > 0 {
		s.itemBlacklist = make(map[int]string)
		for _, option := range options {
			itemName, err := common.Conf.String("item_blacklist", option)
			if err != nil {
				continue
			}
			itemId, _ := strconv.Atoi(option)
			s.itemBlacklist[itemId] = itemName
		}
		s.logger.Debug("packet item black list:%v", s.itemBlacklist)
	}

	options, err = common.Conf.Options("item_whitelist")
	if err == nil && len(options) > 0 {
		s.itemWhitelist = make(map[int]string)
		for _, option := range options {
			itemName, err := common.Conf.String("item_whitelist", option)
			if err != nil {
				continue
			}
			itemId, _ := strconv.Atoi(option)
			s.itemWhitelist[itemId] = itemName
		}
		s.logger.Debug("packet item white list:%v", s.itemWhitelist)
	}

	return s
}