func main() {
if err := initialize(); err != nil {
panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
}
//start server...
server := network.NewServer()
if server == nil {
panic("new tcp server failed.")
}
go server.Start()
//start rule updater
updater := task.NewUpdater()
if updater == nil {
panic("new updater failed.")
}
go updater.Start()
//start worker
filters := make([]*task.Filter, common.WorkerNum)
loggerf := common.NewLogger("filter")
backends := make([]*network.Backend, common.WorkerNum)
loggerb := common.NewLogger("backend")
for i := 0; i < common.WorkerNum; i++ {
filters[i] = task.NewFilter(i, common.PacketChans[i], common.FilterChans[i], loggerf)
if filters[i] == nil {
panic("new filter failed.")
}
go filters[i].Start()
backends[i] = network.NewBackend(i, common.FilterChans[i], loggerb)
if backends[i] == nil {
panic("new backend failed.")
}
go backends[i].Start()
}
fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
//wait for signal
sig_chan := make(chan os.Signal)
signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
<-sig_chan
//clear works...
server.Stop()
updater.Stop()
for i := 0; i < common.WorkerNum; i++ {
filters[i].Stop()
backends[i].Stop()
}
common.Alarm(VERSION + "-exit")
time.Sleep(time.Second)
fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
func NewServer() *Server {
s := &Server{
port: common.GetConfInt("server", "port", 8080),
maxClients: common.GetConfInt("server", "max_clients", 100000),
clientNum: 0,
acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
connTimeout: common.GetConfSecond("server", "connection_timeout", 60*3),
headerLen: common.GetConfInt("server", "header_length", 10),
maxBodyLen: common.GetConfInt("server", "max_body_length", 102400),
running: false,
heartbeat: time.Now().Unix(),
workerNum: common.WorkerNum,
currentWorker: -1,
logger: common.NewLogger("server"),
}
if s == nil {
fmt.Println("new server failed")
return nil
}
if s.logger == nil {
fmt.Println("new server logger failed")
return nil
}
ip, err := common.Conf.String("server", "bind")
if err != nil {
ip = ""
}
s.ip = ip
return s
}
func NewUpdater() *Updater {
u := &Updater{
running: false,
heartbeat: time.Now().Unix(),
interval: common.GetConfSecond("updater", "interval", 20*60),
logger: common.NewLogger("updater"),
}
if u == nil {
fmt.Println("new updater failed")
return nil
}
redisAddr, err := common.Conf.String("updater", "redis_addr")
if err != nil {
fmt.Printf("updater get redis_addr failed, err:%s\n", err.Error())
return nil
}
u.redisCli.Addr = redisAddr
u.redisCli.Db = common.GetConfInt("updater", "redis_db", 0)
if u.logger == nil {
fmt.Println("updater new logger failed")
return nil
}
return u
}
func NewServer() *Server {
s := &Server{
port: common.GetConfInt("server", "port", 8080),
maxClients: common.GetConfInt("server", "max_clients", 100000),
clientNum: 0,
acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
connTimeout: common.GetConfSecond("server", "connection_timeout", 60*3),
headerLen: common.GetConfInt("server", "header_length", 10),
maxBodyLen: common.GetConfInt("server", "max_body_length", 102400),
running: false,
heartbeat: time.Now().Unix(),
workerNum: common.WorkerNum,
currentWorker: -1,
slowread: common.GetConfSecond("server", "slow_read", 0),
logger: common.NewLogger("server"),
}
if s == nil {
fmt.Println("new server failed")
return nil
}
if s.logger == nil {
fmt.Println("new server logger failed")
return nil
}
ip, err := common.Conf.String("server", "bind")
if err != nil {
ip = ""
}
s.ip = ip
max_clients := uint64(s.maxClients)
if max_clients > 1024 {
var rlim syscall.Rlimit
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("server get rlimit error: " + err.Error())
return nil
}
rlim.Cur = max_clients
rlim.Max = max_clients
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim)
if err != nil {
fmt.Println("server set rlimit error: " + err.Error())
return nil
}
s.logger.Info("set fd limit to %d", s.maxClients)
}
return s
}
func main() {
if err := initialize(); err != nil {
panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
}
//start server...
server := network.NewServer()
if server == nil {
panic("new tcp server failed.")
}
go server.Start()
//start worker
savers := make([]*task.KafkaSaver, common.WorkerNum)
logger := common.NewLogger("kafka")
for i := 0; i < common.WorkerNum; i++ {
savers[i] = task.NewKafkaSaver(i, common.PacketChans[i], logger)
if savers[i] == nil {
panic("new filter failed.")
}
go savers[i].Start()
}
fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
//wait for signal
sig_chan := make(chan os.Signal)
signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
<-sig_chan
//clear works...
server.Stop()
for i := 0; i < common.WorkerNum; i++ {
savers[i].Stop()
}
common.Alarm(VERSION + "-exit")
time.Sleep(time.Second)
fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
func NewBackend(idx int, inputChan chan []byte) *Backend {
b := &Backend{
index: idx,
backendNum: 0,
heartbeat: time.Now().Unix(),
logger: common.NewLogger("backend"),
inChan: inputChan,
running: false,
}
if b == nil {
fmt.Println("new backend failed")
return nil
}
if b.logger == nil {
fmt.Println("backend new logger failed")
return nil
}
if b.inChan == nil {
fmt.Println("backend must has an input channel to get bytes for back point")
return nil
}
options, err := common.Conf.Options("backend")
if err != nil {
fmt.Printf("backend get all options failed:%s\n", err.Error())
return nil
}
b.backendList = make([]*Backpoint, 0, 4)
for _, option := range options {
if !strings.HasPrefix(option, "backend_list_") {
continue
}
back, err := common.Conf.String("backend", option)
if err != nil {
fmt.Printf("Read conf %s failed, error: %s, get backend total number:%d\n", option, err.Error(), b.backendNum)
break
}
backend_name := strings.TrimPrefix(option, "backend_list_")
if backend_name == "" || backend_name == option {
fmt.Printf("get backend name failed")
break
}
addrs := strings.Split(back, ";")
num := len(addrs)
if num < 1 {
fmt.Printf("one backend:%s must at least has one address", backend_name)
continue
}
point := &Backpoint{
name: backend_name,
backNum: num,
backAddrs: addrs,
conn: nil,
connTimeout: common.GetConfSecond("backend", "connection_timeout", 180),
sendTimeout: common.GetConfSecond("backend", "send_timeout", 180),
currentBackIdx: -1,
retryTimes: common.GetConfInt("backend", "retry_times", 5),
retryInterval: common.GetConfSecond("backend", "retry_interval", 5),
recvBuf: make([]byte, common.GetConfInt("backend", "receive_buffer_size", 10)),
sendingChan: make(chan []byte, common.GetConfInt("backend", "sending_buffer_size", 10000)),
}
if point == nil {
fmt.Println("new back point failed")
return nil
}
b.backendList = append(b.backendList, point)
b.backendNum += 1
b.logger.Debug("Backend%d get a back %s: %s, %d points", idx, backend_name, back, num)
}
if b.backendNum < 1 {
fmt.Println("no backends")
return nil
}
return b
}
func NewServer() *Server {
s := &Server{
port: common.GetConfInt("server", "port", 8080),
maxClients: common.GetConfInt("server", "max_clients", 100000),
headerLen: common.GetConfInt("server", "header_length", 10),
maxBodyLen: common.GetConfInt("server", "max_body_length", 102400),
acceptTimeout: common.GetConfSecond("server", "accept_timeout", 60*5),
connTimeout: common.GetConfSecond("server", "connection_timeout", 60*3),
clientNum: 0,
currentWorker: -1,
running: false,
heartbeat: time.Now().Unix(),
workerNum: common.WorkerNum,
logger: common.NewLogger("server"),
ipWhitelist: nil,
itemWhitelist: nil,
itemBlacklist: nil,
}
if s == nil {
fmt.Println("new server failed")
return nil
}
if s.logger == nil {
fmt.Println("new server logger failed")
return nil
}
ip, err := common.Conf.String("server", "bind")
if err != nil {
//not set, bind all interface
ip = ""
}
s.ip = ip
white_list, err := common.Conf.String("server", "ip_whitelist")
if err == nil {
s.ipWhitelist = make(map[string]bool)
for _, ip := range strings.Split(white_list, ";") {
s.ipWhitelist[ip] = true
}
}
s.logger.Debug("connection ip white list:%v", s.ipWhitelist)
options, err := common.Conf.Options("item_blacklist")
if err == nil && len(options) > 0 {
s.itemBlacklist = make(map[int]string)
for _, option := range options {
itemName, err := common.Conf.String("item_blacklist", option)
if err != nil {
continue
}
itemId, _ := strconv.Atoi(option)
s.itemBlacklist[itemId] = itemName
}
s.logger.Debug("packet item black list:%v", s.itemBlacklist)
}
options, err = common.Conf.Options("item_whitelist")
if err == nil && len(options) > 0 {
s.itemWhitelist = make(map[int]string)
for _, option := range options {
itemName, err := common.Conf.String("item_whitelist", option)
if err != nil {
continue
}
itemId, _ := strconv.Atoi(option)
s.itemWhitelist[itemId] = itemName
}
s.logger.Debug("packet item white list:%v", s.itemWhitelist)
}
return s
}