Esempio n. 1
0
func (this *Backpoint) connect() error {
	if this.currentBackIdx < 0 || this.currentBackIdx >= this.backNum {
		this.currentBackIdx = time.Now().Nanosecond() % this.backNum
	}

	var addr string
	var conn net.Conn
	var err error

	for i := 0; i < this.retryTimes; i++ {
		this.currentBackIdx = (this.currentBackIdx + 1) % this.backNum
		addr = this.backAddrs[this.currentBackIdx]
		conn, err = net.DialTimeout("tcp", addr, this.connTimeout)
		if err == nil {
			this.conn = conn
			return nil
		}

		time.Sleep(this.retryInterval)
	}

	this.conn = nil
	common.Alarm(this.name + "-down")
	return err
}
Esempio n. 2
0
func (this *Backend) Start() {
	this.running = true
	this.logger.Info("Backend%d start to work", this.index)

	var bytes []byte
	var ok bool
	var point *Backpoint

	for _, point = range this.backendList {
		go this.sending(point)
	}

	for this.running {
		this.heartbeat = time.Now().Unix()

		bytes, ok = <-this.inChan
		if !ok {
			this.logger.Error("Backend%d read from filter channel failed", this.index)
			break
		}
		this.logger.Debug("Backend%d get data from input channel success Length:%d, Content: %v", this.index, len(bytes), bytes)

		for _, point = range this.backendList {
			select {
			case point.sendingChan <- bytes:
			default:
				this.logger.Error("Backend%d insert into backpoint %s sending channel failed, length:%d too more, data dropped:%s", this.index, point.name, len(point.sendingChan), string(bytes))
				common.Alarm(fmt.Sprintf("backend%d_%s_sendingchan_block", this.index, point.name))
			}
		}

		this.logger.Debug("Backend%d dispatch out data to all back %d points success, remaining %d packets in packet channel, %s", this.index, this.backendNum, len(this.inChan), string(bytes))
	}
}
Esempio n. 3
0
func main() {
	if err := initialize(); err != nil {
		panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
	}

	//start server...
	server := network.NewServer()
	if server == nil {
		panic("new tcp server failed.")
	}
	go server.Start()

	//start rule updater
	updater := task.NewUpdater()
	if updater == nil {
		panic("new updater failed.")
	}
	go updater.Start()

	//start worker
	filters := make([]*task.Filter, common.WorkerNum)
	loggerf := common.NewLogger("filter")
	backends := make([]*network.Backend, common.WorkerNum)
	loggerb := common.NewLogger("backend")
	for i := 0; i < common.WorkerNum; i++ {
		filters[i] = task.NewFilter(i, common.PacketChans[i], common.FilterChans[i], loggerf)
		if filters[i] == nil {
			panic("new filter failed.")
		}
		go filters[i].Start()

		backends[i] = network.NewBackend(i, common.FilterChans[i], loggerb)
		if backends[i] == nil {
			panic("new backend failed.")
		}
		go backends[i].Start()
	}

	fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
	//wait for signal
	sig_chan := make(chan os.Signal)
	signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
	<-sig_chan

	//clear works...
	server.Stop()
	updater.Stop()
	for i := 0; i < common.WorkerNum; i++ {
		filters[i].Stop()
		backends[i].Stop()
	}

	common.Alarm(VERSION + "-exit")
	time.Sleep(time.Second)
	fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
Esempio n. 4
0
func (this *Server) Start() {
	addr := fmt.Sprintf("%s:%d", this.ip, this.port)
	tcpAddr, err := net.ResolveTCPAddr("tcp4", addr)
	if err != nil {
		this.logger.Error("tcp server resolve address %s failed:%s", addr, err.Error())
		return
	}
	//autoset: reuseaddr nodelay nonblock
	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		this.logger.Error("tcp server listen on %s failed:%s", addr, err.Error())
		return
	}
	defer listener.Close()

	this.running = true
	this.currentWorker = time.Now().Nanosecond() % this.workerNum
	this.logger.Info("tcp server start to listen on %s", addr)

	for this.running {
		this.heartbeat = time.Now().Unix()

		listener.SetDeadline(time.Now().Add(this.acceptTimeout))
		conn, err := listener.Accept()
		if err != nil {
			this.logger.Error("Tcp server accept connection:%v failed:%s", conn, err.Error())
			this.logger.Info("Server master cron total connections number: %d", this.clientNum)
			for i := 0; i < this.workerNum; i++ {
				this.logger.Info("Server master cron output channel%d length: %d", i, len(common.PacketChans[i]))
			}
			if conn != nil {
				conn.Close()
				time.Sleep(time.Second)
			}
			continue
		}

		if this.clientNum >= this.maxClients-1 {
			this.logger.Error("Client number now is %d, exceed maxClients:%d, connection close, remote addr:%s", this.clientNum+1, this.maxClients, conn.RemoteAddr().String())
			conn.Close()
			this.running = false
			common.Alarm("fd_full_auto_restart")
			time.Sleep(time.Second * 3)

			os.Exit(1)
		}

		this.lock.Lock()
		this.clientNum += 1
		this.lock.Unlock()

		go this.process(conn)
	}

}
Esempio n. 5
0
func (this *Filter) output(bytes []byte, outChan chan []byte) error {
	select {
	case outChan <- bytes:
		return nil
	default:
		this.logger.Error("Filter%d insert into filter channel timeout, channel length:%d too more, drop packet bytes: %s", this.index, len(outChan), string(bytes))
		common.Alarm("filter_insert_channel_block")
		return fmt.Errorf("channel timeout")
	}
	return nil
}
Esempio n. 6
0
func initialize() error {
	fmt.Println("initilize ...")

	confFile := flag.String("c", "./etc/transfer.conf", "config file name")
	if confFile == nil {
		return fmt.Errorf("no config file")
	}
	flag.Parse()

	fmt.Printf("load configuration:%s\n", *confFile)
	conf, err := config.ReadDefault(*confFile)
	if err != nil {
		return fmt.Errorf("load config file failed:%s\n", err.Error())
	}
	common.Conf = conf

	max_processor := common.GetConfInt("global", "max_processor", runtime.NumCPU())
	runtime.GOMAXPROCS(max_processor)

	dir, err := common.Conf.String("global", "root_dir")
	if err == nil {
		//workding directory is resigned, change to that...
		err = os.Chdir(dir)
		if err != nil {
			return fmt.Errorf("change working directory to %s failed:%s\n", dir, err.Error())
		}
	}
	common.Dir, _ = os.Getwd()

	num := common.GetConfInt("global", "worker_num", 1)
	if num < 1 {
		return fmt.Errorf("work number must be larger than 1")
	}

	common.WorkerNum = num
	common.PacketChans = make([]chan []byte, num)
	for i := 0; i < num; i++ {
		common.PacketChans[i] = make(chan []byte, common.GetConfInt("server", "packet_chan_size", 10000))
		if common.PacketChans[i] == nil {
			return fmt.Errorf("make packet channel failed")
		}
	}

	fmt.Println("initilize over")
	fmt.Printf("Program %s start success in %s at: %s, Max processor:%d Worker number:%d\n", VERSION, common.Dir, time.Now(), max_processor, num)
	common.Alarm(VERSION + "-start")

	return nil
}
Esempio n. 7
0
func main() {
	if err := initialize(); err != nil {
		panic(fmt.Sprintf("initialize failed:%s\n", err.Error()))
	}

	//start server...
	server := network.NewServer()
	if server == nil {
		panic("new tcp server failed.")
	}
	go server.Start()

	//start worker
	savers := make([]*task.KafkaSaver, common.WorkerNum)
	logger := common.NewLogger("kafka")
	for i := 0; i < common.WorkerNum; i++ {
		savers[i] = task.NewKafkaSaver(i, common.PacketChans[i], logger)
		if savers[i] == nil {
			panic("new filter failed.")
		}
		go savers[i].Start()
	}

	fmt.Printf("Program %s start success, wait for quit signal...\n", VERSION)
	//wait for signal
	sig_chan := make(chan os.Signal)
	signal.Notify(sig_chan, os.Interrupt, syscall.SIGTERM)
	<-sig_chan

	//clear works...
	server.Stop()
	for i := 0; i < common.WorkerNum; i++ {
		savers[i].Stop()
	}

	common.Alarm(VERSION + "-exit")
	time.Sleep(time.Second)
	fmt.Printf("Program %s quit success at: %s\n", VERSION, time.Now())
}
Esempio n. 8
0
func (this *Server) output(data []byte, outChan chan *common.Packet) error {
	var err error

	m := make(map[string]interface{})
	err = json.Unmarshal(data, &m)
	if err != nil {
		return err
	}

	host, exists := m["host"]
	if !exists {
		return fmt.Errorf("host not exists")
	}
	hostname, ok := host.(string)
	if !ok {
		return fmt.Errorf("host is not a string")
	}

	//time, exists := m["time"]
	//if !exists {
	//      fmt.Printf("json unmarshal from %s failed, time not exists", s)
	//      return nil
	//}

	datas, exists := m["data"]
	if !exists {
		return fmt.Errorf("data not exists")
	}
	items, ok := datas.([]interface{})
	if !ok {
		return fmt.Errorf("data is not []interface")
	}
	length := len(items)
	if length < 1 {
		return fmt.Errorf("data item length < 1")
	}

	for i := 0; i < length; i++ {
		item, ok := items[i].([]interface{})
		if !ok {
			return fmt.Errorf("data item is not []interface{}")
		}
		item_id, ok := item[0].(string)
		if !ok {
			return fmt.Errorf("item id is not a string")
		}
		value, ok := item[1].(string)
		if !ok {
			return fmt.Errorf("value is not a string")
		}
		itemId, _ := strconv.Atoi(item_id)
		val, _ := strconv.ParseFloat(value, 64)

		//insert into out channel
		select {
		case outChan <- common.NewPacket(hostname, itemId, val):
			this.logger.Debug("Insert into packet channel success, hostname:%s, item_id:%s, value:%s", hostname, item_id, value)
		default:
			this.logger.Error("Insert into packet channel timeout, channel length:%d too more, drop packet: hostname:%s, item id:%s, value:%s", len(outChan), hostname, item_id, value)
			common.Alarm("packet_channel_timeout")
			time.Sleep(time.Second)
			return fmt.Errorf("channel timeout")
		}
	}

	this.logger.Info("Put %d alarm packets into packet channel", length)
	return nil
}
Esempio n. 9
0
func (this *Server) output(data []byte, outChan chan *common.Packet) error {
	var err error

	m := make(map[string]interface{})
	err = json.Unmarshal(data, &m)
	if err != nil {
		return err
	}

	host, exists := m["host"]
	if !exists {
		return fmt.Errorf("host not exists")
	}
	hostname, ok := host.(string)
	if !ok {
		return fmt.Errorf("host is not a string, type:%v, host:%v", reflect.TypeOf(host), host)
	}

	//agenttime localtime
	agenttime, exists := m["time"]
	if !exists {
		return fmt.Errorf("time not exists")
	}
	agenttime_s, ok := agenttime.(string)
	if !ok {
		return fmt.Errorf("time is not a string, type:%v, time:%v", reflect.TypeOf(agenttime), agenttime)
	}
	agent_time, _ := strconv.ParseInt(agenttime_s, 10, 64)

	datas, exists := m["data"]
	if !exists {
		return fmt.Errorf("data not exists")
	}
	items, ok := datas.([]interface{})
	if !ok {
		return fmt.Errorf("data is not array: []interface, type:%v, data:%v", reflect.TypeOf(datas), datas)
	}
	length := len(items)
	if length < 1 {
		return fmt.Errorf("data items length < 1")
	}

	p := common.NewPacket(hostname, agent_time)
	var itemName string
	for i := 0; i < length; i++ {
		//item:  [item_id, value, val_type]
		item, ok := items[i].([]interface{})
		if !ok {
			return fmt.Errorf("data item is not array []interface, type:%v, item:%v", reflect.TypeOf(items[i]), items[i])
		}
		if len(item) < 3 {
			return fmt.Errorf("data item length < 3, item:%v", item)
		}
		item_id, ok := item[0].(string)
		if !ok {
			return fmt.Errorf("item id is not string, type:%v, id:%v", reflect.TypeOf(item[0]), item[0])
		}
		itemId, _ := strconv.Atoi(item_id)
		value, ok := item[1].(string)
		if !ok {
			return fmt.Errorf("value is not string, value:%v", item[1])
		}

		if this.itemBlacklist != nil {
			if _, ok = this.itemBlacklist[itemId]; ok {
				this.logger.Debug("item id:%d is in item blacklist, drop", itemId)
				continue
			}
		}

		if this.itemWhitelist != nil {
			if itemName, ok = this.itemWhitelist[itemId]; !ok {
				this.logger.Debug("item id:%d not in item whitelist, drop", itemId)
				continue
			}
		}

		p.AddItem(itemId, value, itemName)
	}

	if p.ItemNum < 1 {
		this.logger.Debug("packet item num < 1, hostname:%s, time:%s", hostname, agenttime_s)
		return nil
	}

	//insert into output channel
	select {
	case outChan <- p:
		this.logger.Debug("Insert into packet channel success: Hostname:%s, time:%s", hostname, agenttime_s)
	default:
		this.logger.Error("Insert into packet channel failed, channel length:%d too more, drop packet: %v", len(outChan), p)
		common.Alarm("packet_channel_block")
		time.Sleep(time.Second)
		return fmt.Errorf("packet channel block")
	}

	this.logger.Info("Put %d packet items into packet channel", p.ItemNum)

	return nil
}
Esempio n. 10
0
func (this *Server) process(conn net.Conn) {
	if conn == nil {
		this.logger.Error("input conn param is nil")
		return
	}
	defer conn.Close()

	headerNum := this.headerLen
	buf := make([]byte, headerNum+this.maxBodyLen)

	var (
		num  int
		data []byte
		err  error
		echo []byte = []byte("ok")
	)

	this.currentWorker = (this.currentWorker + 1) % this.workerNum
	worker := this.currentWorker
	outChan := common.PacketChans[worker]

	addr := conn.RemoteAddr().String()
	this.logger.Info("Accept connection from %s, total clients number now:%d", addr, this.clientNum)

	for this.running {
		if this.slowread > 0 {
			time.Sleep(this.slowread)
		}

		//header
		data, err = this.receive(buf[0:headerNum], conn)
		if err != nil {
			this.logger.Error("Read header from %s failed, error:%s", addr, err.Error())
			break
		}

		num, err = strconv.Atoi(string(data))
		if err != nil || num <= 0 || num > this.maxBodyLen {
			this.logger.Error("Read header from %s format error, header content: %s", addr, string(data))
			break
		}

		//body
		num += headerNum
		data, err = this.receive(buf[headerNum:num], conn)
		if err != nil {
			this.logger.Error("Read body from %s failed, error:%s", addr, err.Error())
			break
		}

		//read success
		this.logger.Info("Read from %s, Length: %d, Content: %s", addr, num, string(data))

		// ack ok back
		err = this.send(echo, conn)
		if err != nil {
			this.logger.Error("Echo back %s to %s failed, error:%s", string(echo), addr, err.Error())
			break
		}

		data = make([]byte, num)
		copy(data, buf[0:num])
		select {
		case outChan <- data:
			this.logger.Debug("Insert into output channel success, bytes:%s", string(data))
		default:
			this.logger.Error("Insert into output channel failed, channel length:%d too more, drop packet: %s", len(outChan), string(data))
			common.Alarm(fmt.Sprintf("packet_channel%d_full", worker))
			time.Sleep(time.Second)
			break
		}
	}

	this.lock.Lock()
	this.clientNum -= 1
	this.lock.Unlock()

	this.logger.Info("connection %s closed, remaining total clients number: %d", addr, this.clientNum)
}