func log(i int) {
logger.Debug("Debug>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Info("Info>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Warn("Warn>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Error("Error>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
logger.Fatal("Fatal>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i))
}
func main() {
logPath := getPath()
// init log
log.SetConsole(true)
log.SetRollingDaily(logPath, "network.log")
log.SetLevel(log.DEBUG)
protocol := &packet.TransProtocolComm{}
reactor := &ComReactor{}
log.Debug("protocol addr is-", protocol)
log.Debug("reactor addr is-", reactor)
tcpServer := socket.NewTcpListenSock(protocol, reactor)
log.Debug("TcpServer start!")
tcpServer.InitTcpServer("localhost:9009")
select {}
}
/*测试读取xml*/
func TestReadXml() {
content, err := ioutil.ReadFile("xml/HostAudit/repeatetimer_主机审计日志.xml")
if err != nil {
logger.Error(err)
}
var result XmlInfo
err = xml.Unmarshal(content, &result)
if err != nil {
logger.Error(err)
}
logger.Debug(result.FileName)
}
func main() {
if len(os.Args) < 3 {
fmt.Fprintf(os.Stderr, "Usage: $0 -c config_file [-t]\n")
os.Exit(1)
}
flag.StringVar(&configFile_, "c", "conf.json", "config file in json")
flag.BoolVar(&isTest, "t", false, "test mode")
flag.Parse()
// SIGHUP to reload configFile, SIGTERM/SIGINT to exit the process
sigChan := make(chan os.Signal, 10)
signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
// start worshop for the first time
if err := StartNewWorkShop(isTest); err != nil {
fmt.Fprintf(os.Stderr, "workshop init fail [%+v]\n", err)
os.Exit(1)
}
if isTest {
os.Exit(0)
}
// loop. wait for signals
MAIN_LOOP:
for {
select {
case sig := <-sigChan:
fmt.Fprintf(os.Stderr, "receive sig [%+v]\n", sig)
// logger.Error("receive sig ", sig)
if sig == syscall.SIGHUP {
// wait for workshop to do some clean-up
StopWorkShop()
ticker_.Stop()
} else if sig == syscall.SIGTERM || sig == syscall.SIGINT {
StopWorkShop()
ticker_.Stop()
break MAIN_LOOP
} else {
logger.Error("main routine receive unexpected signal [", sig, "], this should NEVER happen")
os.Exit(1)
}
case <-ticker_.C:
WSChan_ <- CMD_TICK
logger.Debug("main_routine tick")
}
}
logger.Info("the qtask application exit")
}
func (p *Workshop) Init(ctx *sj.Json, upCtrlChan chan int, upReportChan chan int, isTest bool) error {
p.ctx = ctx
p.upCtrlChan = upCtrlChan
p.upReportChan = upReportChan
p.processedLine = 0
var err error
if err = p.initLog(isTest); err != nil {
return err
}
logger.Debug("workshop init log success")
if err = p.initSrcProvider(); err != nil {
return err
}
logger.Debug("workshop init srcProvider success")
p.cronTickInterval = ctx.Get("main").Get("cron_tick_interval").MustInt()
p.memLimit = ctx.Get("main").Get("mem_limit").MustUint64()
p.PRCtrlChan = make(chan int, 1000) // 1000 is enough
p.PRReportChan = make(chan int)
p.CLCtrlChan = make(chan int, 1000) // 1000 is enough
p.CLReportChan = make(chan int)
if err = p.initChannel(isTest); err != nil {
return err
}
logger.Debug("workshop init channel and subroutine success")
logger.Info("start... config:\n[", *p.ctx, "]\n")
if isTest {
p.provider.Destory()
}
logger.SetConsole(false)
return nil
}
func (sock *TcpListenSocket) acceptConn() error {
if sock._socketType != TCPLISTEN || sock._socketState != LISTEN {
return errors.New("accepet errors")
}
log.Debug("TcpServer begin to accept!")
for {
conn, err := sock._tcpListener.AcceptTCP()
select {
case <-sock.stop:
return nil
default:
fmt.Println("nothing")
}
if err == nil {
tcpSocket := NewTcpSocket(conn, sock._tprotocol, sock._eventReactor)
tcpSocket.Start()
log.Debug("TcpServer accept a new socket")
}
}
}
func (p *Workshop) Run() {
LOOP:
for {
select {
case cmd := <-p.upCtrlChan:
if cmd == CMD_EXIT {
close(p.msgChan) // will send an empty msg to its receivers
p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, p.processorNum)
p.closeSubRoutines(p.CLCtrlChan, p.CLReportChan, p.collectorNum)
break LOOP
} else if cmd == CMD_TICK {
// logger.Debug("run_routine tick")
p.sendCtrlInfo(p.PRCtrlChan, cmd, p.processorNum)
p.sendCtrlInfo(p.CLCtrlChan, cmd, p.collectorNum)
p.cronTick++
if p.cronTick > p.cronTickInterval {
p.Tick()
p.cronTick = 0
}
} else {
logger.Error(" WorkShop receive a un-expected cmd ", cmd)
}
default:
messages, err := p.provider.GetNextMsg()
if err != nil {
logger.Error("src provider get msg err occur with ", len(messages), " msgs, err: ", err)
}
if messages == nil || len(messages) == 0 {
continue
}
for _, v := range messages {
if len(v) > 0 {
logger.Info("emit!")
p.msgChan <- string(v)
}
}
logger.Debug("main routine receive ", len(messages), " msgs from srcProvider")
p.processedLine += len(messages)
p.provider.Ack()
}
}
p.provider.Destory()
p.upReportChan <- RET_EXIT_SUCCESS
}
func (p *Workshop) CollectRoutine(id int) {
collectorInfos := p.ctx.Get("collector").MustMap() // eg: Midd => MiddCollector. Midd is the chan type that the collector will receive
logger.Debug(id, " collectorInfos [", collectorInfos, " ] collectors")
collectorMap := make(map[string]job.Collector)
for category, name := range collectorInfos {
collectorName, ok := name.(string)
if !ok {
logger.Error("collector setting invalid: [", category, "] => [", name, "]")
p.CLReportChan <- RET_INIT_FAIL
return
}
collectorMap[category] = job.NewCollector(collectorName)
if collectorMap[category] == nil {
logger.Error("collector [", collectorName, "] create failed")
p.CLReportChan <- RET_INIT_FAIL
return
}
if err := collectorMap[category].Init(p.ctx, id); err != nil {
logger.Error("collector [", collectorName, "] init failed, ", err)
delete(collectorMap, category)
p.CLReportChan <- RET_INIT_FAIL
return
}
}
p.CLReportChan <- RET_INIT_SUCCESS
logger.Info(id, " th CollectRoutine successful start with ", len(collectorMap), " collectors")
channel := p.itemChans[id]
exitFlag := false
for {
select {
case cmd := <-p.CLCtrlChan:
if cmd == CMD_EXIT {
if len(channel) > 0 {
exitFlag = true
continue
}
goto EXIT_PROCESS
} else if cmd == CMD_TICK {
//logger.Debug("collector_routine tick")
for _, collector := range collectorMap {
collector.Tick()
}
} else {
logger.Error(id, " th CollectRoutine receive a un-expected cmd ", cmd)
}
case processed_item := <-channel:
category := processed_item.Category
if collector, ok := collectorMap[category]; ok {
if err := collector.Collect(processed_item); err != nil {
logger.Error(id, " th CollectRoutine, ", category, " collect fail, [", err, "]")
} else {
//logger.Debug(id, " th CollectRoutine, ", category, " collect success")
}
} else {
logger.Error("data [", category, "] has no proper collector [", collectorMap, "]")
}
if exitFlag && len(channel) <= 0 {
goto EXIT_PROCESS
}
} // select
} // for
EXIT_PROCESS:
for _, collector := range collectorMap {
collector.Destory()
}
p.CLReportChan <- RET_EXIT_SUCCESS
}
func (p *Workshop) ProcessRoutine(id int) {
processorFlags := p.ctx.Get("processor").MustMap() // eg: UrlProcessor => true
logger.Debug(id, " processorInfos [", processorFlags, " ] processors")
processorMap := make(map[string]job.Processor)
for name, flag := range processorFlags {
enable, ok := flag.(bool)
if !ok {
logger.Error("processor [", name, "] setting invalid ", flag)
p.PRReportChan <- RET_INIT_FAIL
return
}
if !enable {
continue
}
processorMap[name] = job.NewProcessor(name)
if processorMap[name] == nil {
logger.Error("processor [", name, "] create failed")
p.PRReportChan <- RET_INIT_FAIL
return
}
if err := processorMap[name].Init(p.ctx, id, p.itemChans); err != nil {
logger.Error("processor [", name, "] init failed, ", err)
delete(processorMap, name)
p.PRReportChan <- RET_INIT_FAIL
return
}
}
if len(processorMap) <= 0 {
logger.Error("no valid processor")
p.PRReportChan <- RET_INIT_FAIL
return
}
p.PRReportChan <- RET_INIT_SUCCESS
logger.Info(id, " th ProcessRoutine successful start with ", len(processorMap), " processors")
exitFlag := false
for {
select {
case cmd := <-p.PRCtrlChan:
if cmd == CMD_EXIT {
if len(p.msgChan) > 0 {
exitFlag = true
continue
}
goto EXIT_PROCESS
} else if cmd == CMD_TICK {
for _, proc := range processorMap {
proc.Tick()
}
} else {
logger.Error(id, " th CollectRoutine receive a un-expected cmd ", cmd)
}
case msg := <-p.msgChan:
for name, proc := range processorMap {
if err := proc.Process(msg); err != nil {
logger.Error(id, " th ProcessRoutine, ", name, " process fail, [", err, "]")
} else {
//logger.Debug(id, " th ProcessRoutine, ", name, " process success")
}
}
if exitFlag && len(p.msgChan) <= 0 {
goto EXIT_PROCESS
}
} // select
} // for
EXIT_PROCESS:
for _, proc := range processorMap {
proc.Destory()
}
p.PRReportChan <- RET_EXIT_SUCCESS
}
func (p *Workshop) initChannel(isTest bool) error {
// create channels
p.processorNum = p.ctx.Get("main").Get("processor_num").MustInt()
p.collectorNum = p.ctx.Get("main").Get("collector_num").MustInt()
msgChanSize := p.ctx.Get("main").Get("msg_chan_size").MustInt()
itemChanSize := p.ctx.Get("main").Get("item_chan_size").MustInt()
p.msgChan = make(chan string, msgChanSize)
p.itemChans = make([](chan job.Item), p.collectorNum)
for i := 0; i < p.collectorNum; i++ {
p.itemChans[i] = make(chan job.Item, itemChanSize)
}
// start processor goroutines
processorSuccessNum := 0
for i := 0; i < p.processorNum; i++ {
go p.ProcessRoutine(i)
}
i := 0
for i < p.processorNum {
select {
case isSuccess := <-p.PRReportChan:
i++
if isSuccess == RET_INIT_SUCCESS {
processorSuccessNum++
}
}
}
if processorSuccessNum < p.processorNum {
p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, processorSuccessNum)
logger.Debug("workshop start ", p.processorNum, " processor, success: ", processorSuccessNum)
return errors.New("some processor init fail")
}
logger.Info("workshop start ", processorSuccessNum, " processor, all success")
// start collector goroutines
collectorSuccessNum := 0
for i := 0; i < p.collectorNum; i++ {
go p.CollectRoutine(i)
}
j := 0
for j < p.collectorNum {
select {
case isSuccess := <-p.CLReportChan:
j++
if isSuccess == RET_INIT_SUCCESS {
collectorSuccessNum++
}
}
}
if collectorSuccessNum < p.collectorNum {
p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, processorSuccessNum)
p.closeSubRoutines(p.CLCtrlChan, p.CLReportChan, collectorSuccessNum)
logger.Debug("workshop start ", p.collectorNum, " colletor, success: ", collectorSuccessNum)
return errors.New("some colletor init fail")
}
logger.Info("workshop start ", collectorSuccessNum, " collector, all success")
if isTest {
p.closeSubRoutines(p.PRCtrlChan, p.PRReportChan, p.processorNum)
p.closeSubRoutines(p.CLCtrlChan, p.CLReportChan, p.collectorNum)
}
return nil
}
