func ConsumerGO(topic, channel string, goCount uint, handle Handler) (*ConsumerT, error) {
msgHandle := &ConsumerT{
consumers: []*nsq.Consumer{},
handle: handle,
msgCh: make(chan *Message, MSG_CHAN_SIZE),
waitGroup: sync_.NewWaitGroup(),
}
addrSlice := strings.Split(g_nsqdMgr.addrs, ",")
for _, addr := range addrSlice {
for i := 0; i < CONSUMER_COUNT_PER_ADDR; i++ {
consumer, err := nsq.NewConsumer(topic, channel, nsq.NewConfig())
if err != nil {
return nil, errors.As(err, topic, channel)
}
consumer.SetLogger(nil, nsq.LogLevelInfo)
consumer.AddHandler(msgHandle)
if err := consumer.ConnectToNSQD(addr); err != nil {
return nil, errors.As(err, topic, channel, g_nsqdMgr.addrs)
}
msgHandle.consumers = append(msgHandle.consumers, consumer)
}
}
g_nsqdMgr.consumers = msgHandle
for i := 0; i < int(goCount); i++ {
go msgHandle.work()
}
return msgHandle, nil
}
func Init(addrs string) {
config := nsq.NewConfig()
config.DefaultRequeueDelay = 0
addrSlice := strings.Split(addrs, ",")
produceCount := len(addrSlice) * PRODUCER_COUNT_PER_ADDR
producers := make([]*nsq.Producer, produceCount)
var err error = nil
var idx int = 0
for _, addr := range addrSlice {
for i := 0; i < PRODUCER_COUNT_PER_ADDR; i++ {
producers[idx], err = nsq.NewProducer(addr, config)
if err != nil {
panic(err)
}
idx++
}
}
g_nsqdMgr = &nsqdMgr{
addrs: addrs,
waitGroup: sync_.NewWaitGroup(),
producers: producers,
msgCh: make(chan nsqdMsg, MSG_CHAN_SIZE),
consumers: nil,
}
for i := 0; i < produceCount; i++ {
go g_nsqdMgr.producer(i)
}
return
}
func (l *fileLogMgr) Init(config *Config) error {
if config.FileLog == false {
return errors.New("file log server not configure!!!")
}
date, err := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
if err != nil {
fmt.Println("time.Parse failed:", err)
return err
}
dir := config.Directory
if len(dir) > 0 && dir[len(dir)-1] != '/' {
dir += "/"
}
file, err := os.OpenFile(dir+config.FileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
fmt.Println("os.OpenFile failed", err)
return err
}
l.waitGroup = sync_.NewWaitGroup()
l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
l.logLevel = config.FileLogLevel
l.filename = config.FileName
l.date = date
l.directory = dir
l.file = file
l.init = true
go l.run()
return nil
}
func (l *nsqdSvrLogMgr) Init(config *Config) error {
// 一个模块只创建一个实例
if config.NsqdSvrLog == false {
return errors.New("nsqd_server log server not configure!!!")
}
// 注册日志驱动
l.logLevel = config.LogLevel
l.logger = make(map[string]Logger)
l.waitGroup = sync_.NewWaitGroup()
nsqdCfg := config.NsqdSvrConfig
if nsqdCfg.ConsoleLog {
l.logger[CONSOLE] = &consoleLogMgr{}
}
if nsqdCfg.FileLog {
l.logger[FILE] = &fileLogMgr{}
}
if nsqdCfg.NsqdLog {
l.logger[NSQD_CLIENT] = &nsqdLogMgr{}
}
if nsqdCfg.NsqdSvrLog {
l.logger[NSQD_SERVER] = &nsqdSvrLogMgr{}
}
if nsqdCfg.MysqlLog {
l.logger[MYSQL] = &MysqlLogMgr{}
}
if nsqdCfg.MongoLog {
l.logger[MONGO] = &MongoLogMgr{}
}
// 初始化日志驱动
for logName, logger := range l.logger {
fmt.Printf("init log driver...[%s] [%s]\n", logName, nsqdCfg.ModelName)
if err := logger.Init(nsqdCfg); err != nil {
fmt.Printf("init log driver failed!!![%s] [%s]; error:%s\n", logName, nsqdCfg.ModelName, err.Error())
continue
}
fmt.Printf("init log driver success...[%s] [%s]\n", logName, nsqdCfg.ModelName)
}
l.consumer = []*nsq.Consumer{}
addrs := strings.Split(config.NsqdAddrs, ",")
for _, addr := range addrs {
for i := 0; i < CONSUME_COUNT_PER_ADDR; i++ {
consumer, err := nsq.NewConsumer(config.NsqdTopic, config.NsqdChannel, nsq.NewConfig())
if err != nil {
fmt.Println("NewConsumer failed:", err)
return err
}
consumer.SetLogger(nil, nsq.LogLevelInfo)
consumer.AddHandler(l)
if err := consumer.ConnectToNSQD(addr); err != nil {
fmt.Println("ConnectToNSQDs failed, error:", err)
return err
}
l.consumer = append(l.consumer, consumer)
}
}
l.init = true
return nil
}
func (l *consoleLogMgr) Init(config *Config) error {
if config.ConsoleLog == false {
return errors.New("console log server not configure!!!")
}
l.init = true
l.waitGroup = sync_.NewWaitGroup()
l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
l.logLevel = config.ConsoleLogLevel
go l.run()
return nil
}
func (l *MongoLogMgr) Init(config *Config) error {
if config.MongoLog == false {
return errors.New("mongo log server not configure!!!")
}
session, err := mgo.Dial(config.MongoAddr)
if err != nil {
panic(err)
}
session.SetMode(mgo.Monotonic, true)
l.init = true
l.waitGroup = sync_.NewWaitGroup()
l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
l.logLevel = config.MongoLogLevel
l.session = session
go l.run()
return nil
}
func QiniuInit() error {
c := common.MongoCollection(FILE_DB, FILE_QINIU_FILE_TABLE)
index := mgo.Index{
Key: []string{"filename"},
Unique: true,
DropDups: true,
}
g_file_mgr = &FileMgr{
fileCh: make(chan *File, 100),
expiredTimer: time.Tick(time.Hour),
waitGroup: sync.NewWaitGroup(),
rsClient: rs.New(&digest.Mac{AccessKey: g_access_key, SecretKey: []byte(g_secret_key)}),
}
go g_file_mgr.run()
return errors.As(c.EnsureIndex(index))
}
func (l *nsqdLogMgr) Init(config *Config) error {
if config.NsqdLog == false {
return errors.New("nsqd log server not configure!!!")
}
nsqConfig := nsq.NewConfig()
nsqConfig.DefaultRequeueDelay = 0
connCountPerAddr := 2
addrSlice := strings.Split(config.NsqdAddrs, ",")
producerCount := len(addrSlice) * connCountPerAddr
producers := make([]*nsq.Producer, producerCount)
var err error
idx := 0
for _, addr := range addrSlice {
for i := 0; i < connCountPerAddr; i++ {
producers[idx], err = nsq.NewProducer(addr, nsqConfig)
if err != nil {
fmt.Println("NewProducer ", addr, " error:", err)
}
//fmt.Println("nsq_client connect success:", i)
idx++
}
}
l.waitGroup = sync_.NewWaitGroup()
l.producers = producers
l.producerCount = uint(producerCount)
l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
l.logTopic = config.NsqdTopic
l.logLevel = config.NsqdLogLevel
l.init = true
for i := 0; i < producerCount; i++ {
go l.run(i)
}
return nil
}
func (l *MysqlLogMgr) Init(config *Config) error {
if config.MysqlLog == false {
return errors.New("mysql log server not configure!!!")
}
settings := db.DataSource{
Host: config.Host,
Port: config.Port,
Database: config.Database,
User: config.User,
Password: config.Password,
Charset: config.Charset,
}
session, err := db.Open("mysql", settings)
if err != nil {
return err
}
db := session.Driver().(*sql.DB)
if err := db.Ping(); err != nil {
fmt.Println("db.Ping failed:", err)
return err
}
l.init = true
l.waitGroup = sync_.NewWaitGroup()
l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
l.logSql = make(chan string, 10)
l.logLevel = config.MysqlLogLevel
l.session = session
for i := 0; i < 10; i++ {
go l.run()
}
go l.writeDB()
return nil
}
Database string
User string
Password string
Charset string
// DB mongo日志
MongoLog bool
MongoLogLevel int
MongoAddr string
}
var (
g_msg_pool = &sync.Pool{New: func() interface{} { return &LogMsg{} }}
g_sysLogger = &logMgr{
init: false,
logMsgCh: make(chan *LogMsg, LOG_SCALE_SIZE),
waitGroup: sync_.NewWaitGroup(),
logger: make(map[string]Logger),
}
g_defaultConfig = &Config{
ModelName: "UnKnow",
LogLevel: ALL_LEVEL,
//console log
ConsoleLog: false,
ConsoleLogLevel: ALL_LEVEL,
//file log
FileLog: false,
FileName: "syslog.log",
Directory: ".",
FileLogLevel: ALL_LEVEL,
//nsqd_client log
NsqdLog: false,