示例#1
0
文件: nsq.go 项目: wxaxiaoyao/linux
func ConsumerGO(topic, channel string, goCount uint, handle Handler) (*ConsumerT, error) {
	msgHandle := &ConsumerT{
		consumers: []*nsq.Consumer{},
		handle:    handle,
		msgCh:     make(chan *Message, MSG_CHAN_SIZE),
		waitGroup: sync_.NewWaitGroup(),
	}
	addrSlice := strings.Split(g_nsqdMgr.addrs, ",")
	for _, addr := range addrSlice {
		for i := 0; i < CONSUMER_COUNT_PER_ADDR; i++ {
			consumer, err := nsq.NewConsumer(topic, channel, nsq.NewConfig())
			if err != nil {
				return nil, errors.As(err, topic, channel)
			}
			consumer.SetLogger(nil, nsq.LogLevelInfo)
			consumer.AddHandler(msgHandle)
			if err := consumer.ConnectToNSQD(addr); err != nil {
				return nil, errors.As(err, topic, channel, g_nsqdMgr.addrs)
			}
			msgHandle.consumers = append(msgHandle.consumers, consumer)
		}
	}
	g_nsqdMgr.consumers = msgHandle
	for i := 0; i < int(goCount); i++ {
		go msgHandle.work()
	}
	return msgHandle, nil
}
示例#2
0
文件: nsq.go 项目: wxaxiaoyao/linux
func Init(addrs string) {
	config := nsq.NewConfig()
	config.DefaultRequeueDelay = 0

	addrSlice := strings.Split(addrs, ",")
	produceCount := len(addrSlice) * PRODUCER_COUNT_PER_ADDR
	producers := make([]*nsq.Producer, produceCount)

	var err error = nil
	var idx int = 0
	for _, addr := range addrSlice {
		for i := 0; i < PRODUCER_COUNT_PER_ADDR; i++ {
			producers[idx], err = nsq.NewProducer(addr, config)
			if err != nil {
				panic(err)
			}
			idx++
		}
	}

	g_nsqdMgr = &nsqdMgr{
		addrs:     addrs,
		waitGroup: sync_.NewWaitGroup(),
		producers: producers,
		msgCh:     make(chan nsqdMsg, MSG_CHAN_SIZE),
		consumers: nil,
	}

	for i := 0; i < produceCount; i++ {
		go g_nsqdMgr.producer(i)
	}
	return
}
示例#3
0
func (l *fileLogMgr) Init(config *Config) error {
	if config.FileLog == false {
		return errors.New("file log server not configure!!!")
	}

	date, err := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
	if err != nil {
		fmt.Println("time.Parse failed:", err)
		return err
	}
	dir := config.Directory
	if len(dir) > 0 && dir[len(dir)-1] != '/' {
		dir += "/"
	}
	file, err := os.OpenFile(dir+config.FileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		fmt.Println("os.OpenFile failed", err)
		return err
	}

	l.waitGroup = sync_.NewWaitGroup()
	l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
	l.logLevel = config.FileLogLevel
	l.filename = config.FileName
	l.date = date
	l.directory = dir
	l.file = file
	l.init = true
	go l.run()

	return nil
}
示例#4
0
func (l *nsqdSvrLogMgr) Init(config *Config) error {
	// 一个模块只创建一个实例
	if config.NsqdSvrLog == false {
		return errors.New("nsqd_server log server not configure!!!")
	}
	// 注册日志驱动
	l.logLevel = config.LogLevel
	l.logger = make(map[string]Logger)
	l.waitGroup = sync_.NewWaitGroup()

	nsqdCfg := config.NsqdSvrConfig
	if nsqdCfg.ConsoleLog {
		l.logger[CONSOLE] = &consoleLogMgr{}
	}
	if nsqdCfg.FileLog {
		l.logger[FILE] = &fileLogMgr{}
	}
	if nsqdCfg.NsqdLog {
		l.logger[NSQD_CLIENT] = &nsqdLogMgr{}
	}
	if nsqdCfg.NsqdSvrLog {
		l.logger[NSQD_SERVER] = &nsqdSvrLogMgr{}
	}
	if nsqdCfg.MysqlLog {
		l.logger[MYSQL] = &MysqlLogMgr{}
	}
	if nsqdCfg.MongoLog {
		l.logger[MONGO] = &MongoLogMgr{}
	}
	// 初始化日志驱动
	for logName, logger := range l.logger {
		fmt.Printf("init log driver...[%s] [%s]\n", logName, nsqdCfg.ModelName)
		if err := logger.Init(nsqdCfg); err != nil {
			fmt.Printf("init log driver failed!!![%s] [%s]; error:%s\n", logName, nsqdCfg.ModelName, err.Error())
			continue
		}
		fmt.Printf("init log driver success...[%s] [%s]\n", logName, nsqdCfg.ModelName)
	}
	l.consumer = []*nsq.Consumer{}
	addrs := strings.Split(config.NsqdAddrs, ",")
	for _, addr := range addrs {
		for i := 0; i < CONSUME_COUNT_PER_ADDR; i++ {
			consumer, err := nsq.NewConsumer(config.NsqdTopic, config.NsqdChannel, nsq.NewConfig())
			if err != nil {
				fmt.Println("NewConsumer failed:", err)
				return err
			}
			consumer.SetLogger(nil, nsq.LogLevelInfo)
			consumer.AddHandler(l)
			if err := consumer.ConnectToNSQD(addr); err != nil {
				fmt.Println("ConnectToNSQDs failed, error:", err)
				return err
			}
			l.consumer = append(l.consumer, consumer)
		}
	}
	l.init = true
	return nil
}
示例#5
0
func (l *consoleLogMgr) Init(config *Config) error {
	if config.ConsoleLog == false {
		return errors.New("console log server not configure!!!")
	}

	l.init = true
	l.waitGroup = sync_.NewWaitGroup()
	l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
	l.logLevel = config.ConsoleLogLevel

	go l.run()
	return nil
}
示例#6
0
func (l *MongoLogMgr) Init(config *Config) error {
	if config.MongoLog == false {
		return errors.New("mongo log server not configure!!!")
	}

	session, err := mgo.Dial(config.MongoAddr)
	if err != nil {
		panic(err)
	}
	session.SetMode(mgo.Monotonic, true)

	l.init = true
	l.waitGroup = sync_.NewWaitGroup()
	l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
	l.logLevel = config.MongoLogLevel
	l.session = session

	go l.run()
	return nil
}
示例#7
0
文件: qiniu.go 项目: wxaxiaoyao/linux
func QiniuInit() error {
	c := common.MongoCollection(FILE_DB, FILE_QINIU_FILE_TABLE)

	index := mgo.Index{
		Key:      []string{"filename"},
		Unique:   true,
		DropDups: true,
	}

	g_file_mgr = &FileMgr{
		fileCh:       make(chan *File, 100),
		expiredTimer: time.Tick(time.Hour),
		waitGroup:    sync.NewWaitGroup(),
		rsClient:     rs.New(&digest.Mac{AccessKey: g_access_key, SecretKey: []byte(g_secret_key)}),
	}

	go g_file_mgr.run()

	return errors.As(c.EnsureIndex(index))
}
示例#8
0
func (l *nsqdLogMgr) Init(config *Config) error {
	if config.NsqdLog == false {
		return errors.New("nsqd log server not configure!!!")
	}

	nsqConfig := nsq.NewConfig()
	nsqConfig.DefaultRequeueDelay = 0
	connCountPerAddr := 2
	addrSlice := strings.Split(config.NsqdAddrs, ",")
	producerCount := len(addrSlice) * connCountPerAddr
	producers := make([]*nsq.Producer, producerCount)

	var err error
	idx := 0
	for _, addr := range addrSlice {
		for i := 0; i < connCountPerAddr; i++ {
			producers[idx], err = nsq.NewProducer(addr, nsqConfig)
			if err != nil {
				fmt.Println("NewProducer ", addr, " error:", err)
			}
			//fmt.Println("nsq_client connect success:", i)
			idx++
		}
	}

	l.waitGroup = sync_.NewWaitGroup()
	l.producers = producers
	l.producerCount = uint(producerCount)
	l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
	l.logTopic = config.NsqdTopic
	l.logLevel = config.NsqdLogLevel
	l.init = true

	for i := 0; i < producerCount; i++ {
		go l.run(i)
	}
	return nil
}
示例#9
0
文件: mysql.go 项目: wxaxiaoyao/linux
func (l *MysqlLogMgr) Init(config *Config) error {
	if config.MysqlLog == false {
		return errors.New("mysql log server not configure!!!")
	}

	settings := db.DataSource{
		Host:     config.Host,
		Port:     config.Port,
		Database: config.Database,
		User:     config.User,
		Password: config.Password,
		Charset:  config.Charset,
	}

	session, err := db.Open("mysql", settings)
	if err != nil {
		return err
	}
	db := session.Driver().(*sql.DB)
	if err := db.Ping(); err != nil {
		fmt.Println("db.Ping failed:", err)
		return err
	}

	l.init = true
	l.waitGroup = sync_.NewWaitGroup()
	l.logMsgCh = make(chan *LogMsg, getChildLogScaleSize())
	l.logSql = make(chan string, 10)
	l.logLevel = config.MysqlLogLevel
	l.session = session

	for i := 0; i < 10; i++ {
		go l.run()
	}
	go l.writeDB()
	return nil
}
示例#10
0
文件: log.go 项目: sunyuantao/windows
	Database      string
	User          string
	Password      string
	Charset       string
	// DB mongo日志
	MongoLog      bool
	MongoLogLevel int
	MongoAddr     string
}

var (
	g_msg_pool  = &sync.Pool{New: func() interface{} { return &LogMsg{} }}
	g_sysLogger = &logMgr{
		init:      false,
		logMsgCh:  make(chan *LogMsg, LOG_SCALE_SIZE),
		waitGroup: sync_.NewWaitGroup(),
		logger:    make(map[string]Logger),
	}
	g_defaultConfig = &Config{
		ModelName: "UnKnow",
		LogLevel:  ALL_LEVEL,
		//console log
		ConsoleLog:      false,
		ConsoleLogLevel: ALL_LEVEL,
		//file log
		FileLog:      false,
		FileName:     "syslog.log",
		Directory:    ".",
		FileLogLevel: ALL_LEVEL,
		//nsqd_client log
		NsqdLog:      false,