Exemplo n.º 1
0
//工厂初始化函数
func EtlOutputerInit(buffer chan bytes.Buffer, config map[string]string) (e etlOutputer) {

	e.buffer = buffer
	saveDir, _ := config["save_dir"]
	e.saveDir = saveDir
	e.dataDir = filepath.Join(saveDir, "log_data")
	e.headerDir = filepath.Join(saveDir, "headers")
	e.icDir = filepath.Join(saveDir, "received")
	e.etlDir = filepath.Join(saveDir, "etl")
	e.etlDoneDir = filepath.Join(saveDir, "etl_done")
	e.etlFailDir = filepath.Join(saveDir, "etl_fail")
	e.ic = integrity.NewIntegrityChecker(e.icDir)

	os.MkdirAll(e.dataDir, 0775)
	os.MkdirAll(e.headerDir, 0775)
	os.MkdirAll(e.etlDir, 0775)
	os.MkdirAll(e.etlDoneDir, 0775)
	os.MkdirAll(e.etlFailDir, 0775)

	e.writers = make(map[string]*os.File)
	e.headerWriters = make(map[string]*os.File)
	e.config = config
	e.wq = lib.NewWaitQuit("etl outputer", -1) //不限退出超时时间,以便etl能做完

	return e
}
Exemplo n.º 2
0
//工厂初始化函数
func ReceiverInit(buffer chan bytes.Buffer, c chan map[string]string, listBufferSize int, nTailedLines int) (r Receiver) {
	// var r Receiver
	r.sendBuffer = buffer
	r.logList = list.New()
	r.receiveChan = c
	r.listBufferSize = listBufferSize
	r.wq = lib.NewWaitQuit("receiver")
	r.nTailedLines = nTailedLines
	return r
}
Exemplo n.º 3
0
//工厂初始化函数
func TcpReceiverInit(buffer chan bytes.Buffer, addr string) (t TcpReceiver) {

	t.buffer = buffer
	t.receiveFromAddress = addr

	t.footPrintFile = getFilePath()
	t.footPrint = t.loadFootPrint(t.footPrintFile)
	t.mutex = &sync.RWMutex{}
	t.wq = lib.NewWaitQuit("tcp receiver", -1)
	return t
}
Exemplo n.º 4
0
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
    mo.buffer = buffer
    mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
    mo.mongosAddr, _ = config["mongos"]
    mo.db         , _ = config["db"]
    mo.collection , _ = config["collection"]
    mo.session = initMongoDbSession(mo.mongosAddr)
    //暂时出错直接退出
    if mo.session == nil {
        loglib.Error("init mongodb session failed")
        os.Exit(1)
    }

    upsert, _ := config["upsert"]
    if upsert == "true" {
        mo.isUpsert = true
    }else{
        mo.isUpsert = false
    }

    bulkSize, _ := config["bulk_size"]
    nBulk, err := strconv.Atoi(bulkSize)
    if err == nil {
        mo.bulkSize = nBulk
    }else{
        mo.bulkSize = 50
    }

    savers, _ := config["savers"]
    nSavers, err := strconv.Atoi(savers)
    if err == nil {
        mo.savers = nSavers
    }else{
        mo.savers = 20
    }
    
    //创建文件缓存目录
    mo.file_mem_folder_name = "tempfile"
    if !lib.FileExists(mo.file_mem_folder_name) {
        os.MkdirAll(mo.file_mem_folder_name, 0775)
    }
    
    mo.transactionIdKey = "transaction_id"
    mo.fileList = lib.GlobalListInit()
    return mo
}
Exemplo n.º 5
0
//工厂初始化函数
func FileOutputerInit(buffer chan bytes.Buffer, saveDir string) (f fileOutputer) {

	f.buffer = buffer
	f.saveDir = saveDir
	f.dataDir = filepath.Join(saveDir, "log_data")
	f.headerDir = filepath.Join(saveDir, "headers")
	f.icDir = filepath.Join(saveDir, "received")
	f.ic = integrity.NewIntegrityChecker(f.icDir)

	os.MkdirAll(f.dataDir, 0775)
	os.MkdirAll(f.headerDir, 0775)

	f.writers = make(map[string]*os.File)
	f.headerWriters = make(map[string]*os.File)
	f.checkTime = time.Now().Add(2 * time.Minute)

	f.wq = lib.NewWaitQuit("file outputer", -1)
	return f
}
Exemplo n.º 6
0
//工厂初始化函数
//增加备用地址,暂时支持一个备用地址
func SenderInit(buffer chan bytes.Buffer, addr string, bakAddr string, id int) (s Sender) {
	// var s Sender
	// s = new(Sender)
	s.id = id
	s.sBuffer = buffer
	s.memBuffer = make(chan bytes.Buffer, 20)
	s.file_mem_folder_name = "tempfile"
	//auto make dir
	if _, err := os.Stat(s.file_mem_folder_name); err != nil && os.IsNotExist(err) {
		os.MkdirAll(s.file_mem_folder_name, 0775)
	}
	s.sendToAddress = addr
	s.connection = SingleConnectionInit(s.sendToAddress, bakAddr)
	a := 1
	s.status = &a
	s.wq = lib.NewWaitQuit("sender", -1)

	return s
}
Exemplo n.º 7
0
func NewTailler(config map[string]string) *Tailler {
	val, ok := config[logFileKey]
	if !ok || val == "" {
		loglib.Error("config need log_file!")
		os.Exit(1)
	}
	logPath := val
	val, ok = config[recordFileKey]
	if !ok || val == "" {
		config[recordFileKey] = getRecordPath()
	}
	lineNum, fname := getLineRecord(config[recordFileKey])
	goFmt, nLT := extractTimeFmt(logPath)
	if goFmt == "" {
		loglib.Error("log path has no time format!")
		os.Exit(1)
	}
	wq := lib.NewWaitQuit("tailler")
	bufSize, _ := strconv.Atoi(config["recv_buffer_size"])

	return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}
Exemplo n.º 8
0
func NewHeartBeat(port string, monitorAddr string, role string) *HeartBeat {
	wq := lib.NewWaitQuit("heart beat", 5)
	return &HeartBeat{port, monitorAddr, role, wq}
}