//工厂初始化函数
func EtlOutputerInit(buffer chan bytes.Buffer, config map[string]string) (e etlOutputer) {
e.buffer = buffer
saveDir, _ := config["save_dir"]
e.saveDir = saveDir
e.dataDir = filepath.Join(saveDir, "log_data")
e.headerDir = filepath.Join(saveDir, "headers")
e.icDir = filepath.Join(saveDir, "received")
e.etlDir = filepath.Join(saveDir, "etl")
e.etlDoneDir = filepath.Join(saveDir, "etl_done")
e.etlFailDir = filepath.Join(saveDir, "etl_fail")
e.ic = integrity.NewIntegrityChecker(e.icDir)
os.MkdirAll(e.dataDir, 0775)
os.MkdirAll(e.headerDir, 0775)
os.MkdirAll(e.etlDir, 0775)
os.MkdirAll(e.etlDoneDir, 0775)
os.MkdirAll(e.etlFailDir, 0775)
e.writers = make(map[string]*os.File)
e.headerWriters = make(map[string]*os.File)
e.config = config
e.wq = lib.NewWaitQuit("etl outputer", -1) //不限退出超时时间，以便etl能做完
return e
}
//工厂初始化函数
func ReceiverInit(buffer chan bytes.Buffer, c chan map[string]string, listBufferSize int, nTailedLines int) (r Receiver) {
// var r Receiver
r.sendBuffer = buffer
r.logList = list.New()
r.receiveChan = c
r.listBufferSize = listBufferSize
r.wq = lib.NewWaitQuit("receiver")
r.nTailedLines = nTailedLines
return r
}
//工厂初始化函数
func TcpReceiverInit(buffer chan bytes.Buffer, addr string) (t TcpReceiver) {
t.buffer = buffer
t.receiveFromAddress = addr
t.footPrintFile = getFilePath()
t.footPrint = t.loadFootPrint(t.footPrintFile)
t.mutex = &sync.RWMutex{}
t.wq = lib.NewWaitQuit("tcp receiver", -1)
return t
}
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
mo.buffer = buffer
mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
mo.mongosAddr, _ = config["mongos"]
mo.db , _ = config["db"]
mo.collection , _ = config["collection"]
mo.session = initMongoDbSession(mo.mongosAddr)
//暂时出错直接退出
if mo.session == nil {
loglib.Error("init mongodb session failed")
os.Exit(1)
}
upsert, _ := config["upsert"]
if upsert == "true" {
mo.isUpsert = true
}else{
mo.isUpsert = false
}
bulkSize, _ := config["bulk_size"]
nBulk, err := strconv.Atoi(bulkSize)
if err == nil {
mo.bulkSize = nBulk
}else{
mo.bulkSize = 50
}
savers, _ := config["savers"]
nSavers, err := strconv.Atoi(savers)
if err == nil {
mo.savers = nSavers
}else{
mo.savers = 20
}
//创建文件缓存目录
mo.file_mem_folder_name = "tempfile"
if !lib.FileExists(mo.file_mem_folder_name) {
os.MkdirAll(mo.file_mem_folder_name, 0775)
}
mo.transactionIdKey = "transaction_id"
mo.fileList = lib.GlobalListInit()
return mo
}
//工厂初始化函数
func FileOutputerInit(buffer chan bytes.Buffer, saveDir string) (f fileOutputer) {
f.buffer = buffer
f.saveDir = saveDir
f.dataDir = filepath.Join(saveDir, "log_data")
f.headerDir = filepath.Join(saveDir, "headers")
f.icDir = filepath.Join(saveDir, "received")
f.ic = integrity.NewIntegrityChecker(f.icDir)
os.MkdirAll(f.dataDir, 0775)
os.MkdirAll(f.headerDir, 0775)
f.writers = make(map[string]*os.File)
f.headerWriters = make(map[string]*os.File)
f.checkTime = time.Now().Add(2 * time.Minute)
f.wq = lib.NewWaitQuit("file outputer", -1)
return f
}
//工厂初始化函数
//增加备用地址，暂时支持一个备用地址
func SenderInit(buffer chan bytes.Buffer, addr string, bakAddr string, id int) (s Sender) {
// var s Sender
// s = new(Sender)
s.id = id
s.sBuffer = buffer
s.memBuffer = make(chan bytes.Buffer, 20)
s.file_mem_folder_name = "tempfile"
//auto make dir
if _, err := os.Stat(s.file_mem_folder_name); err != nil && os.IsNotExist(err) {
os.MkdirAll(s.file_mem_folder_name, 0775)
}
s.sendToAddress = addr
s.connection = SingleConnectionInit(s.sendToAddress, bakAddr)
a := 1
s.status = &a
s.wq = lib.NewWaitQuit("sender", -1)
return s
}
func NewTailler(config map[string]string) *Tailler {
val, ok := config[logFileKey]
if !ok || val == "" {
loglib.Error("config need log_file!")
os.Exit(1)
}
logPath := val
val, ok = config[recordFileKey]
if !ok || val == "" {
config[recordFileKey] = getRecordPath()
}
lineNum, fname := getLineRecord(config[recordFileKey])
goFmt, nLT := extractTimeFmt(logPath)
if goFmt == "" {
loglib.Error("log path has no time format!")
os.Exit(1)
}
wq := lib.NewWaitQuit("tailler")
bufSize, _ := strconv.Atoi(config["recv_buffer_size"])
return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}
func NewHeartBeat(port string, monitorAddr string, role string) *HeartBeat {
wq := lib.NewWaitQuit("heart beat", 5)
return &HeartBeat{port, monitorAddr, role, wq}
}