Example #1
0
/*
* 行数为非负值表示已tail的行数
* 行数为负值则都将从文件末尾开始
* 自动保存的行数只可能是bufSize的倍数或者文件总行数
 */
func getLineRecord(path string) (line int, fname string) {
	fin, err := os.Open(path)
	if err != nil {
		_, f, l, _ := runtime.Caller(0)
		loglib.Error(fmt.Sprintf("%s:%d open line record `%s` error\n", f, l, path))
		return -1, "" //从最后开始读
	}
	var txt string
	var lineStr = ""
	//只读第一行
	scanner := bufio.NewScanner(fin)
	for scanner.Scan() {
		txt = strings.Trim(scanner.Text(), " ")
		break
	}
	fin.Close()
	parts := strings.Split(txt, " ")
	if len(parts) == 2 {
		fname = parts[0]
		lineStr = parts[1]
	} else {
		lineStr = parts[0]
	}
	line, err = strconv.Atoi(lineStr)
	if err != nil {
		loglib.Error("convert line record error:" + err.Error())
		line = -1
	}
	return line, fname
}
Example #2
0
//保存footprint
func (t *TcpReceiver) saveFootPrint() {
	vbytes, err := json.Marshal(t.footPrint)
	if err != nil {
		loglib.Error("marshal footprint error:" + err.Error())
		return
	}
	err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664)
	if err == nil {
		loglib.Info("save footprint success !")
	} else {
		loglib.Error("save footprint error:" + err.Error())
	}
}
Example #3
0
func saveLineRecord(path string, fname string, lineNum int) {
	fout, err := os.Create(path)
	defer fout.Close()
	if err != nil {
		loglib.Error("save line record error: " + err.Error())
		return
	}
	_, err = fmt.Fprintf(fout, "%s %d", fname, lineNum)
	if err != nil {
		loglib.Error("Write line record error" + err.Error())
		return
	}
	loglib.Info("save line record success!")
}
Example #4
0
func (this *IntegrityChecker) SaveStatus() {
	m := make(map[string]map[string]map[string]map[string]int)
	m["hour_received"] = this.hourReceived
	m["day_received"] = this.dayReceived
	vbytes, err := json.Marshal(m)
	if err != nil {
		loglib.Error("marshal log received error:" + err.Error())
		return
	}
	err = ioutil.WriteFile(this.statusFile, vbytes, 0664)
	if err == nil {
		loglib.Info("save log received success !")
	} else {
		loglib.Error("save log received error:" + err.Error())
	}
}
Example #5
0
func (this *Tailler) Tailling(receiveChan chan map[string]string) {
	if this.currFile == "" {
		//兼容老格式,老格式无文件路径
		this.currFile = this.getLogFileByTime(time.Now())
	}
	var err error
	this.fileHour, err = this.getTimeFromLogName(this.currFile)
	if err != nil {
		loglib.Error("can't get time from current log file:" + this.currFile + "error:" + err.Error())
		os.Exit(1)
	}
	isQuit := false
	for time.Since(this.fileHour).Hours() >= 1 {
		//说明重启时已经跟记录行号时不属于同一个小时了
		isQuit = this.taillingPrevious(this.currFile, this.lineNum, this.fileHour.Format(this.hourStrFmt), receiveChan)
		if isQuit {
			break
		}
		//继续下一个小时
		this.fileHour = this.fileHour.Add(time.Hour)
		this.currFile = this.getLogFileByTime(this.fileHour)
		this.lineNum = 0
	}
	if !isQuit {
		//处理当前这个小时
		this.taillingCurrent(receiveChan)
	}
	close(receiveChan)
	this.wq.AllDone()
}
Example #6
0
func (this *MongoDbOutputer) Start() {
    wg := &sync.WaitGroup{}
    defer func(){
        if err := recover(); err != nil {
            loglib.Error(fmt.Sprintf("mongodb outputer panic:%v", err))
        }
        if this.session != nil {
            this.session.Close()
        }
        this.wq.AllDone()
    }()

    this.reloadFileCache()
   
    go this.reConnMongoDb()

    wg.Add(this.savers)

    for i:=0; i<this.savers; i++ {
        go this.runParse(i, wg)
    }

    nRetry := this.savers / 6 + 1
    wg.Add(nRetry)
    for i:=0; i<nRetry; i++ {
        go this.retrySave(wg, i)
    }

    wg.Wait()


}
Example #7
0
func (lr *LogReceiver) Run() {
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", lr.port))
	if err != nil {
		loglib.Error("[log receiver] " + err.Error())
		return
	}
	defer l.Close()
	for {
		conn, err := l.Accept()
		if err != nil {
			loglib.Error("[log receiver] " + err.Error())
			return
		}
		go lr.handleConnection(conn)
	}

}
Example #8
0
func (f *fileOutputer) extract(bp *bytes.Buffer) {
	buf := make([]byte, 4)
	bp.Read(buf)

	l, _ := binary.Uvarint(buf)
	headerLen := int(l)
	//get pack header
	buf = make([]byte, headerLen)
	bp.Read(buf)
	header := tcp_pack.ParseHeader(buf)

	r, err := zlib.NewReader(bp)
	if err != nil {
		loglib.Error("zlib reader Error: " + err.Error())
	} else {
		lines, _ := strconv.Atoi(header["lines"])
		done := false
		if header["done"] == "1" {
			done = true
		}
		f.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

		writerKey := header["ip"] + "_" + header["hour"]
		fout := f.getWriter(f.writers, f.dataDir, writerKey)

		//一头一尾写头信息,节省硬盘
		buf = append(buf, '\n')
		//fout.Write(buf)
		nn, err := io.Copy(fout, r)
		if err != nil {
			loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
		}
		//fout.Write(buf)

		//单独存一份header便于查数
		fout = f.getWriter(f.headerWriters, f.headerDir, writerKey)
		n, err := fout.Write(buf)
		if err != nil {
			loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
		}

		if done || time.Now().Unix() > f.checkTime.Unix() {
			hourFinish, _ := f.ic.Check()
			for ip, hours := range hourFinish {
				for _, hour := range hours {
					writerKey = ip + "_" + hour
				}
			}
			f.closeWriters(f.writers)
			f.closeWriters(f.headerWriters)
			f.checkTime.Add(2 * time.Minute)
		}

		r.Close()
	}
}
Example #9
0
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
	fp := make(map[string]PackAppear)
	if lib.FileExists(fname) {
		vbytes, err := ioutil.ReadFile(fname)
		if err != nil {
			loglib.Error("read footprint file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &fp)
			if err != nil {
				loglib.Error("unmarshal footprint error:" + err.Error())
			} else {
				loglib.Info("load footprint success !")
			}
		}
	} else {
		loglib.Warning("footprint file " + fname + " not found!")
	}
	return fp
}
Example #10
0
//init conn list from addrMap
func (sc *SingleConnection) initConnection() {
	newConn, err := createSingleConnection(sc.currentAddr)
	if err != nil {
		loglib.Error("init err:" + err.Error())
	} else {
		sc.conn = newConn
	}

	lib.CheckError(err)
}
Example #11
0
func initMongoDbSession(mongosAddr string) *mgo.Session {
    session, err := mgo.Dial(mongosAddr)
    if err != nil {
        loglib.Error(fmt.Sprintf("init mongodb session error:%v", err))
        return nil
    }   

    session.SetMode(mgo.Monotonic, true)    //设置read preference
    session.SetSafe(&mgo.Safe{W:2})         //设置write concern
    return session
}
Example #12
0
func (e *etlOutputer) Start() {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
		}

		e.wq.AllDone()
	}()

	spiderList, _ := e.config["spider_list"]
	colsFile, _ := e.config["columns_file"]
	hostsList, _ := e.config["hosts_white_list"]
	ipBlackList, _ := e.config["ip_black_list"]

	if colsFile != "" {
		e.runEtl(spiderList, colsFile, hostsList, ipBlackList)
	} else {
		loglib.Error("[error] miss columns map file!")
	}
}
Example #13
0
func (this *MongoDbOutputer) bulkSaveBson(coll *mgo.Collection, docs ...interface{}) (err error) {
    if coll != nil {
        err = coll.Insert(docs...)
        if err != nil {
            tmp := make([]string, 0)
            for _, doc := range docs {
                m, _ := doc.(bson.M)
                tid, _ := m[this.transactionIdKey].(string)
                tmp = append(tmp, tid)
            }
            tids := strings.Join(tmp, ",")
            loglib.Error(fmt.Sprintf("save %d bsons [%s] error:%v", len(docs), tids, err))
        }
    }else{
        err = errors.New("bulk: collection is nil")
        loglib.Error(fmt.Sprintf("save bsons error:%v", err))

    }
    return
}
Example #14
0
//touch一个文件表明某一天接收完
func (this *IntegrityChecker) makeDayTag(ip string, day string) bool {
	fname := fmt.Sprintf("%s_%s", ip, day)
	filename := filepath.Join(this.dir, fname)
	fout, err := os.Create(filename)
	if err != nil {
		loglib.Error("tag " + fname + " error: " + err.Error())
		return false
	} else {
		fout.Close()
	}
	return true
}
Example #15
0
func (this *Tailler) GetTotalLines(fname string) int {
	cmd := exec.Command("/bin/sh", "-c", `wc -l `+fname+` | awk '{print $1}'`)
	out, err := cmd.Output()
	if err == nil {
		n, err := strconv.Atoi(strings.Trim(string(out), "\n"))
		if err != nil {
			loglib.Error("trans total lines " + string(out) + " error: " + err.Error())
		}
		return n
	}
	return 0
}
Example #16
0
func (this *IntegrityChecker) LoadStatus(filename string) map[string]map[string]map[string]map[string]int {
	m := make(map[string]map[string]map[string]map[string]int)
	m["hour_received"] = make(map[string]map[string]map[string]int)
	m["day_received"] = make(map[string]map[string]map[string]int)
	if lib.FileExists(filename) {
		vbytes, err := ioutil.ReadFile(filename)
		if err != nil {
			loglib.Error("read log received file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &m)
			if err != nil {
				loglib.Error("unmarshal log received error:" + err.Error())
			} else {
				loglib.Info("load log received success !")
			}
		}
	} else {
		loglib.Warning("log received file " + filename + " not found!")
	}
	return m
}
Example #17
0
func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) {
    m, _ := selector.(bson.M)
    tid, _ := m[this.transactionIdKey].(string)

    if coll != nil {
        info, err = coll.Upsert(selector, doc)

        if err != nil {
            loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
        }else{
            if info.Updated > 0 {
                loglib.Info(fmt.Sprintf("bson [%s] updated", tid))
            }
        }
    }else{
        info = &mgo.ChangeInfo{}
        err = errors.New("upsert: collection is nil")
        loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
    }
    return
}
Example #18
0
func NewTailler(config map[string]string) *Tailler {
	val, ok := config[logFileKey]
	if !ok || val == "" {
		loglib.Error("config need log_file!")
		os.Exit(1)
	}
	logPath := val
	val, ok = config[recordFileKey]
	if !ok || val == "" {
		config[recordFileKey] = getRecordPath()
	}
	lineNum, fname := getLineRecord(config[recordFileKey])
	goFmt, nLT := extractTimeFmt(logPath)
	if goFmt == "" {
		loglib.Error("log path has no time format!")
		os.Exit(1)
	}
	wq := lib.NewWaitQuit("tailler")
	bufSize, _ := strconv.Atoi(config["recv_buffer_size"])

	return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}
Example #19
0
func (f *fileOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File {
	w, ok := writers[key]
	if !ok || w == nil {
		fname := filepath.Join(parentDir, key)
		w1, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
		writers[key] = w1
		w = w1
		if err != nil {
			loglib.Error(fmt.Sprintf("file outputer create writer: %s error: %s", fname, err.Error()))
		}
	}
	return w
}
Example #20
0
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, hostsList string, ipBlackList string, wg *sync.WaitGroup) {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
		}

		wg.Done()
	}()
	loglib.Info("etl routine start")
	for fkey := range fkeyChan {
		sv := etl.NewFileSaver(colsFile, etlDir, fkey)
		d := etl.NewDispatcher(sv, 6, hostsList, ipBlackList)
		g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
		go g.Start(false)

		fname := filepath.Join(logDataDir, fkey)
		loglib.Info("start etl for " + fname)

		err := g.ParseFile(fname)
		g.Wait()
		// etl success
		// mark success
		if err == nil {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("finish etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
				}
			}
		} else {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlFailDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("failed etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
				}
			}

		}
	}
	loglib.Info("etl routine finish")
}
Example #21
0
func check(addr string, ch chan CheckResult) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		msg := "[heart beat] connect to " + addr + " error"
		loglib.Error(msg)
		ch <- CheckResult{addr, true, msg}
	} else {
		conn.SetReadDeadline(time.Now().Add(10 * time.Second)) //读超时
		var buf = make([]byte, 2)
		n, err := conn.Read(buf)
		if err != nil {
			msg := "[heart beat] read data from " + addr + " error"
			loglib.Error(msg)
			ch <- CheckResult{addr, true, msg}
		}
		if string(buf[:n]) == "!" {
			ch <- CheckResult{addr, false, ""}
		} else {
			ch <- CheckResult{addr, true, "unknow return code"}
		}
		conn.Close()
	}
}
Example #22
0
func New(configFile string) *Monitor {
	//new Monitor
	monitor := new(Monitor)
	monitor.configFile = configFile
	config := lib.ReadConfig(configFile)

	// heart beat checker
	cfg, ok := config["heart_beat"]
	if !ok {
		loglib.Error("miss heart beat config")
		return nil
	}
	checkInterval, _ := strconv.Atoi(cfg["check_interval"])
	delete(cfg, "check_interval")
	mutex := &sync.RWMutex{}

	monitor.mutex = mutex
	monitor.checkInterval = checkInterval

	hbChecker := heart_beat.NewHeartBeatChecker()
	monitor.hbChecker = hbChecker

	//log receiver
	cfg, ok = config["receiver"]
	if !ok {
		loglib.Error("miss receiver config!")
		return nil
	}
	mysql := db.NewMysql(cfg["db_host"], cfg["db_port"], cfg["db_uname"], cfg["db_passwd"], cfg["db_db"], cfg["db_charset"])
	monitor.dbConn = mysql
	monitor.ipRoleMap = getIpRoleMap(mysql)
	recvPort, _ := strconv.Atoi(cfg["recv_port"])
	receiver := NewLogReceiver(recvPort, mysql, monitor.ipRoleMap, monitor.mutex)
	monitor.receiver = receiver

	return monitor
}
Example #23
0
func (this *HeartBeat) Run() {
	i := 0

	//尝试注册3次
	for i < 3 && !this.registerSelf(true) {
		i++
		time.Sleep(10 * time.Second)
	}

	l, err := net.Listen("tcp", ":"+this.port)
	if err != nil {
		loglib.Error("heart beat " + err.Error())
		return
	}
	defer l.Close()

	go lib.HandleQuitSignal(func() {
		this.registerSelf(false)
		l.Close()
		loglib.Info("close heart beat listener.")
	})
	//heart beat 不是太重要,所以退出处理比较简单
	for {
		conn, err := l.Accept()
		if conn == nil {
			//listener关闭
			break
		}
		if err != nil {
			loglib.Error("heart beat " + err.Error())
			break
		}
		go this.handleConnection(conn)
	}

	this.wq.AllDone()
}
Example #24
0
//向monitor注册或取消注册,reg为true表示注册,否则是取消注册,返回true表示成功
func (this *HeartBeat) registerSelf(reg bool) bool {
	conn, err := lib.GetConnection(this.monitorAddr)
	if err != nil {
		return false
	}
	defer conn.Close()

	req := "register"
	if !reg {
		req = "unregister"
	}
	m := map[string]string{"req": req, "ip": lib.GetIp(), "port": this.port, "hostname": lib.GetHostname(), "role": this.role}
	msg, err := json.Marshal(m)
	if err != nil {
		loglib.Error("marshal " + req + " info error " + err.Error())
		return false
	}
	_, err = conn.Write(tcp_pack.Pack(msg))
	if err != nil {
		loglib.Error("send " + req + " info failed" + err.Error())
		return false
	} else {
		plen, ret := tcp_pack.UnPack(conn)

		if plen > 0 {
			err = json.Unmarshal(ret, &m)
			r, ok := m["err"]
			if err == nil && ok && r == "0" {
				loglib.Info(req + " to monitor success!")
				return true
			}
			loglib.Error(req + " heart beat failed!")
		}
	}
	return false
}
Example #25
0
//从日志文件名获取时间,不依赖系统时间
func (this *Tailler) getTimeFromLogName(name string) (time.Time, error) {
	size := len(name)
	timePart := ""
	if this.nLT[0] < size && this.nLT[1] < size {
		timePart = name[this.nLT[0] : size-this.nLT[1]]
	}
	layout := this.goFmt

	loc, _ := time.LoadLocation("Local")
	t, err := time.ParseInLocation(layout, timePart, loc)
	if err != nil {
		loglib.Error("parse " + timePart + " against " + layout + " error:" + err.Error())
	}
	return t, err
}
Example #26
0
func NewIntegrityChecker(dir string) *IntegrityChecker {
	err := os.MkdirAll(dir, 0755)
	if err != nil {
		loglib.Error("make integrity dir error:" + err.Error())
		return nil
	}
	statusFile := getFilePath()
	ic := &IntegrityChecker{}
	ic.dir = dir
	ic.statusFile = statusFile

	status := ic.LoadStatus(ic.statusFile)
	ic.hourReceived = status["hour_received"]
	ic.dayReceived = status["day_received"]
	return ic
}
Example #27
0
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
    mo.buffer = buffer
    mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
    mo.mongosAddr, _ = config["mongos"]
    mo.db         , _ = config["db"]
    mo.collection , _ = config["collection"]
    mo.session = initMongoDbSession(mo.mongosAddr)
    //暂时出错直接退出
    if mo.session == nil {
        loglib.Error("init mongodb session failed")
        os.Exit(1)
    }

    upsert, _ := config["upsert"]
    if upsert == "true" {
        mo.isUpsert = true
    }else{
        mo.isUpsert = false
    }

    bulkSize, _ := config["bulk_size"]
    nBulk, err := strconv.Atoi(bulkSize)
    if err == nil {
        mo.bulkSize = nBulk
    }else{
        mo.bulkSize = 50
    }

    savers, _ := config["savers"]
    nSavers, err := strconv.Atoi(savers)
    if err == nil {
        mo.savers = nSavers
    }else{
        mo.savers = 20
    }
    
    //创建文件缓存目录
    mo.file_mem_folder_name = "tempfile"
    if !lib.FileExists(mo.file_mem_folder_name) {
        os.MkdirAll(mo.file_mem_folder_name, 0775)
    }
    
    mo.transactionIdKey = "transaction_id"
    mo.fileList = lib.GlobalListInit()
    return mo
}
Example #28
0
func (f *fileOutputer) Start() {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("file outputer panic:%v", err))
		}

		f.ic.SaveStatus()
		f.closeWriters(f.writers)
		f.closeWriters(f.headerWriters)
		f.wq.AllDone()
	}()

	//使用range遍历,方便安全退出,只要发送方退出时关闭chan,这里就可以退出了
	for b := range f.buffer {
		f.extract(&b)
	}
}
Example #29
0
func (t *TcpReceiver) Start() {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress)
	lib.CheckError(err)
	listener, err := net.ListenTCP("tcp", tcpAddr)
	lib.CheckError(err)

	wg := &sync.WaitGroup{}

	wg.Add(1)
	go t.clearFootPrint(wg)

	//主routine信号处理
	go lib.HandleQuitSignal(func() {
		//接收到信号关闭listenner,此时Accept会马上返回一个nil 的conn
		listener.Close()
		loglib.Info("close tcp receiver's listener.")
	})

	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err))
		}

		loglib.Info("wait connections finish...")
		wg.Wait()
		loglib.Info("all connections have been processed. quit.")
		close(t.buffer) //关闭chan
		t.saveFootPrint()

		t.wq.AllDone()

	}()

	for {
		conn, err := listener.Accept()
		if conn == nil {
			break
		}
		lib.CheckError(err)
		wg.Add(1)
		go t.handleConnnection(conn, wg)
	}

}
Example #30
0
func createSingleConnection(address string) (conn *net.TCPConn, err error) {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", address)
	lib.CheckError(err)
	if err != nil {
		return nil, err
	}
	conn, err = net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		loglib.Error("get connection from " + address + " failed! Error:" + err.Error())
		return nil, err
	} else {
		loglib.Info("get connection from " + address + " success! remote addr " + conn.RemoteAddr().String())
	}
	lib.CheckError(err)

	return conn, nil
}