Пример #1
0
/*
* 行数为非负值表示已tail的行数
* 行数为负值则都将从文件末尾开始
* 自动保存的行数只可能是bufSize的倍数或者文件总行数
 */
func getLineRecord(path string) (line int, fname string) {
	fin, err := os.Open(path)
	if err != nil {
		_, f, l, _ := runtime.Caller(0)
		loglib.Error(fmt.Sprintf("%s:%d open line record `%s` error\n", f, l, path))
		return -1, "" //从最后开始读
	}
	var txt string
	var lineStr = ""
	//只读第一行
	scanner := bufio.NewScanner(fin)
	for scanner.Scan() {
		txt = strings.Trim(scanner.Text(), " ")
		break
	}
	fin.Close()
	parts := strings.Split(txt, " ")
	if len(parts) == 2 {
		fname = parts[0]
		lineStr = parts[1]
	} else {
		lineStr = parts[0]
	}
	line, err = strconv.Atoi(lineStr)
	if err != nil {
		loglib.Error("convert line record error:" + err.Error())
		line = -1
	}
	return line, fname
}
Пример #2
0
//保存footprint
func (t *TcpReceiver) saveFootPrint() {
	vbytes, err := json.Marshal(t.footPrint)
	if err != nil {
		loglib.Error("marshal footprint error:" + err.Error())
		return
	}
	err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664)
	if err == nil {
		loglib.Info("save footprint success !")
	} else {
		loglib.Error("save footprint error:" + err.Error())
	}
}
Пример #3
0
func saveLineRecord(path string, fname string, lineNum int) {
	fout, err := os.Create(path)
	defer fout.Close()
	if err != nil {
		loglib.Error("save line record error: " + err.Error())
		return
	}
	_, err = fmt.Fprintf(fout, "%s %d", fname, lineNum)
	if err != nil {
		loglib.Error("Write line record error" + err.Error())
		return
	}
	loglib.Info("save line record success!")
}
Пример #4
0
func (this *MongoDbOutputer) Start() {
	wg := &sync.WaitGroup{}
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("mongodb outputer panic:%v", err))
		}
		if this.session != nil {
			this.session.Close()
		}
		this.wq.AllDone()
	}()

	this.reloadFileCache()

	go this.reConnMongoDb()

	wg.Add(this.savers)

	for i := 0; i < this.savers; i++ {
		go this.runParse(i, wg)
	}

	nRetry := this.savers/6 + 1
	wg.Add(nRetry)
	for i := 0; i < nRetry; i++ {
		go this.retrySave(wg, i)
	}

	wg.Wait()

}
Пример #5
0
func (this *Tailler) Tailling(receiveChan chan map[string]string) {
	if this.currFile == "" {
		//兼容老格式,老格式无文件路径
		this.currFile = this.getLogFileByTime(time.Now())
	}
	var err error
	this.fileHour, err = this.getTimeFromLogName(this.currFile)
	if err != nil {
		loglib.Error("can't get time from current log file:" + this.currFile + "error:" + err.Error())
		os.Exit(1)
	}
	isQuit := false
	for time.Since(this.fileHour).Hours() >= 1 {
		//说明重启时已经跟记录行号时不属于同一个小时了
		isQuit = this.taillingPrevious(this.currFile, this.lineNum, this.fileHour.Format(this.hourStrFmt), receiveChan)
		if isQuit {
			break
		}
		//继续下一个小时
		this.fileHour = this.fileHour.Add(time.Hour)
		this.currFile = this.getLogFileByTime(this.fileHour)
		this.lineNum = 0
	}
	if !isQuit {
		//处理当前这个小时
		this.taillingCurrent(receiveChan)
	}
	close(receiveChan)
	this.wq.AllDone()
}
Пример #6
0
func (lr *LogReceiver) Run() {
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", lr.port))
	if err != nil {
		loglib.Error("[log receiver] " + err.Error())
		return
	}
	defer l.Close()
	for {
		conn, err := l.Accept()
		if err != nil {
			loglib.Error("[log receiver] " + err.Error())
			return
		}
		go lr.handleConnection(conn)
	}

}
Пример #7
0
func (f *fileOutputer) extract(bp *bytes.Buffer) {
	buf := make([]byte, 4)
	bp.Read(buf)

	l, _ := binary.Uvarint(buf)
	headerLen := int(l)
	//get pack header
	buf = make([]byte, headerLen)
	bp.Read(buf)
	header := tcp_pack.ParseHeader(buf)

	r, err := zlib.NewReader(bp)
	if err != nil {
		loglib.Error("zlib reader Error: " + err.Error())
	} else {
		lines, _ := strconv.Atoi(header["lines"])
		done := false
		if header["done"] == "1" {
			done = true
		}
		f.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

		writerKey := header["ip"] + "_" + header["hour"]
		fout := f.getWriter(f.writers, f.dataDir, writerKey)

		//一头一尾写头信息,节省硬盘
		buf = append(buf, '\n')
		//fout.Write(buf)
		nn, err := io.Copy(fout, r)
		if err != nil {
			loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
		}
		//fout.Write(buf)

		//单独存一份header便于查数
		fout = f.getWriter(f.headerWriters, f.headerDir, writerKey)
		n, err := fout.Write(buf)
		if err != nil {
			loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
		}

		if done || time.Now().Unix() > f.checkTime.Unix() {
			hourFinish, _ := f.ic.Check()
			for ip, hours := range hourFinish {
				for _, hour := range hours {
					writerKey = ip + "_" + hour
				}
			}
			f.closeWriters(f.writers)
			f.closeWriters(f.headerWriters)
			f.checkTime.Add(2 * time.Minute)
		}

		r.Close()
	}
}
Пример #8
0
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
	fp := make(map[string]PackAppear)
	if lib.FileExists(fname) {
		vbytes, err := ioutil.ReadFile(fname)
		if err != nil {
			loglib.Error("read footprint file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &fp)
			if err != nil {
				loglib.Error("unmarshal footprint error:" + err.Error())
			} else {
				loglib.Info("load footprint success !")
			}
		}
	} else {
		loglib.Warning("footprint file " + fname + " not found!")
	}
	return fp
}
Пример #9
0
//init conn list from addrMap
func (sc *SingleConnection) initConnection() {
	newConn, err := createSingleConnection(sc.currentAddr)
	if err != nil {
		loglib.Error("init err:" + err.Error())
	} else {
		sc.conn = newConn
	}

	lib.CheckError(err)
}
Пример #10
0
func (e *etlOutputer) Start() {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
		}

		e.wq.AllDone()
	}()

	spiderList, _ := e.config["spider_list"]
	colsFile, _ := e.config["columns_file"]
	hostsList, _ := e.config["hosts_white_list"]
	ipBlackList, _ := e.config["ip_black_list"]

	if colsFile != "" {
		e.runEtl(spiderList, colsFile, hostsList, ipBlackList)
	} else {
		loglib.Error("[error] miss columns map file!")
	}
}
Пример #11
0
func initMongoDbSession(mongosAddr string) *mgo.Session {
	session, err := mgo.Dial(mongosAddr)
	if err != nil {
		loglib.Error(fmt.Sprintf("init mongodb session error:%v", err))
		return nil
	}

	session.SetMode(mgo.Monotonic, true) //设置read preference
	session.SetSafe(&mgo.Safe{W: 2})     //设置write concern
	return session
}
Пример #12
0
func (this *MongoDbOutputer) bulkSaveBson(coll *mgo.Collection, docs ...interface{}) (err error) {
	if coll != nil {
		err = coll.Insert(docs...)
		if err != nil {
			tmp := make([]string, 0)
			for _, doc := range docs {
				m, _ := doc.(bson.M)
				tid, _ := m[this.transactionIdKey].(string)
				tmp = append(tmp, tid)
			}
			tids := strings.Join(tmp, ",")
			loglib.Error(fmt.Sprintf("save %d bsons [%s] error:%v", len(docs), tids, err))
		}
	} else {
		err = errors.New("bulk: collection is nil")
		loglib.Error(fmt.Sprintf("save bsons error:%v", err))

	}
	return
}
Пример #13
0
func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) {
	m, _ := selector.(bson.M)
	tid, _ := m[this.transactionIdKey].(string)

	if coll != nil {
		info, err = coll.Upsert(selector, doc)

		if err != nil {
			loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
		} else {
			if info.Updated > 0 {
				loglib.Info(fmt.Sprintf("bson [%s] updated", tid))
			}
		}
	} else {
		info = &mgo.ChangeInfo{}
		err = errors.New("upsert: collection is nil")
		loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
	}
	return
}
Пример #14
0
func (this *Tailler) GetTotalLines(fname string) int {
	cmd := exec.Command("/bin/sh", "-c", `wc -l `+fname+` | awk '{print $1}'`)
	out, err := cmd.Output()
	if err == nil {
		n, err := strconv.Atoi(strings.Trim(string(out), "\n"))
		if err != nil {
			loglib.Error("trans total lines " + string(out) + " error: " + err.Error())
		}
		return n
	}
	return 0
}
Пример #15
0
func NewTailler(config map[string]string) *Tailler {
	val, ok := config[logFileKey]
	if !ok || val == "" {
		loglib.Error("config need log_file!")
		os.Exit(1)
	}
	logPath := val
	val, ok = config[recordFileKey]
	if !ok || val == "" {
		config[recordFileKey] = getRecordPath()
	}
	lineNum, fname := getLineRecord(config[recordFileKey])
	goFmt, nLT := extractTimeFmt(logPath)
	if goFmt == "" {
		loglib.Error("log path has no time format!")
		os.Exit(1)
	}
	wq := lib.NewWaitQuit("tailler")
	bufSize, _ := strconv.Atoi(config["recv_buffer_size"])

	return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}
Пример #16
0
func (e *etlOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File {
	w, ok := writers[key]
	if !ok || w == nil {
		fname := filepath.Join(parentDir, key)
		w1, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
		writers[key] = w1
		w = w1
		if err != nil {
			loglib.Error(fmt.Sprintf("etl outputer create writer: %s error: %s", fname, err.Error()))
		}
	}
	return w
}
Пример #17
0
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, hostsList string, ipBlackList string, wg *sync.WaitGroup) {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
		}

		wg.Done()
	}()
	loglib.Info("etl routine start")
	for fkey := range fkeyChan {
		sv := etl.NewFileSaver(colsFile, etlDir, fkey)
		d := etl.NewDispatcher(sv, 6, hostsList, ipBlackList)
		g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
		go g.Start(false)

		fname := filepath.Join(logDataDir, fkey)
		loglib.Info("start etl for " + fname)

		err := g.ParseFile(fname)
		g.Wait()
		// etl success
		// mark success
		if err == nil {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("finish etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
				}
			}
		} else {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlFailDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("failed etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
				}
			}

		}
	}
	loglib.Info("etl routine finish")
}
Пример #18
0
func check(addr string, ch chan CheckResult) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		msg := "[heart beat] connect to " + addr + " error"
		loglib.Error(msg)
		ch <- CheckResult{addr, true, msg}
	} else {
		conn.SetReadDeadline(time.Now().Add(10 * time.Second)) //读超时
		var buf = make([]byte, 2)
		n, err := conn.Read(buf)
		if err != nil {
			msg := "[heart beat] read data from " + addr + " error"
			loglib.Error(msg)
			ch <- CheckResult{addr, true, msg}
		}
		if string(buf[:n]) == "!" {
			ch <- CheckResult{addr, false, ""}
		} else {
			ch <- CheckResult{addr, true, "unknow return code"}
		}
		conn.Close()
	}
}
Пример #19
0
func New(configFile string) *Monitor {
	//new Monitor
	monitor := new(Monitor)
	monitor.configFile = configFile
	config := lib.ReadConfig(configFile)

	// heart beat checker
	cfg, ok := config["heart_beat"]
	if !ok {
		loglib.Error("miss heart beat config")
		return nil
	}
	checkInterval, _ := strconv.Atoi(cfg["check_interval"])
	delete(cfg, "check_interval")
	mutex := &sync.RWMutex{}

	monitor.mutex = mutex
	monitor.checkInterval = checkInterval

	hbChecker := heart_beat.NewHeartBeatChecker()
	monitor.hbChecker = hbChecker

	//log receiver
	cfg, ok = config["receiver"]
	if !ok {
		loglib.Error("miss receiver config!")
		return nil
	}
	mysql := db.NewMysql(cfg["db_host"], cfg["db_port"], cfg["db_uname"], cfg["db_passwd"], cfg["db_db"], cfg["db_charset"])
	monitor.dbConn = mysql
	monitor.ipRoleMap = getIpRoleMap(mysql)
	recvPort, _ := strconv.Atoi(cfg["recv_port"])
	receiver := NewLogReceiver(recvPort, mysql, monitor.ipRoleMap, monitor.mutex)
	monitor.receiver = receiver

	return monitor
}
Пример #20
0
func (this *HeartBeat) Run() {
	i := 0

	//尝试注册3次
	for i < 3 && !this.registerSelf(true) {
		i++
		time.Sleep(10 * time.Second)
	}

	l, err := net.Listen("tcp", ":"+this.port)
	if err != nil {
		loglib.Error("heart beat " + err.Error())
		return
	}
	defer l.Close()

	go lib.HandleQuitSignal(func() {
		this.registerSelf(false)
		l.Close()
		loglib.Info("close heart beat listener.")
	})
	//heart beat 不是太重要,所以退出处理比较简单
	for {
		conn, err := l.Accept()
		if conn == nil {
			//listener关闭
			break
		}
		if err != nil {
			loglib.Error("heart beat " + err.Error())
			break
		}
		go this.handleConnection(conn)
	}

	this.wq.AllDone()
}
Пример #21
0
//从日志文件名获取时间,不依赖系统时间
func (this *Tailler) getTimeFromLogName(name string) (time.Time, error) {
	size := len(name)
	timePart := ""
	if this.nLT[0] < size && this.nLT[1] < size {
		timePart = name[this.nLT[0] : size-this.nLT[1]]
	}
	layout := this.goFmt

	loc, _ := time.LoadLocation("Local")
	t, err := time.ParseInLocation(layout, timePart, loc)
	if err != nil {
		loglib.Error("parse " + timePart + " against " + layout + " error:" + err.Error())
	}
	return t, err
}
Пример #22
0
//向monitor注册或取消注册,reg为true表示注册,否则是取消注册,返回true表示成功
func (this *HeartBeat) registerSelf(reg bool) bool {
	conn, err := lib.GetConnection(this.monitorAddr)
	if err != nil {
		return false
	}
	defer conn.Close()

	req := "register"
	if !reg {
		req = "unregister"
	}
	m := map[string]string{"req": req, "ip": lib.GetIp(), "port": this.port, "hostname": lib.GetHostname(), "role": this.role}
	msg, err := json.Marshal(m)
	if err != nil {
		loglib.Error("marshal " + req + " info error " + err.Error())
		return false
	}
	_, err = conn.Write(tcp_pack.Pack(msg))
	if err != nil {
		loglib.Error("send " + req + " info failed" + err.Error())
		return false
	} else {
		plen, ret := tcp_pack.UnPack(conn)

		if plen > 0 {
			err = json.Unmarshal(ret, &m)
			r, ok := m["err"]
			if err == nil && ok && r == "0" {
				loglib.Info(req + " to monitor success!")
				return true
			}
			loglib.Error(req + " heart beat failed!")
		}
	}
	return false
}
Пример #23
0
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
	mo.buffer = buffer
	mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
	mo.mongosAddr, _ = config["mongos"]
	mo.db, _ = config["db"]
	mo.collection, _ = config["collection"]
	mo.session = initMongoDbSession(mo.mongosAddr)
	//暂时出错直接退出
	if mo.session == nil {
		loglib.Error("init mongodb session failed")
		os.Exit(1)
	}

	upsert, _ := config["upsert"]
	if upsert == "true" {
		mo.isUpsert = true
	} else {
		mo.isUpsert = false
	}

	bulkSize, _ := config["bulk_size"]
	nBulk, err := strconv.Atoi(bulkSize)
	if err == nil {
		mo.bulkSize = nBulk
	} else {
		mo.bulkSize = 50
	}

	savers, _ := config["savers"]
	nSavers, err := strconv.Atoi(savers)
	if err == nil {
		mo.savers = nSavers
	} else {
		mo.savers = 20
	}

	//创建文件缓存目录
	mo.file_mem_folder_name = "tempfile"
	if !lib.FileExists(mo.file_mem_folder_name) {
		os.MkdirAll(mo.file_mem_folder_name, 0775)
	}

	mo.transactionIdKey = "transaction_id"
	mo.fileList = lib.GlobalListInit()
	return mo
}
Пример #24
0
func (f *fileOutputer) Start() {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("file outputer panic:%v", err))
		}

		f.ic.SaveStatus()
		f.closeWriters(f.writers)
		f.closeWriters(f.headerWriters)
		f.wq.AllDone()
	}()

	//使用range遍历,方便安全退出,只要发送方退出时关闭chan,这里就可以退出了
	for b := range f.buffer {
		f.extract(&b)
	}
}
Пример #25
0
func (t *TcpReceiver) Start() {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress)
	lib.CheckError(err)
	listener, err := net.ListenTCP("tcp", tcpAddr)
	lib.CheckError(err)

	wg := &sync.WaitGroup{}

	wg.Add(1)
	go t.clearFootPrint(wg)

	//主routine信号处理
	go lib.HandleQuitSignal(func() {
		//接收到信号关闭listenner,此时Accept会马上返回一个nil 的conn
		listener.Close()
		loglib.Info("close tcp receiver's listener.")
	})

	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err))
		}

		loglib.Info("wait connections finish...")
		wg.Wait()
		loglib.Info("all connections have been processed. quit.")
		close(t.buffer) //关闭chan
		t.saveFootPrint()

		t.wq.AllDone()

	}()

	for {
		conn, err := listener.Accept()
		if conn == nil {
			break
		}
		lib.CheckError(err)
		wg.Add(1)
		go t.handleConnnection(conn, wg)
	}

}
Пример #26
0
func createSingleConnection(address string) (conn *net.TCPConn, err error) {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", address)
	lib.CheckError(err)
	if err != nil {
		return nil, err
	}
	conn, err = net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		loglib.Error("get connection from " + address + " failed! Error:" + err.Error())
		return nil, err
	} else {
		loglib.Info("get connection from " + address + " success! remote addr " + conn.RemoteAddr().String())
	}
	lib.CheckError(err)

	return conn, nil
}
Пример #27
0
func (this *MongoDbOutputer) extract(bp *bytes.Buffer) (r io.ReadCloser, packId string, date string, err error) {
	buf := make([]byte, 4)
	bp.Read(buf)

	l, _ := binary.Uvarint(buf)
	headerLen := int(l)
	//get pack header
	buf = make([]byte, headerLen)
	bp.Read(buf)
	header := tcp_pack.ParseHeader(buf)

	r, err = zlib.NewReader(bp)
	if err != nil {
		loglib.Error("zlib reader Error: " + err.Error())
	}
	date = header["hour"][0:8] //用于按天分库
	packId = fmt.Sprintf("%s_%s_%s", header["ip"], header["hour"], header["id"])
	return
}
Пример #28
0
func getIpRoleMap(dbConn *db.Mysql) map[string]string {
	sql := "select ip, port, role from " + registerTable
	res, err := dbConn.Query(sql)
	m := make(map[string]string)
	if err != nil {
		loglib.Error("read registered nodes failed! Error: " + err.Error())
	} else {
		for _, row := range res.Rows {
			ip := row[0]
			port := row[1]
			role := row[2]
			if port != "" {
				ip = ip + ":" + port
			}
			m[ip] = role
		}
		loglib.Info("readed working nodes from database.")
	}
	return m
}
Пример #29
0
//缓存写入mongodb失败的数据
//typeStr为bulk或upsert
func (this *MongoDbOutputer) cacheData(data interface{}, typeStr string, date string, routineId int) {
	mp := bson.M{"type": typeStr, "date": date, "data": data}
	saveTry := 3
	b, err := json.Marshal(mp)
	arr, ok := data.([]bson.M)
	cnt := 1
	if ok {
		cnt = len(arr)
	}
	if err != nil {
		loglib.Error(fmt.Sprintf("cache data error when marshal, discard %d item(s), error:%v", cnt, err))
		return
	}
	fname := this.createFileName(routineId)
	for i := 0; i < saveTry; i++ {
		err = ioutil.WriteFile(fname, b, 0666)
		if err == nil {
			this.fileList.PushBack(fname)
			loglib.Info(fmt.Sprintf("cache %d bson", cnt))
			break
		}
	}
}
Пример #30
0
//goroutine
func (s *Sender) Start() {
	// conn := s.getConnection()
	//初始化fileCacheList
	once.Do(s.reloadFileCache)

	//收尾工作
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("sender %d panic:%v", s.id, err))
		}

		s.saveBufferInChan()

		//s.saveMemCache()

		s.connection.close()

		s.wq.AllDone()

	}()

	go s.pickPacks()
	//var connLost = 0
	var quit = false
	go lib.HandleQuitSignal(func() {
		quit = true
		s.connection.close()
	})

	var sendInterval = time.Duration(2000) //间隔稍大,避免发送文件缓存时因无连接或其他错误进入死循环

	var timeoutChan = time.After(sendInterval * time.Millisecond)
	for !quit {

		select {
		case b := <-s.memBuffer:
			//send b
			result := s.sendBuffer(b)
			if result == false {
				//改为直接放入文件缓存
				s.writeToFile(b)
			}

		case <-timeoutChan:
			timeoutChan = time.After(sendInterval * time.Millisecond)

			// send from file
			e := fileList.Remove()
			if e != nil { // file list is not empty
				filename := e.Value.(string)
				// fmt.Println("sender ",s.id,": get file :",filename)
				data, err := ioutil.ReadFile(filename)
				if err != nil {
					// fmt.Println("sender ",s.id,":",err)
					if _, ok := err.(*os.PathError); !ok {
						fileList.PushBack(filename)
					}
					loglib.Error(fmt.Sprintf("sender%d read file cache %s error:%s", s.id, filename, err.Error()))
				} else {

					packId := tcp_pack.GetPackId(data)                                                                          //debug info
					loglib.Info(fmt.Sprintf("sender%d read pack %s from file: %s, len: %d", s.id, packId, filename, len(data))) //debug info
					result := s.sendData2(data)
					if result == true {
						// s.fileCacheList.Remove(front)
						// log.Println("sender ",s.id,":removed file:",filename, "for pack", packId)//debug info
						err = os.Remove(filename)
						lib.CheckError(err)
						timeoutChan = time.After(time.Millisecond) //发送成功,不用再等待
					} else {
						fileList.PushBack(filename)
						// fmt.Println("sender ",s.id,": pushback file :",filename)
					}
				}
			}

		}
	}

}