Example #1
0
func (this *MongoDbOutputer) runParse(routineId int, wg *sync.WaitGroup) {
	var session *mgo.Session
	//routine一般都要copy或clone session,clone能保证一致性
	if this.session != nil {
		session = this.session.Clone()
	}

	defer func() {
		if session != nil {
			session.Close()
		}
		wg.Done()
		loglib.Info(fmt.Sprintf("mongodb outputer parse routine %d quit", routineId))
	}()

	loglib.Info(fmt.Sprintf("mongodb outputer parse routine %d start", routineId))
	for b := range this.buffer {
		r, packId, date, err := this.extract(&b)
		if err == nil {
			if !this.isUpsert {
				this.bulkSave(&session, r, packId, date, routineId)
			} else {
				this.upsert(&session, r, packId, date, routineId)
			}
			r.Close()
		}
	}
}
Example #2
0
//用于检验session可用性并适时重连的routine func
//用于重连main session
func (this *MongoDbOutputer) reConnMongoDb() {
	nPingFail := 0 //ping失败次数
	reDial := false
	for {
		reDial = false
		if this.session == nil {
			//session未初始化
			reDial = true
		} else if this.session.Ping() != nil {
			//session连接丢失
			nPingFail++
			if nPingFail == 3 {
				reDial = true
			}
		}

		if reDial {
			nPingFail = 0
			this.session = initMongoDbSession(this.mongosAddr)
			if this.session == nil {
				loglib.Info("session re-dial failed!")
			} else {
				loglib.Info("session re-dial success!")
			}
		}
		time.Sleep(time.Second)
	}
}
Example #3
0
func (s *Sender) saveBufferInChan() {
	loglib.Info(fmt.Sprintf("sender%d begin to save pack in chan", s.id))
	i := 0
	for b := range s.memBuffer {
		s.writeToFile(b)
		i++
	}
	loglib.Info(fmt.Sprintf("sender%d saved num of pack in chan: %d", s.id, i))
}
Example #4
0
func (s Sender) sendData(data []byte, conn *net.TCPConn) bool {
	if len(data) == 0 {
		return true
	}

	if conn == nil {
		return false
	}
	/*
	   lenBuf := make([]byte, 4)
	   nData := len(data)
	   binary.PutUvarint(lenBuf, uint64(nData))
	   data = append(lenBuf, data...)
	*/

	st := time.Now()
	packId := tcp_pack.GetPackId(data)

	conn.SetDeadline(time.Now().Add(5 * time.Minute)) //设置超时
	loglib.Info(fmt.Sprintf("sender%d start sending pack:%s length:%d", s.id, packId, len(data)))
	n, err := conn.Write(data)
	ed := time.Now()
	loglib.Info(fmt.Sprintf("sender%d end sending pack:%s length:%d elapse:%s", s.id, packId, n, ed.Sub(st)))

	lib.CheckError(err)

	//写失败了就不用等应答了,肯定拿不到
	if err == nil {
		conn.SetReadDeadline(time.Now().Add(8 * time.Minute)) //设置超时
		time1 := time.Now()
		var temp []byte = make([]byte, 128)
		count, err := conn.Read(temp)
		if err == nil {
			loglib.Info(fmt.Sprintf("sender%d get anwser data len:%d for pack:%s elapse:%s", s.id, count, packId, time.Now().Sub(time1)))
		} else {
			loglib.Info(fmt.Sprintf("sender%d get anwser data len:%d for pack:%s elapse:%s, error:%s", s.id, count, packId, time.Now().Sub(time1), err.Error()))
		}

		temp = temp[:count]
		if string(temp) == "ok" { //发送成功
			return true
		} else if string(temp) == "wrong header" {
			//包头错误,丢弃
			loglib.Info(packId + " has wrong header, retry later!")
			return false
		} else { //发送失败
			//报警
			return false
		}
	} else {
		loglib.Warning(fmt.Sprintf("write pack %s error:%s", packId, err.Error()))
	}
	return false
}
Example #5
0
//处理注册请求
func (lr *LogReceiver) handleRegister(registerInfo map[string]string) map[string]string {
	req, ok := registerInfo["req"]
	m := map[string]string{"err": "-1", "msg": "unkown request <" + req + ">"}
	ip, _ := registerInfo["ip"]
	port, _ := registerInfo["port"]
	hostname, _ := registerInfo["hostname"]
	role, _ := registerInfo["role"]
	addr := ip
	loglib.Info(ip + ":" + port + " " + req)

	if ok && (req == "register" || req == "unregister") {
		if port != "" {
			addr = ip + ":" + port
		}
		if req == "register" {
			ret, err := lr.register(ip, port, role, hostname)
			if ret {
				lr.mutex.Lock()
				lr.ipRoleMap[addr] = role
				lr.mutex.Unlock()

				m["err"] = "0"
				m["msg"] = "success"
			} else {
				m["err"] = "1"
				if err != nil {
					m["msg"] = err.Error()
				} else {
					m["msg"] = "done"
				}
			}
		} else if req == "unregister" {
			ret, err := lr.unRegister(ip, port)
			if ret {
				lr.mutex.Lock()
				delete(lr.ipRoleMap, addr)
				lr.mutex.Unlock()

				m["err"] = "0"
				m["msg"] = "success"
			} else {
				m["err"] = "1"
				if err != nil {
					m["msg"] = err.Error()
				} else {
					m["msg"] = "done"
				}
			}
		}
	}
	loglib.Info("[monitor] " + ip + ":" + port + " " + role + " " + req + ": " + m["msg"])
	return m
}
Example #6
0
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, hostsList string, ipBlackList string, wg *sync.WaitGroup) {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
		}

		wg.Done()
	}()
	loglib.Info("etl routine start")
	for fkey := range fkeyChan {
		sv := etl.NewFileSaver(colsFile, etlDir, fkey)
		d := etl.NewDispatcher(sv, 6, hostsList, ipBlackList)
		g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
		go g.Start(false)

		fname := filepath.Join(logDataDir, fkey)
		loglib.Info("start etl for " + fname)

		err := g.ParseFile(fname)
		g.Wait()
		// etl success
		// mark success
		if err == nil {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("finish etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
				}
			}
		} else {
			//采用循环,增加打tag的成功率
			for i := 0; i < 5; i++ {
				fd, err := os.Create(filepath.Join(etlFailDir, fkey))
				if err == nil {
					fd.Close()
					loglib.Info("failed etl for " + fname)
					break
				} else {
					loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
				}
			}

		}
	}
	loglib.Info("etl routine finish")
}
Example #7
0
func (this *WaitQuit) Quit() bool {
	ret := false
	select {
	case <-this.ch:
		loglib.Info(this.name + " safe quit.")
		ret = true
	case <-time.After(2 * time.Second):
		loglib.Info(this.name + " quit timeout")
		this.nTimeout++
		if this.allowTimeout > 0 && this.nTimeout >= this.allowTimeout {
			ret = true
		}
	}
	return ret

}
Example #8
0
func (t *TcpReceiver) clearFootPrint(wg *sync.WaitGroup) {
	defer wg.Done()

	ch1 := make(chan bool)       //用于安全退出
	ch2 := time.After(time.Hour) //用于定时任务
	go lib.HandleQuitSignal(func() {
		ch1 <- true
	})

loop:
	for {
		select {
		//监听一个chan以便安全退出
		case <-ch1:
			break loop
		case <-ch2:
			//若这个case未执行完,而ch1已可读,select会保证这个case执行完
			now := time.Now().Unix()
			t.mutex.Lock()
			for code, appear := range t.footPrint {
				if now-appear.Time >= 86400 {
					delete(t.footPrint, code)
				}
			}
			t.saveFootPrint()
			t.mutex.Unlock()
			ch2 = time.After(time.Hour) //用于定时任务
		}
	}
	loglib.Info("clear footprint quit!")
}
Example #9
0
//载入cache文件列表
func (this *MongoDbOutputer) reloadFileCache() {
	list := lib.GetFilelist(this.file_mem_folder_name)
	for _, filename := range list {
		loglib.Info("reloading:" + filename)
		this.fileList.PushBack(filename)
	}
}
Example #10
0
//should be run by once
func (s *Sender) reloadFileCache() {
	list := lib.GetFilelist(s.file_mem_folder_name)
	for _, filename := range list {
		// s.fileCacheList.PushBack(filename)
		loglib.Info("reloading:" + filename)
		fileList.PushBack(filename)
	}
}
Example #11
0
func (t *TcpReceiver) Start() {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress)
	lib.CheckError(err)
	listener, err := net.ListenTCP("tcp", tcpAddr)
	lib.CheckError(err)

	wg := &sync.WaitGroup{}

	wg.Add(1)
	go t.clearFootPrint(wg)

	//主routine信号处理
	go lib.HandleQuitSignal(func() {
		//接收到信号关闭listenner,此时Accept会马上返回一个nil 的conn
		listener.Close()
		loglib.Info("close tcp receiver's listener.")
	})

	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err))
		}

		loglib.Info("wait connections finish...")
		wg.Wait()
		loglib.Info("all connections have been processed. quit.")
		close(t.buffer) //关闭chan
		t.saveFootPrint()

		t.wq.AllDone()

	}()

	for {
		conn, err := listener.Accept()
		if conn == nil {
			break
		}
		lib.CheckError(err)
		wg.Add(1)
		go t.handleConnnection(conn, wg)
	}

}
Example #12
0
func (f *fileOutputer) extract(bp *bytes.Buffer) {
	buf := make([]byte, 4)
	bp.Read(buf)

	l, _ := binary.Uvarint(buf)
	headerLen := int(l)
	//get pack header
	buf = make([]byte, headerLen)
	bp.Read(buf)
	header := tcp_pack.ParseHeader(buf)

	r, err := zlib.NewReader(bp)
	if err != nil {
		loglib.Error("zlib reader Error: " + err.Error())
	} else {
		lines, _ := strconv.Atoi(header["lines"])
		done := false
		if header["done"] == "1" {
			done = true
		}
		f.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

		writerKey := header["ip"] + "_" + header["hour"]
		fout := f.getWriter(f.writers, f.dataDir, writerKey)

		//一头一尾写头信息,节省硬盘
		buf = append(buf, '\n')
		//fout.Write(buf)
		nn, err := io.Copy(fout, r)
		if err != nil {
			loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
		}
		//fout.Write(buf)

		//单独存一份header便于查数
		fout = f.getWriter(f.headerWriters, f.headerDir, writerKey)
		n, err := fout.Write(buf)
		if err != nil {
			loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
		}

		if done || time.Now().Unix() > f.checkTime.Unix() {
			hourFinish, _ := f.ic.Check()
			for ip, hours := range hourFinish {
				for _, hour := range hours {
					writerKey = ip + "_" + hour
				}
			}
			f.closeWriters(f.writers)
			f.closeWriters(f.headerWriters)
			f.checkTime.Add(2 * time.Minute)
		}

		r.Close()
	}
}
Example #13
0
func (s *Sender) sendData2(data []byte) bool {
	result := s.sendData(data, s.connection.getConn())
	//发送失败,tcp连接可能已经失效,重新建立tcp连接
	if result == false {
		s.connection.reconnect(s.connection.getConn())
		*s.status = -1
		loglib.Info(fmt.Sprintf("sender%d reconnected by sendData2(),status:%d", s.id, *s.status))
	}
	return result
}
Example #14
0
//从公用的chan读pack到私有的chan,若私有chan已满则写入文件缓存
//保证公用chan不会阻塞
func (s *Sender) pickPacks() {
	for buf := range s.sBuffer {
		select {
		case s.memBuffer <- buf:
			break
		default:
			loglib.Info(fmt.Sprintf("sender%d mem buffer is full, total %d, pub chan:%d", s.id, len(s.memBuffer), len(s.sBuffer)))
			s.writeToFile(buf)
		}
	}
	close(s.memBuffer)
}
Example #15
0
func tailerGo(cfg map[string]map[string]string) {
	qlst := lib.NewQuitList()

	receiveChan := make(chan map[string]string, 10000) //非阻塞
	sendBuffer := make(chan bytes.Buffer, 500)
	recvBufferSize, _ := strconv.Atoi(cfg["tail"]["recv_buffer_size"])
	tailler := NewTailler(cfg["tail"])
	r := ReceiverInit(sendBuffer, receiveChan, recvBufferSize, tailler.GetLineNum())

	//make a new log tailler
	go tailler.Tailling(receiveChan)
	//start receiver to receive log
	go r.Start()

	//一定要发送方先退出
	qlst.Append(tailler.Quit)
	qlst.Append(r.Quit)
	// heart beat
	port, _ := cfg["monitor"]["hb_port"]
	monAddr, _ := cfg["monitor"]["mon_addr"]
	if port != "" && monAddr != "" {
		hb := heart_beat.NewHeartBeat(port, monAddr, "tail")
		go hb.Run()
		qlst.Append(hb.Quit)
	}

	addrs := strings.Split(cfg["tail"]["send_to"], ",")
	addr := strings.Trim(addrs[0], " ")
	bakAddr := addr
	//有备用地址?
	if len(addrs) > 1 {
		bakAddr = strings.Trim(addrs[1], " ")
	}

	//加大发送并发,sender阻塞会影响tail的进度
	nSenders := 2
	senders, ok := cfg["tail"]["senders"]
	if ok {
		tmp, err := strconv.Atoi(senders)
		if err == nil {
			nSenders = tmp
		}
	}
	for i := 1; i <= nSenders; i++ {
		s := SenderInit(sendBuffer, addr, bakAddr, i)
		go s.Start()
		qlst.Append(s.Quit)
	}
	loglib.Info(fmt.Sprintf("total senders %d", nSenders))

	qlst.HandleQuitSignal()
	qlst.ExecQuit()
}
Example #16
0
//更新插入,按字段更新
func (this *MongoDbOutputer) upsert(psession **mgo.Session, r io.Reader, packId string, date string, routineId int) {
	nDiscard := 0
	nCached := 0
	nUpdated := 0
	nInserted := 0
	var coll *mgo.Collection = nil

	if *psession != nil {
		coll = (*psession).DB(this.db + date).C(this.collection) //按天分库
	}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		m := this.parseLogLine(line)
		if len(m) > 0 {
			selector := bson.M{this.transactionIdKey: m[this.transactionIdKey]}
			up := bson.M{"$set": m}
			info, err := this.upsertBson(coll, selector, up)
			if err != nil {
				this.cacheData(m, "upsert", date, routineId)
				nCached++
				//ping fail, re-connect, clone main session
				if (*psession).Ping() != nil {
					//refresh go-routine's session if possible
					this.reCloneRoutineSession(psession)
					if (*psession).Ping() == nil {
						loglib.Info(fmt.Sprintf("parse routine %d re-conn", routineId))
					}
				}
			} else {
				nInserted++
				nUpdated += info.Updated
			}
		} else {
			nDiscard++
		}
	}

	loglib.Info(fmt.Sprintf("save pack %s: inserted:%d, updated:%d, cached:%d, discard %d items", packId, nInserted, nUpdated, nCached, nDiscard))
}
Example #17
0
//保存footprint
func (t *TcpReceiver) saveFootPrint() {
	vbytes, err := json.Marshal(t.footPrint)
	if err != nil {
		loglib.Error("marshal footprint error:" + err.Error())
		return
	}
	err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664)
	if err == nil {
		loglib.Info("save footprint success !")
	} else {
		loglib.Error("save footprint error:" + err.Error())
	}
}
Example #18
0
func saveLineRecord(path string, fname string, lineNum int) {
	fout, err := os.Create(path)
	defer fout.Close()
	if err != nil {
		loglib.Error("save line record error: " + err.Error())
		return
	}
	_, err = fmt.Fprintf(fout, "%s %d", fname, lineNum)
	if err != nil {
		loglib.Error("Write line record error" + err.Error())
		return
	}
	loglib.Info("save line record success!")
}
Example #19
0
//处理日志上报
func (lr *LogReceiver) handleLog(logInfo map[string]string) {
	ip, ok := logInfo["ip"]
	port, ok := logInfo["port"]
	remoteAddr := ip
	if port != "" {
		remoteAddr += ":" + port
	}
	role, ok := lr.ipRoleMap[remoteAddr]
	if !ok {
		role = ""
	}
	res, err := lr.AddLog(logInfo["time"], remoteAddr, role, logInfo["type"], logInfo["msg"])
	if err == nil {
		loglib.Info(fmt.Sprintf("[log receiver] add %d error log", res.NumRows))
	}

}
Example #20
0
func collectorGo(cfg map[string]map[string]string) {
	qlst := lib.NewQuitList()

	bufferChan := make(chan bytes.Buffer, 500)
	rAddr := cfg["collector"]["listen"]
	tr := TcpReceiverInit(bufferChan, rAddr)
	go tr.Start()

	qlst.Append(tr.Quit)

	addrs := strings.Split(cfg["collector"]["send_to"], ",")
	addr := addrs[0]
	bakAddr := addr
	//有备用地址?
	if len(addrs) > 1 {
		bakAddr = addrs[1]
	}

	nSenders := 10
	senders, ok := cfg["collector"]["senders"]
	if ok {
		tmp, err := strconv.Atoi(senders)
		if err == nil {
			nSenders = tmp
		}
	}
	for i := 1; i <= nSenders; i++ {
		s := SenderInit(bufferChan, addr, bakAddr, i)
		go s.Start()
		qlst.Append(s.Quit)
	}
	loglib.Info(fmt.Sprintf("total senders %d", nSenders))

	// heart beat
	port, _ := cfg["monitor"]["hb_port"]
	monAddr, _ := cfg["monitor"]["mon_addr"]
	if port != "" && monAddr != "" {
		hb := heart_beat.NewHeartBeat(port, monAddr, "collector")
		go hb.Run()
		qlst.Append(hb.Quit)
	}

	qlst.HandleQuitSignal()
	qlst.ExecQuit()
}
Example #21
0
func createSingleConnection(address string) (conn *net.TCPConn, err error) {

	tcpAddr, err := net.ResolveTCPAddr("tcp4", address)
	lib.CheckError(err)
	if err != nil {
		return nil, err
	}
	conn, err = net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		loglib.Error("get connection from " + address + " failed! Error:" + err.Error())
		return nil, err
	} else {
		loglib.Info("get connection from " + address + " success! remote addr " + conn.RemoteAddr().String())
	}
	lib.CheckError(err)

	return conn, nil
}
Example #22
0
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
	fp := make(map[string]PackAppear)
	if lib.FileExists(fname) {
		vbytes, err := ioutil.ReadFile(fname)
		if err != nil {
			loglib.Error("read footprint file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &fp)
			if err != nil {
				loglib.Error("unmarshal footprint error:" + err.Error())
			} else {
				loglib.Info("load footprint success !")
			}
		}
	} else {
		loglib.Warning("footprint file " + fname + " not found!")
	}
	return fp
}
Example #23
0
func getIpRoleMap(dbConn *db.Mysql) map[string]string {
	sql := "select ip, port, role from " + registerTable
	res, err := dbConn.Query(sql)
	m := make(map[string]string)
	if err != nil {
		loglib.Error("read registered nodes failed! Error: " + err.Error())
	} else {
		for _, row := range res.Rows {
			ip := row[0]
			port := row[1]
			role := row[2]
			if port != "" {
				ip = ip + ":" + port
			}
			m[ip] = role
		}
		loglib.Info("readed working nodes from database.")
	}
	return m
}
Example #24
0
func (s *Sender) writeToFile(data bytes.Buffer) {
	//写入文件
	filename := createFileName(s.id)
	//创建文件
	_, err := os.Create(filename)
	lib.CheckError(err)

	d := data.Bytes()

	packId := tcp_pack.GetPackId(d)

	loglib.Info(fmt.Sprintf("sender%d save pack %s to file %s len:%d", s.id, packId, filename, len(d)))
	err = ioutil.WriteFile(filename, d, 0666)
	if err != nil {
		loglib.Warning("write to file " + filename + " error:" + err.Error())
		lib.CheckError(err)
	} else {
		//追加fileCacheList
		fileList.PushBack(filename)
	}
}
Example #25
0
func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) {
	m, _ := selector.(bson.M)
	tid, _ := m[this.transactionIdKey].(string)

	if coll != nil {
		info, err = coll.Upsert(selector, doc)

		if err != nil {
			loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
		} else {
			if info.Updated > 0 {
				loglib.Info(fmt.Sprintf("bson [%s] updated", tid))
			}
		}
	} else {
		info = &mgo.ChangeInfo{}
		err = errors.New("upsert: collection is nil")
		loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
	}
	return
}
Example #26
0
//缓存写入mongodb失败的数据
//typeStr为bulk或upsert
func (this *MongoDbOutputer) cacheData(data interface{}, typeStr string, date string, routineId int) {
	mp := bson.M{"type": typeStr, "date": date, "data": data}
	saveTry := 3
	b, err := json.Marshal(mp)
	arr, ok := data.([]bson.M)
	cnt := 1
	if ok {
		cnt = len(arr)
	}
	if err != nil {
		loglib.Error(fmt.Sprintf("cache data error when marshal, discard %d item(s), error:%v", cnt, err))
		return
	}
	fname := this.createFileName(routineId)
	for i := 0; i < saveTry; i++ {
		err = ioutil.WriteFile(fname, b, 0666)
		if err == nil {
			this.fileList.PushBack(fname)
			loglib.Info(fmt.Sprintf("cache %d bson", cnt))
			break
		}
	}
}
Example #27
0
func (this *HeartBeat) Run() {
	i := 0

	//尝试注册3次
	for i < 3 && !this.registerSelf(true) {
		i++
		time.Sleep(10 * time.Second)
	}

	l, err := net.Listen("tcp", ":"+this.port)
	if err != nil {
		loglib.Error("heart beat " + err.Error())
		return
	}
	defer l.Close()

	go lib.HandleQuitSignal(func() {
		this.registerSelf(false)
		l.Close()
		loglib.Info("close heart beat listener.")
	})
	//heart beat 不是太重要,所以退出处理比较简单
	for {
		conn, err := l.Accept()
		if conn == nil {
			//listener关闭
			break
		}
		if err != nil {
			loglib.Error("heart beat " + err.Error())
			break
		}
		go this.handleConnection(conn)
	}

	this.wq.AllDone()
}
Example #28
0
//向monitor注册或取消注册,reg为true表示注册,否则是取消注册,返回true表示成功
func (this *HeartBeat) registerSelf(reg bool) bool {
	conn, err := lib.GetConnection(this.monitorAddr)
	if err != nil {
		return false
	}
	defer conn.Close()

	req := "register"
	if !reg {
		req = "unregister"
	}
	m := map[string]string{"req": req, "ip": lib.GetIp(), "port": this.port, "hostname": lib.GetHostname(), "role": this.role}
	msg, err := json.Marshal(m)
	if err != nil {
		loglib.Error("marshal " + req + " info error " + err.Error())
		return false
	}
	_, err = conn.Write(tcp_pack.Pack(msg))
	if err != nil {
		loglib.Error("send " + req + " info failed" + err.Error())
		return false
	} else {
		plen, ret := tcp_pack.UnPack(conn)

		if plen > 0 {
			err = json.Unmarshal(ret, &m)
			r, ok := m["err"]
			if err == nil && ok && r == "0" {
				loglib.Info(req + " to monitor success!")
				return true
			}
			loglib.Error(req + " heart beat failed!")
		}
	}
	return false
}
Example #29
0
//goroutine
func (s *Sender) Start() {
	// conn := s.getConnection()
	//初始化fileCacheList
	once.Do(s.reloadFileCache)

	//收尾工作
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("sender %d panic:%v", s.id, err))
		}

		s.saveBufferInChan()

		//s.saveMemCache()

		s.connection.close()

		s.wq.AllDone()

	}()

	go s.pickPacks()
	//var connLost = 0
	var quit = false
	go lib.HandleQuitSignal(func() {
		quit = true
		s.connection.close()
	})

	var sendInterval = time.Duration(2000) //间隔稍大,避免发送文件缓存时因无连接或其他错误进入死循环

	var timeoutChan = time.After(sendInterval * time.Millisecond)
	for !quit {

		select {
		case b := <-s.memBuffer:
			//send b
			result := s.sendBuffer(b)
			if result == false {
				//改为直接放入文件缓存
				s.writeToFile(b)
			}

		case <-timeoutChan:
			timeoutChan = time.After(sendInterval * time.Millisecond)

			// send from file
			e := fileList.Remove()
			if e != nil { // file list is not empty
				filename := e.Value.(string)
				// fmt.Println("sender ",s.id,": get file :",filename)
				data, err := ioutil.ReadFile(filename)
				if err != nil {
					// fmt.Println("sender ",s.id,":",err)
					if _, ok := err.(*os.PathError); !ok {
						fileList.PushBack(filename)
					}
					loglib.Error(fmt.Sprintf("sender%d read file cache %s error:%s", s.id, filename, err.Error()))
				} else {

					packId := tcp_pack.GetPackId(data)                                                                          //debug info
					loglib.Info(fmt.Sprintf("sender%d read pack %s from file: %s, len: %d", s.id, packId, filename, len(data))) //debug info
					result := s.sendData2(data)
					if result == true {
						// s.fileCacheList.Remove(front)
						// log.Println("sender ",s.id,":removed file:",filename, "for pack", packId)//debug info
						err = os.Remove(filename)
						lib.CheckError(err)
						timeoutChan = time.After(time.Millisecond) //发送成功,不用再等待
					} else {
						fileList.PushBack(filename)
						// fmt.Println("sender ",s.id,": pushback file :",filename)
					}
				}
			}

		}
	}

}
Example #30
0
//goroutine
//clear list & zipping & send_to_buffer
func (r Receiver) writeList() {
	//收尾工作
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("receiver panic:%v", err))
		}
		close(r.sendBuffer)
	}()

	st := time.Now()
	var nLines = 0
	var id = r.initId()
	ip := lib.GetIp()
	var changed = false

	for logMap := range r.receiveChan {
		logLine := logMap["line"]
		changed = false

		if logLine == "logfile changed" {
			changed = true
		} else {
			r.logList.PushBack(logLine)
		}
		nLines = r.logList.Len()
		//达到指定行数或发现日志rotate
		//因此每小时只有最后一个包比listBufferSize小
		//如果quit时包小于listBufferSize就丢弃,重启后再读
		if nLines >= r.listBufferSize || changed {
			hour := logMap["hour"]
			repull, ok := logMap["repull"] //兼容补拉

			b := r.clearList()
			//r.sendBuffer <- b
			ed := time.Now()
			elapse := ed.Sub(st)
			loglib.Info(fmt.Sprintf("add a pack, id: %s_%d, lines:%d, elapse: %s", hour, id, nLines, elapse))

			//route信息
			m := make(map[string]string)
			m["ip"] = ip
			m["hour"] = hour
			m["id"] = fmt.Sprintf("%d", id)
			m["lines"] = fmt.Sprintf("%d", nLines)
			m["stage"] = "make pack"
			m["st"] = st.Format("2006-01-02 15:04:05.000")
			m["ed"] = ed.Format("2006-01-02 15:04:05.000")
			m["elapse"] = elapse.String()
			if ok && repull == "1" {
				m["repull"] = "1"
			}

			if changed {
				m["done"] = "1"
				//这种空包用于给那些日志行数正好是listBufferSize倍数的小时标记结束
				//设置repull为1以便空包能够不被拦截
				if nLines == 0 {
					m["repull"] = "1"
				}
			}

			vbytes := tcp_pack.Packing(b.Bytes(), m, false)
			b.Reset()
			b.Write(vbytes)
			r.sendBuffer <- b
			id++
			st = time.Now()
			nLines = 0
		}

		if changed {
			id = 1 //每小时id刷新
		}

	}

	if nLines > 0 {
		loglib.Info(fmt.Sprintf("receiver abandon %d lines", nLines))
	}

}