func (s Sender) sendData(data []byte, conn *net.TCPConn) bool {
if len(data) == 0 {
return true
}
if conn == nil {
return false
}
/*
lenBuf := make([]byte, 4)
nData := len(data)
binary.PutUvarint(lenBuf, uint64(nData))
data = append(lenBuf, data...)
*/
st := time.Now()
packId := tcp_pack.GetPackId(data)
conn.SetDeadline(time.Now().Add(5 * time.Minute)) //设置超时
loglib.Info(fmt.Sprintf("sender%d start sending pack:%s length:%d", s.id, packId, len(data)))
n, err := conn.Write(data)
ed := time.Now()
loglib.Info(fmt.Sprintf("sender%d end sending pack:%s length:%d elapse:%s", s.id, packId, n, ed.Sub(st)))
lib.CheckError(err)
//写失败了就不用等应答了，肯定拿不到
if err == nil {
conn.SetReadDeadline(time.Now().Add(8 * time.Minute)) //设置超时
time1 := time.Now()
var temp []byte = make([]byte, 128)
count, err := conn.Read(temp)
if err == nil {
loglib.Info(fmt.Sprintf("sender%d get anwser data len:%d for pack:%s elapse:%s", s.id, count, packId, time.Now().Sub(time1)))
} else {
loglib.Info(fmt.Sprintf("sender%d get anwser data len:%d for pack:%s elapse:%s, error:%s", s.id, count, packId, time.Now().Sub(time1), err.Error()))
}
temp = temp[:count]
if string(temp) == "ok" { //发送成功
return true
} else if string(temp) == "wrong header" {
//包头错误,丢弃
loglib.Info(packId + " has wrong header, retry later!")
return false
} else { //发送失败
//报警
return false
}
} else {
loglib.Warning(fmt.Sprintf("write pack %s error:%s", packId, err.Error()))
}
return false
}
func (s *Sender) writeToFile(data bytes.Buffer) {
//写入文件
filename := createFileName(s.id)
//创建文件
_, err := os.Create(filename)
lib.CheckError(err)
d := data.Bytes()
packId := tcp_pack.GetPackId(d)
loglib.Info(fmt.Sprintf("sender%d save pack %s to file %s len:%d", s.id, packId, filename, len(d)))
err = ioutil.WriteFile(filename, d, 0666)
if err != nil {
loglib.Warning("write to file " + filename + " error:" + err.Error())
lib.CheckError(err)
} else {
//追加fileCacheList
fileList.PushBack(filename)
}
}
func (t *TcpReceiver) handleConnnection(conn net.Conn, wg *sync.WaitGroup) {
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("tcp receiver connection panic:%v", err))
}
conn.Close()
wg.Done()
}()
/*
用于标识收到退出信号后，能否直接退出
只要接收信号时，包没有收完，都是可退出的，
发送方会缓存以后重传;
如果收完了就不能直接退出，可能包已传给下一级处理但是
却告诉发送方发送失败
*/
var quit = false //用于标识是否要退出
go lib.HandleQuitSignal(func() {
//关闭连接，避免阻塞在网络io上
conn.Close()
quit = true
})
request := make([]byte, 512*1024) //缓冲为512k
var packLen int = 0
currLen := 0
var b = new(bytes.Buffer)
var content = new(bytes.Buffer)
inAddr := conn.RemoteAddr().String()
parts := strings.Split(inAddr, ":")
inIp := parts[0]
packId := "unkown"
var routeInfo map[string]string
var rePull = false //是否补拉，如果是补拉就不做重复包检验
loglib.Info("incoming: " + inAddr)
outer:
for !quit {
st := time.Now()
if packLen == 0 {
conn.SetReadDeadline(time.Now().Add(5 * time.Minute))
time1 := time.Now() //时间打点
// read zlib pack header length
buf := make([]byte, 4)
_, err := conn.Read(buf)
if err != nil {
loglib.Warning(fmt.Sprintf("conn:%s, get header len, tcp receiver read error:%s, elapse:%s", inAddr, err.Error(), time.Now().Sub(time1)))
break
}
l, _ := binary.Uvarint(buf)
headerLen := int(l)
//get pack header
headerBuf := make([]byte, headerLen)
time2 := time.Now()
_, err = conn.Read(headerBuf)
if err != nil {
loglib.Warning(fmt.Sprintf("conn:%s, get header, tcp receiver read error:%s, elapse:%s", inAddr, err.Error(), time.Now().Sub(time2)))
break
}
//是否补拉
route0 := tcp_pack.ParseHeader(headerBuf)
if v, ok := route0["repull"]; ok && v == "1" {
rePull = true
} else {
rePull = false
}
buf = append(buf, headerBuf...)
header, _, err := tcp_pack.ExtractHeader(buf)
if err != nil {
loglib.Error("wrong format header " + string(headerBuf) + " " + err.Error())
conn.Write([]byte("wrong header"))
break
}
packId = tcp_pack.GetPackId(buf)
packLen = header.PackLen
currLen = 0
routeInfo = make(map[string]string)
b = new(bytes.Buffer)
content = new(bytes.Buffer)
loglib.Info(fmt.Sprintf("conn:%s, start receive pack %s, pack len:%d, header len:%d, header elapse:%s", inAddr, packId, packLen, headerLen, time.Now().Sub(time1)))
b.Write(buf)
routeInfo["ip"] = lib.GetIp()
routeInfo["stage"] = "tcp recv"
routeInfo["st"] = st.Format("2006-01-02 15:04:05.000")
}
//读包体的超时
conn.SetReadDeadline(time.Now().Add(5 * time.Minute))
time3 := time.Now()
//read enough bytes
for currLen < packLen {
requestLen, err := conn.Read(request)
if requestLen == 0 || err != nil {
//sender有重发机制，所以可丢弃
packLen = 0 //设为0以便读取新的包
ed := time.Now()
loglib.Warning(fmt.Sprintf("conn:%s, not full! ip:%s, packid:%s, received:%d, end recv:%s, elapse:%s, body elapse:%s, error:%s", inAddr, inIp, packId, currLen, ed, ed.Sub(st), ed.Sub(time3), err.Error()))
break outer //连接出错直接跳出外层循环
}
currLen += requestLen
content.Write(request[:requestLen])
}
if packLen > 0 && currLen >= packLen {
//收完马上应答
_, err := conn.Write([]byte("ok"))
if err != nil {
loglib.Warning(fmt.Sprintf("ip:%s, packid:%s received, but response back error:%s", inIp, packId, err.Error()))
} else {
loglib.Info(fmt.Sprintf("conn:%s, response to packid:%s", inAddr, packId))
}
//避免收到重复包（补拉例外）
appeared, ok, code := t.hasAppeared(content)
if !ok || rePull {
ed := time.Now()
routeInfo["ed"] = ed.Format("2006-01-02 15:04:05.000")
routeInfo["elapse"] = ed.Sub(st).String()
b.Write(content.Bytes())
vbytes := tcp_pack.Packing(b.Bytes(), routeInfo, true)
b = bytes.NewBuffer(vbytes)
t.buffer <- *b
packAppear := PackAppear{time.Now().Unix(), packId}
t.mutex.Lock()
t.footPrint[code] = packAppear //这里挂过
t.mutex.Unlock()
loglib.Info(fmt.Sprintf("conn:%s, finish ip:%s, packid:%s, repull:%v, received:%d, elapse:%s, body elapse:%s", inAddr, inIp, packId, rePull, currLen, ed.Sub(st), ed.Sub(time3)))
} else {
loglib.Info(fmt.Sprintf("conn:%s, pack %s repeat %s already appear at %s", inAddr, packId, appeared.Id, time.Unix(appeared.Time, 0)))
}
packLen = 0
}
}
loglib.Info("conn finish: " + inAddr)
}
//goroutine
func (s *Sender) Start() {
// conn := s.getConnection()
//初始化fileCacheList
once.Do(s.reloadFileCache)
//收尾工作
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("sender %d panic:%v", s.id, err))
}
s.saveBufferInChan()
//s.saveMemCache()
s.connection.close()
s.wq.AllDone()
}()
go s.pickPacks()
//var connLost = 0
var quit = false
go lib.HandleQuitSignal(func() {
quit = true
s.connection.close()
})
var sendInterval = time.Duration(2000) //间隔稍大，避免发送文件缓存时因无连接或其他错误进入死循环
var timeoutChan = time.After(sendInterval * time.Millisecond)
for !quit {
select {
case b := <-s.memBuffer:
//send b
result := s.sendBuffer(b)
if result == false {
//改为直接放入文件缓存
s.writeToFile(b)
}
case <-timeoutChan:
timeoutChan = time.After(sendInterval * time.Millisecond)
// send from file
e := fileList.Remove()
if e != nil { // file list is not empty
filename := e.Value.(string)
// fmt.Println("sender ",s.id,": get file :",filename)
data, err := ioutil.ReadFile(filename)
if err != nil {
// fmt.Println("sender ",s.id,":",err)
if _, ok := err.(*os.PathError); !ok {
fileList.PushBack(filename)
}
loglib.Error(fmt.Sprintf("sender%d read file cache %s error:%s", s.id, filename, err.Error()))
} else {
packId := tcp_pack.GetPackId(data) //debug info
loglib.Info(fmt.Sprintf("sender%d read pack %s from file: %s, len: %d", s.id, packId, filename, len(data))) //debug info
result := s.sendData2(data)
if result == true {
// s.fileCacheList.Remove(front)
// log.Println("sender ",s.id,":removed file:",filename, "for pack", packId)//debug info
err = os.Remove(filename)
lib.CheckError(err)
timeoutChan = time.After(time.Millisecond) //发送成功，不用再等待
} else {
fileList.PushBack(filename)
// fmt.Println("sender ",s.id,": pushback file :",filename)
}
}
}
}
}
}