// @deprecated
func DownloadDir(ctx *Context, remoteRoot string, localRoot string) (err error) {
entries, err := ListPrefix(ctx, remoteRoot)
if err != nil {
kmgErr.LogErrorWithStack(err)
return err
}
if len(entries) == 0 {
return ErrNoFile
}
// TODO 多线程提高性能.
tm := kmgTask.NewLimitThreadErrorHandleTaskManager(ThreadNum, 3)
defer tm.Close()
for _, entry := range entries {
entry := entry
refPath, err := filepath.Rel(remoteRoot, entry.Key)
if err != nil {
kmgErr.LogErrorWithStack(err)
return err
}
tm.AddTask(func() (err error) {
err = DownloadFileWithHash(ctx, entry.Key, filepath.Join(localRoot, refPath), entry.Hash)
return err
})
}
tm.Wait()
return nil
}
func LessDelayTcpConn(conn *net.TCPConn) (connOut net.Conn, err error) {
//err = conn.SetKeepAlive(true)
//if err!=nil{
// kmgErr.LogErrorWithStack(err)
// return nil,err
//}
//err = conn.SetKeepAlivePeriod(5*time.Second) //5s太小,耗流量非常凶残.
//if err!=nil{
// kmgErr.LogErrorWithStack(err)
// return nil,err
//}
fd, err := conn.File()
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
conn1, err := net.FileConn(fd)
if err != nil {
fd.Close()
kmgErr.LogErrorWithStack(err)
return
}
conn.Close()
//尝试将连接重新设置回 block 模式,减少cpu占用,此方案不稳定,并且不知道如何解决不稳定的问题.
//err = unix.SetNonblock(int(fd.Fd()),true)
//if err!=nil{
// fd.Close()
// kmgErr.LogErrorWithStack(err)
// return nil,err
//}
//return NewDebugConn(fasterTcpConn{TCPConn: conn, fd: fd},conn.LocalAddr().String()+"_"+conn.RemoteAddr().String()), nil
return &fasterTcpConn{TCPConn: conn1.(*net.TCPConn), fd: fd}, nil
}
func (c *Context) sessionInit() {
if c.sessionMap != nil {
return
}
cookie, err := c.req.Cookie(SessionCookieName)
if err != nil {
//kmgErr.LogErrorWithStack(err)
// 这个地方没有cookie是正常情况
c.sessionMap = map[string]string{}
//没有Cooke
return
}
output, err := kmgCrypto.CompressAndEncryptBase64Decode(&SessionPsk, cookie.Value)
if err != nil {
kmgErr.LogErrorWithStack(err)
c.sessionMap = map[string]string{}
return
}
err = json.Unmarshal(output, &c.sessionMap)
if err != nil {
kmgErr.LogErrorWithStack(err)
c.sessionMap = map[string]string{}
return
}
}
// @deprecated
// TODO 这个会多建一个空目录,这个算安全隐患.
func DownloadFile(ctx *Context, remotePath string, localPath string) (err error) {
err = kmgFile.MkdirForFile(localPath)
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
f, err := os.Create(localPath)
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
defer f.Close()
return ctx.DownloadToWriter(remotePath, f)
}
//下载单个文件,到本地,会覆盖本地已经存在的文件,会创建所有父级目录,会使用hash检查文件是否存在.
// @deprecated
func DownloadFileWithHash(ctx *Context, remotePath string, localPath string, hash string) (err error) {
fhash, err := ComputeHashFromFile(localPath)
if err == nil && fhash == hash {
return
}
if err != nil && !os.IsNotExist(err) {
kmgErr.LogErrorWithStack(err)
return
}
if fhash == hash {
return
}
return DownloadFile(ctx, remotePath, localPath)
}
func (fm *FileMutex) getFd() int {
if fm.f != nil {
return int(fm.f.Fd())
}
if fm.Name == "" {
panic(`no specialed file [Name] for locking，Example：l := &kmgProcessMutex.FileMutex{Name: "abc"}`)
}
fm.filePath = filepath.Join("/tmp", fm.Name)
var err error
fm.f, err = os.OpenFile(fm.filePath, os.O_CREATE|os.O_EXCL, os.FileMode(0777))
if err == nil {
return int(fm.f.Fd())
}
if os.IsExist(err) {
fm.f, err = os.OpenFile(fm.filePath, os.O_RDONLY, os.FileMode(0777))
} else {
kmgErr.LogErrorWithStack(err)
}
return int(fm.f.Fd())
}
// 列出所有前缀是xxx的数据,
// 已处理1000个限制
// 已看过sdk的文档,此处没有很简单的办法可以进行并发优化.
func ListPrefix(ctx *Context, prefix string) (entries []rsf.ListItem, err error) {
var marker = ""
for {
var thisEntries []rsf.ListItem
thisEntries, marker, err = ctx.rsfClient.ListPrefix(nil, ctx.bucket,
prefix, marker, 1000)
entries = append(entries, thisEntries...)
if err == io.EOF {
return entries, nil
}
if err != nil {
kmgErr.LogErrorWithStack(err)
return entries, err
}
if len(thisEntries) < 1000 {
break
}
}
return
}
func (conn *fasterTcpConn) Read(b []byte) (nr int, err error) {
err = conn.TCPConn.SetReadDeadline(time.Now().Add(10 * time.Minute))
if err != nil {
return 0, err
}
nr, err = conn.TCPConn.Read(b)
if err != nil {
return nr, err
}
conn.closeLock.Lock()
defer conn.closeLock.Unlock()
fdNum := int(conn.fd.Fd())
if fdNum == -1 { //已经关闭过了,此处不管了.
return nr, err
}
err = unix.SetsockoptInt(fdNum, unix.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
if err != nil { //TODO 此处总是会爆bad file descriptor,原因不明.
kmgErr.LogErrorWithStack(err)
return
}
return
}
func ZipUncompressFromBytesToDir(zipB []byte, dir string, trimPrefix string) (err error) {
buf := bytes.NewReader(zipB)
reader, err := zip.NewReader(buf, int64(len(zipB)))
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
for _, file := range reader.File {
fullPath := filepath.Join(dir, strings.TrimPrefix(file.Name, trimPrefix))
if file.FileInfo().IsDir() {
err = kmgFile.Mkdir(fullPath)
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
continue
}
err = kmgFile.MkdirForFile(fullPath)
if err != nil {
kmgErr.LogErrorWithStack(err)
return
}
rc, err := file.Open()
if err != nil {
kmgErr.LogErrorWithStack(err)
return err
}
f, err := os.Create(fullPath)
if err != nil {
kmgErr.LogErrorWithStack(err)
rc.Close()
return err
}
_, err = io.Copy(f, rc)
rc.Close()
f.Close()
if err != nil {
kmgErr.LogErrorWithStack(err)
return err
}
}
return nil
}
//杀不死就算了
func (p *Process) Kill() {
sysProcess, err := os.FindProcess(p.Id)
kmgErr.LogErrorWithStack(err)
err = sysProcess.Kill()
kmgErr.LogErrorWithStack(err)
}
