// @deprecated func DownloadDir(ctx *Context, remoteRoot string, localRoot string) (err error) { entries, err := ListPrefix(ctx, remoteRoot) if err != nil { kmgErr.LogErrorWithStack(err) return err } if len(entries) == 0 { return ErrNoFile } // TODO 多线程提高性能. tm := kmgTask.NewLimitThreadErrorHandleTaskManager(ThreadNum, 3) defer tm.Close() for _, entry := range entries { entry := entry refPath, err := filepath.Rel(remoteRoot, entry.Key) if err != nil { kmgErr.LogErrorWithStack(err) return err } tm.AddTask(func() (err error) { err = DownloadFileWithHash(ctx, entry.Key, filepath.Join(localRoot, refPath), entry.Hash) return err }) } tm.Wait() return nil }
func LessDelayTcpConn(conn *net.TCPConn) (connOut net.Conn, err error) { //err = conn.SetKeepAlive(true) //if err!=nil{ // kmgErr.LogErrorWithStack(err) // return nil,err //} //err = conn.SetKeepAlivePeriod(5*time.Second) //5s太小,耗流量非常凶残. //if err!=nil{ // kmgErr.LogErrorWithStack(err) // return nil,err //} fd, err := conn.File() if err != nil { kmgErr.LogErrorWithStack(err) return } conn1, err := net.FileConn(fd) if err != nil { fd.Close() kmgErr.LogErrorWithStack(err) return } conn.Close() //尝试将连接重新设置回 block 模式,减少cpu占用,此方案不稳定,并且不知道如何解决不稳定的问题. //err = unix.SetNonblock(int(fd.Fd()),true) //if err!=nil{ // fd.Close() // kmgErr.LogErrorWithStack(err) // return nil,err //} //return NewDebugConn(fasterTcpConn{TCPConn: conn, fd: fd},conn.LocalAddr().String()+"_"+conn.RemoteAddr().String()), nil return &fasterTcpConn{TCPConn: conn1.(*net.TCPConn), fd: fd}, nil }
func (c *Context) sessionInit() { if c.sessionMap != nil { return } cookie, err := c.req.Cookie(SessionCookieName) if err != nil { //kmgErr.LogErrorWithStack(err) // 这个地方没有cookie是正常情况 c.sessionMap = map[string]string{} //没有Cooke return } output, err := kmgCrypto.CompressAndEncryptBase64Decode(&SessionPsk, cookie.Value) if err != nil { kmgErr.LogErrorWithStack(err) c.sessionMap = map[string]string{} return } err = json.Unmarshal(output, &c.sessionMap) if err != nil { kmgErr.LogErrorWithStack(err) c.sessionMap = map[string]string{} return } }
// @deprecated // TODO 这个会多建一个空目录,这个算安全隐患. func DownloadFile(ctx *Context, remotePath string, localPath string) (err error) { err = kmgFile.MkdirForFile(localPath) if err != nil { kmgErr.LogErrorWithStack(err) return } f, err := os.Create(localPath) if err != nil { kmgErr.LogErrorWithStack(err) return } defer f.Close() return ctx.DownloadToWriter(remotePath, f) }
//下载单个文件,到本地,会覆盖本地已经存在的文件,会创建所有父级目录,会使用hash检查文件是否存在. // @deprecated func DownloadFileWithHash(ctx *Context, remotePath string, localPath string, hash string) (err error) { fhash, err := ComputeHashFromFile(localPath) if err == nil && fhash == hash { return } if err != nil && !os.IsNotExist(err) { kmgErr.LogErrorWithStack(err) return } if fhash == hash { return } return DownloadFile(ctx, remotePath, localPath) }
func (fm *FileMutex) getFd() int { if fm.f != nil { return int(fm.f.Fd()) } if fm.Name == "" { panic(`no specialed file [Name] for locking,Example:l := &kmgProcessMutex.FileMutex{Name: "abc"}`) } fm.filePath = filepath.Join("/tmp", fm.Name) var err error fm.f, err = os.OpenFile(fm.filePath, os.O_CREATE|os.O_EXCL, os.FileMode(0777)) if err == nil { return int(fm.f.Fd()) } if os.IsExist(err) { fm.f, err = os.OpenFile(fm.filePath, os.O_RDONLY, os.FileMode(0777)) } else { kmgErr.LogErrorWithStack(err) } return int(fm.f.Fd()) }
// 列出所有前缀是xxx的数据, // 已处理1000个限制 // 已看过sdk的文档,此处没有很简单的办法可以进行并发优化. func ListPrefix(ctx *Context, prefix string) (entries []rsf.ListItem, err error) { var marker = "" for { var thisEntries []rsf.ListItem thisEntries, marker, err = ctx.rsfClient.ListPrefix(nil, ctx.bucket, prefix, marker, 1000) entries = append(entries, thisEntries...) if err == io.EOF { return entries, nil } if err != nil { kmgErr.LogErrorWithStack(err) return entries, err } if len(thisEntries) < 1000 { break } } return }
func (conn *fasterTcpConn) Read(b []byte) (nr int, err error) { err = conn.TCPConn.SetReadDeadline(time.Now().Add(10 * time.Minute)) if err != nil { return 0, err } nr, err = conn.TCPConn.Read(b) if err != nil { return nr, err } conn.closeLock.Lock() defer conn.closeLock.Unlock() fdNum := int(conn.fd.Fd()) if fdNum == -1 { //已经关闭过了,此处不管了. return nr, err } err = unix.SetsockoptInt(fdNum, unix.IPPROTO_TCP, unix.TCP_QUICKACK, 1) if err != nil { //TODO 此处总是会爆bad file descriptor,原因不明. kmgErr.LogErrorWithStack(err) return } return }
func ZipUncompressFromBytesToDir(zipB []byte, dir string, trimPrefix string) (err error) { buf := bytes.NewReader(zipB) reader, err := zip.NewReader(buf, int64(len(zipB))) if err != nil { kmgErr.LogErrorWithStack(err) return } for _, file := range reader.File { fullPath := filepath.Join(dir, strings.TrimPrefix(file.Name, trimPrefix)) if file.FileInfo().IsDir() { err = kmgFile.Mkdir(fullPath) if err != nil { kmgErr.LogErrorWithStack(err) return } continue } err = kmgFile.MkdirForFile(fullPath) if err != nil { kmgErr.LogErrorWithStack(err) return } rc, err := file.Open() if err != nil { kmgErr.LogErrorWithStack(err) return err } f, err := os.Create(fullPath) if err != nil { kmgErr.LogErrorWithStack(err) rc.Close() return err } _, err = io.Copy(f, rc) rc.Close() f.Close() if err != nil { kmgErr.LogErrorWithStack(err) return err } } return nil }
//杀不死就算了 func (p *Process) Kill() { sysProcess, err := os.FindProcess(p.Id) kmgErr.LogErrorWithStack(err) err = sysProcess.Kill() kmgErr.LogErrorWithStack(err) }