Exemple #1
0
// @deprecated
func DownloadDir(ctx *Context, remoteRoot string, localRoot string) (err error) {
	entries, err := ListPrefix(ctx, remoteRoot)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return err
	}
	if len(entries) == 0 {
		return ErrNoFile
	}
	// TODO 多线程提高性能.
	tm := kmgTask.NewLimitThreadErrorHandleTaskManager(ThreadNum, 3)
	defer tm.Close()
	for _, entry := range entries {
		entry := entry
		refPath, err := filepath.Rel(remoteRoot, entry.Key)
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return err
		}
		tm.AddTask(func() (err error) {
			err = DownloadFileWithHash(ctx, entry.Key, filepath.Join(localRoot, refPath), entry.Hash)
			return err
		})
	}
	tm.Wait()
	return nil
}
func LessDelayTcpConn(conn *net.TCPConn) (connOut net.Conn, err error) {
	//err = conn.SetKeepAlive(true)
	//if err!=nil{
	//	kmgErr.LogErrorWithStack(err)
	//	return nil,err
	//}
	//err = conn.SetKeepAlivePeriod(5*time.Second) //5s太小,耗流量非常凶残.
	//if err!=nil{
	//	kmgErr.LogErrorWithStack(err)
	//	return nil,err
	//}
	fd, err := conn.File()
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return
	}
	conn1, err := net.FileConn(fd)
	if err != nil {
		fd.Close()
		kmgErr.LogErrorWithStack(err)
		return
	}
	conn.Close()
	//尝试将连接重新设置回 block 模式,减少cpu占用,此方案不稳定,并且不知道如何解决不稳定的问题.
	//err = unix.SetNonblock(int(fd.Fd()),true)
	//if err!=nil{
	//	fd.Close()
	//	kmgErr.LogErrorWithStack(err)
	//	return nil,err
	//}
	//return NewDebugConn(fasterTcpConn{TCPConn: conn, fd: fd},conn.LocalAddr().String()+"_"+conn.RemoteAddr().String()), nil
	return &fasterTcpConn{TCPConn: conn1.(*net.TCPConn), fd: fd}, nil
}
Exemple #3
0
func (c *Context) sessionInit() {
	if c.sessionMap != nil {
		return
	}
	cookie, err := c.req.Cookie(SessionCookieName)
	if err != nil {
		//kmgErr.LogErrorWithStack(err)
		// 这个地方没有cookie是正常情况
		c.sessionMap = map[string]string{}
		//没有Cooke
		return
	}
	output, err := kmgCrypto.CompressAndEncryptBase64Decode(&SessionPsk, cookie.Value)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		c.sessionMap = map[string]string{}
		return
	}
	err = json.Unmarshal(output, &c.sessionMap)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		c.sessionMap = map[string]string{}
		return
	}
}
Exemple #4
0
// @deprecated
// TODO 这个会多建一个空目录,这个算安全隐患.
func DownloadFile(ctx *Context, remotePath string, localPath string) (err error) {
	err = kmgFile.MkdirForFile(localPath)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return
	}
	f, err := os.Create(localPath)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return
	}
	defer f.Close()
	return ctx.DownloadToWriter(remotePath, f)
}
Exemple #5
0
//下载单个文件,到本地,会覆盖本地已经存在的文件,会创建所有父级目录,会使用hash检查文件是否存在.
// @deprecated
func DownloadFileWithHash(ctx *Context, remotePath string, localPath string, hash string) (err error) {
	fhash, err := ComputeHashFromFile(localPath)
	if err == nil && fhash == hash {
		return
	}
	if err != nil && !os.IsNotExist(err) {
		kmgErr.LogErrorWithStack(err)
		return
	}
	if fhash == hash {
		return
	}
	return DownloadFile(ctx, remotePath, localPath)
}
Exemple #6
0
func (fm *FileMutex) getFd() int {
	if fm.f != nil {
		return int(fm.f.Fd())
	}
	if fm.Name == "" {
		panic(`no specialed file [Name] for locking,Example:l := &kmgProcessMutex.FileMutex{Name: "abc"}`)
	}
	fm.filePath = filepath.Join("/tmp", fm.Name)
	var err error
	fm.f, err = os.OpenFile(fm.filePath, os.O_CREATE|os.O_EXCL, os.FileMode(0777))
	if err == nil {
		return int(fm.f.Fd())
	}
	if os.IsExist(err) {
		fm.f, err = os.OpenFile(fm.filePath, os.O_RDONLY, os.FileMode(0777))
	} else {
		kmgErr.LogErrorWithStack(err)
	}
	return int(fm.f.Fd())
}
Exemple #7
0
// 列出所有前缀是xxx的数据,
// 已处理1000个限制
// 已看过sdk的文档,此处没有很简单的办法可以进行并发优化.
func ListPrefix(ctx *Context, prefix string) (entries []rsf.ListItem, err error) {
	var marker = ""
	for {
		var thisEntries []rsf.ListItem
		thisEntries, marker, err = ctx.rsfClient.ListPrefix(nil, ctx.bucket,
			prefix, marker, 1000)
		entries = append(entries, thisEntries...)
		if err == io.EOF {
			return entries, nil
		}
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return entries, err
		}
		if len(thisEntries) < 1000 {
			break
		}
	}
	return
}
func (conn *fasterTcpConn) Read(b []byte) (nr int, err error) {
	err = conn.TCPConn.SetReadDeadline(time.Now().Add(10 * time.Minute))
	if err != nil {
		return 0, err
	}
	nr, err = conn.TCPConn.Read(b)
	if err != nil {
		return nr, err
	}
	conn.closeLock.Lock()
	defer conn.closeLock.Unlock()
	fdNum := int(conn.fd.Fd())
	if fdNum == -1 { //已经关闭过了,此处不管了.
		return nr, err
	}
	err = unix.SetsockoptInt(fdNum, unix.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
	if err != nil { //TODO 此处总是会爆bad file descriptor,原因不明.
		kmgErr.LogErrorWithStack(err)
		return
	}
	return
}
Exemple #9
0
func ZipUncompressFromBytesToDir(zipB []byte, dir string, trimPrefix string) (err error) {
	buf := bytes.NewReader(zipB)
	reader, err := zip.NewReader(buf, int64(len(zipB)))
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return
	}
	for _, file := range reader.File {

		fullPath := filepath.Join(dir, strings.TrimPrefix(file.Name, trimPrefix))
		if file.FileInfo().IsDir() {
			err = kmgFile.Mkdir(fullPath)
			if err != nil {
				kmgErr.LogErrorWithStack(err)
				return
			}
			continue
		}
		err = kmgFile.MkdirForFile(fullPath)
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return
		}
		rc, err := file.Open()
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return err
		}
		f, err := os.Create(fullPath)
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			rc.Close()
			return err
		}
		_, err = io.Copy(f, rc)
		rc.Close()
		f.Close()
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return err
		}
	}
	return nil
}
Exemple #10
0
//杀不死就算了
func (p *Process) Kill() {
	sysProcess, err := os.FindProcess(p.Id)
	kmgErr.LogErrorWithStack(err)
	err = sysProcess.Kill()
	kmgErr.LogErrorWithStack(err)
}