Example #1
0
// @deprecated
func DownloadDir(ctx *Context, remoteRoot string, localRoot string) (err error) {
	entries, err := ListPrefix(ctx, remoteRoot)
	if err != nil {
		kmgErr.LogErrorWithStack(err)
		return err
	}
	if len(entries) == 0 {
		return ErrNoFile
	}
	// TODO 多线程提高性能.
	tm := kmgTask.NewLimitThreadErrorHandleTaskManager(ThreadNum, 3)
	defer tm.Close()
	for _, entry := range entries {
		entry := entry
		refPath, err := filepath.Rel(remoteRoot, entry.Key)
		if err != nil {
			kmgErr.LogErrorWithStack(err)
			return err
		}
		tm.AddTask(func() (err error) {
			err = DownloadFileWithHash(ctx, entry.Key, filepath.Join(localRoot, refPath), entry.Hash)
			return err
		})
	}
	tm.Wait()
	return nil
}
Example #2
0
//多线程上传目录
//1.某个文件仅在一个线程中上传,
//2.检查同名和同内容的文件是否已经存在了,如果存在,且hash相同便不上传(断点续传)
func UploadDirMulitThread(ctx *Context, localRoot string, remoteRoot string) (err error) {
	tm := kmgTask.NewLimitThreadErrorHandleTaskManager(5, 3)
	defer tm.Close()
	requestList := []uploadFileRequest{}
	//dispatch task 分配任务
	err = filepath.Walk(localRoot, func(path string, info os.FileInfo, inErr error) (err error) {
		if inErr != nil {
			return inErr
		}
		if info.IsDir() {
			return
		}
		relPath, err := filepath.Rel(localRoot, path)
		if err != nil {
			return fmt.Errorf("[qiniuUploadDir] filepath.Rel err:%s", err.Error())
		}
		remotePath := filepath.Join(remoteRoot, relPath)
		expectHash, err := ComputeHashFromFile(path)
		if err != nil {
			return
		}
		requestList = append(requestList, uploadFileRequest{
			localPath:  path,
			remotePath: NormalizeRemotePath(remotePath),
			expectHash: expectHash,
		})
		return
	})
	//群发状态询问消息减少网络连接数量,加快速度
	entryPathList := make([]rs.EntryPath, len(requestList))
	for i, req := range requestList {
		entryPathList[i].Bucket = ctx.bucket
		entryPathList[i].Key = req.remotePath
	}
	batchRet, _ := ctx.client.BatchStat(nil, entryPathList)
	/*
		//此处返回的错误很奇怪,有大量文件不存在信息,应该是正常情况,此处最简单的解决方案就是假设没有错误
		if err != nil{
			fmt.Printf("%T %#v\n",err,err)
			err1,ok:=err.(*rpc.ErrorInfo)
			if !ok{
				return err
			}
			if err1.Code!=298{
				return err
			}
		}
	*/
	if len(batchRet) != len(entryPathList) {
		return fmt.Errorf("[UploadDirMulitThread] len(batchRet)[%d]!=len(entryPathList)[%d]",
			len(batchRet), len(entryPathList))
	}
	for i, ret := range batchRet {
		i := i
		//验证hash,当文件不存在时,err是空
		if ret.Error != "" && ret.Error != "no such file or directory" {
			return fmt.Errorf("[UploadDirMulitThread] [remotePath:%s]ctx.client.BatchStat err[%s]",
				requestList[i].remotePath, ret.Error)
		}
		if ret.Data.Hash == requestList[i].expectHash {
			continue
		}
		tm.AddTask(func() (err error) {
			return UploadFileWithHash(ctx, requestList[i].localPath, requestList[i].remotePath, requestList[i].expectHash)
		})
	}
	tm.Wait()
	if err != nil {
		return err
	}
	err = tm.GetError()
	return
}