func BatchCopy(client rs.Client, entries []CopyEntryPath) (ret []BatchItemRet, err error) {
b := make([]string, len(entries))
for i, e := range entries {
b[i] = rs.URICopy(e.SrcBucket, e.SrcKey, e.DestBucket, e.DestKey)
}
err = client.Batch(nil, &ret, b)
return
}
func BatchDelete(client rs.Client, entries []rs.EntryPath) (ret []BatchItemRet, err error) {
b := make([]string, len(entries))
for i, e := range entries {
b[i] = rs.URIDelete(e.Bucket, e.Key)
}
err = client.Batch(nil, &ret, b)
return
}
func BatchRename(client rs.Client, entries []RenameEntryPath) (ret []BatchItemRet, err error) {
b := make([]string, len(entries))
for i, e := range entries {
b[i] = rs.URIMove(e.Bucket, e.OldKey, e.Bucket, e.NewKey)
}
err = client.Batch(nil, &ret, b)
return
}
func BatchChgm(client rs.Client, entries []ChgmEntryPath) (ret []BatchItemRet, err error) {
b := make([]string, len(entries))
for i, e := range entries {
b[i] = rs.URIChangeMime(e.Bucket, e.Key, e.MimeType)
}
err = client.Batch(nil, &ret, b)
return
}
func QiniuUpload(threadCount int, uploadConfig *UploadConfig) {
timeStart := time.Now()
//make SrcDir the full path
uploadConfig.SrcDir, _ = filepath.Abs(uploadConfig.SrcDir)
dirCache := DirCache{}
pathSep := string(os.PathSeparator)
//create job id
jobId := Md5Hex(fmt.Sprintf("%s:%s", uploadConfig.SrcDir, uploadConfig.Bucket))
//local storage path
storePath := filepath.Join(".qshell", "qupload", jobId)
if err := os.MkdirAll(storePath, 0775); err != nil {
log.Errorf("Failed to mkdir `%s' due to `%s'", storePath, err)
return
}
//cache file
rescanLocalDir := false
cacheResultName := filepath.Join(storePath, jobId+".cache")
cacheTempName := filepath.Join(storePath, jobId+".cache.temp")
cacheCountName := filepath.Join(storePath, jobId+".count")
if _, statErr := os.Stat(cacheResultName); statErr == nil {
//file already exists
rescanLocalDir = uploadConfig.RescanLocal
} else {
rescanLocalDir = true
}
var totalFileCount int64
if rescanLocalDir {
fmt.Println("Listing local sync dir, this can take a long time, please wait patiently...")
totalFileCount = dirCache.Cache(uploadConfig.SrcDir, cacheTempName)
if rErr := os.Remove(cacheResultName); rErr != nil {
log.Debug("Remove the old cached file error", rErr)
}
if rErr := os.Rename(cacheTempName, cacheResultName); rErr != nil {
fmt.Println("Rename the temp cached file error", rErr)
return
}
//write the total count to local file
if cFp, cErr := os.Create(cacheCountName); cErr == nil {
func() {
defer cFp.Close()
uploadInfo := UploadInfo{
TotalFileCount: totalFileCount,
}
uploadInfoBytes, mErr := json.Marshal(&uploadInfo)
if mErr == nil {
if _, wErr := cFp.Write(uploadInfoBytes); wErr != nil {
log.Errorf("Write local cached count file error %s", cErr)
} else {
cFp.Close()
}
}
}()
} else {
log.Errorf("Open local cached count file error %s", cErr)
}
} else {
fmt.Println("Use the last cached local sync dir file list ...")
//read from local cache
if rFp, rErr := os.Open(cacheCountName); rErr == nil {
func() {
defer rFp.Close()
uploadInfo := UploadInfo{}
decoder := json.NewDecoder(rFp)
if dErr := decoder.Decode(&uploadInfo); dErr == nil {
totalFileCount = uploadInfo.TotalFileCount
}
}()
} else {
log.Warnf("Open local cached count file error %s", rErr)
}
}
//leveldb folder
leveldbFileName := filepath.Join(storePath, jobId+".ldb")
ldb, err := leveldb.OpenFile(leveldbFileName, nil)
if err != nil {
log.Errorf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err)
return
}
defer ldb.Close()
//sync
ufp, err := os.Open(cacheResultName)
if err != nil {
log.Errorf("Open cache file `%s' failed due to `%s'", cacheResultName, err)
return
}
defer ufp.Close()
bScanner := bufio.NewScanner(ufp)
bScanner.Split(bufio.ScanLines)
var currentFileCount int64 = 0
var successFileCount int64 = 0
var failureFileCount int64 = 0
var skippedFileCount int64 = 0
ldbWOpt := opt.WriteOptions{
Sync: true,
}
upWorkGroup := sync.WaitGroup{}
upCounter := 0
threadThreshold := threadCount + 1
//chunk upload threshold
putThreshold := DEFAULT_PUT_THRESHOLD
if uploadConfig.PutThreshold > 0 {
putThreshold = uploadConfig.PutThreshold
}
//check zone, default nb
switch uploadConfig.Zone {
case ZoneAWS:
SetZone(ZoneAWSConfig)
case ZoneBC:
SetZone(ZoneBCConfig)
default:
SetZone(ZoneNBConfig)
}
//use host if not empty, overwrite the default config
if uploadConfig.UpHost != "" {
conf.UP_HOST = uploadConfig.UpHost
}
//set resume upload settings
rio.SetSettings(&upSettings)
mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}
//check bind net interface card
var transport *http.Transport
var rsClient rs.Client
if uploadConfig.BindNicIp != "" {
transport = &http.Transport{
Dial: (&net.Dialer{
LocalAddr: &net.TCPAddr{
IP: net.ParseIP(uploadConfig.BindNicIp),
},
}).Dial,
}
}
if transport != nil {
rsClient = rs.NewMacEx(&mac, transport, "")
} else {
rsClient = rs.NewMac(&mac)
}
//check remote rs ip bind
if uploadConfig.BindRsIp != "" {
rsClient.Conn.BindRemoteIp = uploadConfig.BindRsIp
}
//scan lines and upload
for bScanner.Scan() {
line := strings.TrimSpace(bScanner.Text())
items := strings.Split(line, "\t")
if len(items) != 3 {
log.Errorf("Invalid cache line `%s'", line)
continue
}
localFpath := items[0]
currentFileCount += 1
skip := false
//check skip local file or folder
if uploadConfig.SkipPathPrefixes != "" {
//unpack skip prefix
skipPathPrefixes := strings.Split(uploadConfig.SkipPathPrefixes, ",")
for _, prefix := range skipPathPrefixes {
if strings.HasPrefix(localFpath, strings.TrimSpace(prefix)) {
log.Debug(fmt.Sprintf("Skip by path prefix '%s' for local file %s",
strings.TrimSpace(prefix), localFpath))
skip = true
skippedFileCount += 1
break
}
}
if skip {
continue
}
}
if uploadConfig.SkipFilePrefixes != "" {
//unpack skip prefix
skipFilePrefixes := strings.Split(uploadConfig.SkipFilePrefixes, ",")
for _, prefix := range skipFilePrefixes {
localFname := filepath.Base(localFpath)
if strings.HasPrefix(localFname, strings.TrimSpace(prefix)) {
log.Debug(fmt.Sprintf("Skip by file prefix '%s' for local file %s",
strings.TrimSpace(prefix), localFpath))
skip = true
skippedFileCount += 1
break
}
}
if skip {
continue
}
}
if uploadConfig.SkipSuffixes != "" {
skipSuffixes := strings.Split(uploadConfig.SkipSuffixes, ",")
for _, suffix := range skipSuffixes {
if strings.HasSuffix(localFpath, strings.TrimSpace(suffix)) {
log.Debug(fmt.Sprintf("Skip by suffix '%s' for local file %s",
strings.TrimSpace(suffix), localFpath))
skip = true
skippedFileCount += 1
break
}
}
if skip {
continue
}
}
//pack the upload file key
localFlmd, _ := strconv.ParseInt(items[2], 10, 64)
uploadFileKey := localFpath
if uploadConfig.IgnoreDir {
if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
uploadFileKey = uploadFileKey[i+1:]
}
}
if uploadConfig.KeyPrefix != "" {
uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
}
//convert \ to / under windows
if runtime.GOOS == "windows" {
uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
}
localFilePath := filepath.Join(uploadConfig.SrcDir, localFpath)
fstat, err := os.Stat(localFilePath)
if err != nil {
log.Errorf("Error stat local file `%s' due to `%s'", localFilePath, err)
continue
}
fsize := fstat.Size()
ldbKey := fmt.Sprintf("%s => %s", localFilePath, uploadFileKey)
if totalFileCount != 0 {
fmt.Println(fmt.Sprintf("Uploading %s [%d/%d, %.1f%%] ...", ldbKey, currentFileCount, totalFileCount,
float32(currentFileCount)*100/float32(totalFileCount)))
} else {
fmt.Println(fmt.Sprintf("Uploading %s ...", ldbKey))
}
//check exists
if uploadConfig.CheckExists {
rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
if checkErr == nil {
//compare hash
localEtag, cErr := GetEtag(localFilePath)
if cErr != nil {
atomic.AddInt64(&failureFileCount, 1)
log.Error("Calc local file hash failed,", cErr)
continue
}
if rsEntry.Hash == localEtag {
atomic.AddInt64(&skippedFileCount, 1)
log.Debug(fmt.Sprintf("File %s already exists in bucket, ignore this upload", uploadFileKey))
continue
}
} else {
if _, ok := checkErr.(*rpc.ErrorInfo); !ok {
//not logic error, should be network error
atomic.AddInt64(&failureFileCount, 1)
continue
}
}
} else {
//check leveldb
ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
flmd, _ := strconv.ParseInt(string(ldbFlmd), 10, 64)
//not exist, return ErrNotFound
//check last modified
if err == nil && localFlmd == flmd {
log.Debug("Skip by local log for file", localFpath)
atomic.AddInt64(&skippedFileCount, 1)
continue
}
}
//worker
upCounter += 1
if upCounter%threadThreshold == 0 {
upWorkGroup.Wait()
}
upWorkGroup.Add(1)
//start to upload
go func() {
defer upWorkGroup.Done()
policy := rs.PutPolicy{}
policy.Scope = uploadConfig.Bucket
if uploadConfig.Overwrite {
policy.Scope = fmt.Sprintf("%s:%s", uploadConfig.Bucket, uploadFileKey)
policy.InsertOnly = 0
}
policy.Expires = 30 * 24 * 3600
uptoken := policy.Token(&mac)
if fsize > putThreshold {
var putClient rpc.Client
if transport != nil {
putClient = rio.NewClientEx(uptoken, transport, uploadConfig.BindUpIp)
} else {
putClient = rio.NewClient(uptoken, uploadConfig.BindUpIp)
}
putRet := rio.PutRet{}
putExtra := rio.PutExtra{}
progressFkey := Md5Hex(fmt.Sprintf("%s:%s|%s:%s", uploadConfig.SrcDir, uploadConfig.Bucket, localFpath, uploadFileKey))
progressFname := fmt.Sprintf("%s.progress", progressFkey)
progressFpath := filepath.Join(storePath, progressFname)
putExtra.ProgressFile = progressFpath
err := rio.PutFile(putClient, nil, &putRet, uploadFileKey, localFilePath, &putExtra)
if err != nil {
atomic.AddInt64(&failureFileCount, 1)
if pErr, ok := err.(*rpc.ErrorInfo); ok {
log.Errorf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, pErr.Err)
} else {
log.Errorf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, err)
}
} else {
os.Remove(progressFpath)
atomic.AddInt64(&successFileCount, 1)
perr := ldb.Put([]byte(ldbKey), []byte(fmt.Sprintf("%d", localFlmd)), &ldbWOpt)
if perr != nil {
log.Errorf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)
}
}
} else {
var putClient rpc.Client
if transport != nil {
putClient = rpc.NewClientEx(transport, uploadConfig.BindUpIp)
} else {
putClient = rpc.NewClient(uploadConfig.BindUpIp)
}
putRet := fio.PutRet{}
err := fio.PutFile(putClient, nil, &putRet, uptoken, uploadFileKey, localFilePath, nil)
if err != nil {
atomic.AddInt64(&failureFileCount, 1)
if pErr, ok := err.(*rpc.ErrorInfo); ok {
log.Errorf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, pErr.Err)
} else {
log.Errorf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, err)
}
} else {
atomic.AddInt64(&successFileCount, 1)
perr := ldb.Put([]byte(ldbKey), []byte(fmt.Sprintf("%d", localFlmd)), &ldbWOpt)
if perr != nil {
log.Errorf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)
}
}
}
}()
}
upWorkGroup.Wait()
fmt.Println()
fmt.Println("-------Upload Result-------")
fmt.Println("Total: \t", currentFileCount)
fmt.Println("Success: \t", successFileCount)
fmt.Println("Failure: \t", failureFileCount)
fmt.Println("Skipped: \t", skippedFileCount)
fmt.Println("Duration:\t", time.Since(timeStart))
fmt.Println("-------------------------")
}