func testXVar(t *testing.T, token string, extra *PutExtra) { type Ret struct { PutRet X1 string `json:"x:1"` } var ret Ret f, err := os.Open(testFile) if err != nil { t.Fatal(err) } defer f.Close() fi, err := f.Stat() if err != nil { t.Fatal(err) } err = Put(nil, &ret, token, testKey, f, fi.Size(), extra) if err != nil { t.Fatal(err) } defer rs.New(nil).Delete(nil, bucket, ret.Key) if ret.X1 != "1" { t.Fatal("test xVar failed:", ret.X1) } }
func GetBuckets(mac *digest.Mac) (buckets []string, err error) { buckets = make([]string, 0) client := rs.New(mac) bucketsUri := fmt.Sprintf("%s/buckets", conf.RS_HOST) err = client.Conn.Call(nil, &buckets, bucketsUri) return }
func Copy(cmd string, params ...string) { if len(params) == 3 || len(params) == 4 { srcBucket := params[0] srcKey := params[1] destBucket := params[2] destKey := srcKey if len(params) == 4 { destKey = params[3] } accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) err := client.Copy(nil, srcBucket, srcKey, destBucket, destKey) if err != nil { log.Error("Copy error,", err) } else { fmt.Println("Done!") } } else { CmdHelp(cmd) } }
func BatchRename(cmd string, params ...string) { //confirm rcode := CreateRandString(6) if rcode == "" { log.Error("Create confirm code failed") return } rcode2 := "" fmt.Print(fmt.Sprintf("\033[31m<DANGER>\033[0m Input \033[32m%s\033[0m to confirm operation: ", rcode)) fmt.Scanln(&rcode2) if rcode != rcode2 { fmt.Println("Task quit!") return } if len(params) == 2 { bucket := params[0] oldNewKeyMapFile := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) fp, err := os.Open(oldNewKeyMapFile) if err != nil { log.Error("Open old new key map file error") return } defer fp.Close() scanner := bufio.NewScanner(fp) scanner.Split(bufio.ScanLines) entries := make([]qshell.RenameEntryPath, 0) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) items := strings.Split(line, "\t") if len(items) == 2 { oldKey := items[0] newKey := items[1] if oldKey != "" && newKey != "" { entry := qshell.RenameEntryPath{bucket, oldKey, newKey} entries = append(entries, entry) } } if len(entries) == BATCH_ALLOW_MAX { batchRename(client, entries) entries = make([]qshell.RenameEntryPath, 0) } } if len(entries) > 0 { batchRename(client, entries) } fmt.Println("All Renamed!") } else { CmdHelp(cmd) } }
func GetDomainsOfBucket(mac *digest.Mac, bucket string) (domains []string, err error) { domains = make([]string, 0) client := rs.New(mac) getDomainsUrl := "http://api.qiniu.com/v6/domain/list" postData := map[string][]string{ "tbl": []string{bucket}, } err = client.Conn.CallWithForm(nil, &domains, getDomainsUrl, postData) return }
func Fetch(mac *digest.Mac, remoteResUrl, bucket, key string) (fetchResult FetchResult, err error) { client := rs.New(mac) entry := bucket if key != "" { entry += ":" + key } fetchUri := fmt.Sprintf("/fetch/%s/to/%s", base64.URLEncoding.EncodeToString([]byte(remoteResUrl)), base64.URLEncoding.EncodeToString([]byte(entry))) err = client.Conn.Call(nil, &fetchResult, conf.IO_HOST+fetchUri) return }
func init() { ACCESS_KEY = os.Getenv("QINIU_ACCESS_KEY") SECRET_KEY = os.Getenv("QINIU_SECRET_KEY") if ACCESS_KEY == "" || SECRET_KEY == "" { panic("require ACCESS_KEY & SECRET_KEY") } bucket = os.Getenv("QINIU_TEST_BUCKET") if bucket == "" { panic("require QINIU_TEST_BUCKET") } rs.New(nil).Delete(nil, bucket, testKey) }
func testPutFile(t *testing.T, token string, extra *PutExtra) { var ret PutRet f, err := os.Open(testFile) if err != nil { t.Fatal(err) } defer f.Close() err = PutFile(nil, &ret, token, testKey, testFile, extra) if err != nil { t.Fatal(err) } defer rs.New(nil).Delete(nil, bucket, ret.Key) }
func BatchDelete(cmd string, params ...string) { if len(params) == 2 { bucket := params[0] keyListFile := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) fp, err := os.Open(keyListFile) if err != nil { log.Error("Open key list file error", err) return } defer fp.Close() scanner := bufio.NewScanner(fp) scanner.Split(bufio.ScanLines) entries := make([]rs.EntryPath, 0) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) items := strings.Split(line, "\t") if len(items) > 0 { key := items[0] if key != "" { entry := rs.EntryPath{ bucket, key, } entries = append(entries, entry) } } //check 1000 limit if len(entries) == BATCH_ALLOW_MAX { batchDelete(client, entries) //reset slice entries = make([]rs.EntryPath, 0) } } //delete the last batch if len(entries) > 0 { batchDelete(client, entries) } fmt.Println("All deleted!") } else { CmdHelp(cmd) } }
func rsDemo(bucket, key, bucketSrc, keySrc, bucketDest, keyDest string) { // @gist rsPre //此操作前 请确保 accesskey和secretkey 已被正确赋值 var rsCli = rs.New(nil) var err error // @endgist // @gist rsStat var ret rs.Entry ret, err = rsCli.Stat(nil, bucket, key) if err != nil { // 产生错误 log.Println("rs.Stat failed:", err) return } // 处理返回值 log.Println(ret) // @endgist // @gist rsCopy err = rsCli.Copy(nil, bucketSrc, keySrc, bucketDest, keyDest) if err != nil { // 产生错误 log.Println("rs.Copy failed:", err) return } // @endgist // @gist rsMove err = rsCli.Move(nil, bucketSrc, keySrc, bucketDest, keyDest) if err != nil { //产生错误 log.Println("rs.Copy failed:", err) return } // @endgist // @gist rsDelete err = rsCli.Delete(nil, bucket, key) if err != nil { // 产生错误 log.Println("rs.Copy failed:", err) return } // @endgist }
func BatchCopy(cmd string, params ...string) { if len(params) == 3 { srcBucket := params[0] destBucket := params[1] srcDestKeyMapFile := params[2] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) fp, err := os.Open(srcDestKeyMapFile) if err != nil { log.Error("Open src dest key map file error") return } defer fp.Close() scanner := bufio.NewScanner(fp) scanner.Split(bufio.ScanLines) entries := make([]qshell.CopyEntryPath, 0) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) items := strings.Split(line, "\t") if len(items) == 1 || len(items) == 2 { srcKey := items[0] destKey := srcKey if len(items) == 2 { destKey = items[1] } if srcKey != "" && destKey != "" { entry := qshell.CopyEntryPath{srcBucket, destBucket, srcKey, destKey} entries = append(entries, entry) } } if len(entries) == BATCH_ALLOW_MAX { batchCopy(client, entries) entries = make([]qshell.CopyEntryPath, 0) } } if len(entries) > 0 { batchCopy(client, entries) } fmt.Println("All Copyed!") } else { CmdHelp(cmd) } }
func BatchRename(cmd string, params ...string) { if len(params) == 2 { bucket := params[0] oldNewKeyMapFile := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) fp, err := os.Open(oldNewKeyMapFile) if err != nil { log.Error("Open old new key map file error") return } defer fp.Close() scanner := bufio.NewScanner(fp) scanner.Split(bufio.ScanLines) entries := make([]qshell.RenameEntryPath, 0) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) items := strings.Split(line, "\t") if len(items) == 2 { oldKey := items[0] newKey := items[1] if oldKey != "" && newKey != "" { entry := qshell.RenameEntryPath{bucket, oldKey, newKey} entries = append(entries, entry) } } if len(entries) == BATCH_ALLOW_MAX { batchRename(client, entries) entries = make([]qshell.RenameEntryPath, 0) } } if len(entries) > 0 { batchRename(client, entries) } fmt.Println("All Renamed!") } else { CmdHelp(cmd) } }
func DeleteHandlers() []martini.Handler { var bind = binding.Bind(DeleteReqData{}) var deleteHandler = func(data DeleteReqData, w http.ResponseWriter) { index := strings.LastIndex(data.Url, "/") runes := []rune(data.Url) key := string(runes[index+1:]) bucket, err := bucketdb.FindByName(data.Bucket) rsCli := rs.New(&digest.Mac{bucket.Ak, []byte(bucket.Sk)}) err = rsCli.Delete(nil, bucket.Name, key) if err != nil { //产生错误 glog.Infoln("删除图片错误:", err) io.WriteString(w, data.Callback+"({error:1})") return } io.WriteString(w, data.Callback+"({error:0})") } return []martini.Handler{bind, deleteHandler} }
func M3u8Delete(cmd string, params ...string) { if len(params) == 2 || len(params) == 3 { bucket := params[0] m3u8Key := params[1] isPrivate := false if len(params) == 3 { isPrivate, _ = strconv.ParseBool(params[2]) } accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } m3u8FileList, err := qshell.M3u8FileList(&mac, bucket, m3u8Key, isPrivate) if err != nil { log.Error(err) return } client := rs.New(&mac) entryCnt := len(m3u8FileList) if entryCnt == 0 { log.Error("no m3u8 slices found") return } if entryCnt <= BATCH_ALLOW_MAX { batchDelete(client, m3u8FileList) } else { batchCnt := entryCnt / BATCH_ALLOW_MAX for i := 0; i < batchCnt; i++ { end := (i + 1) * BATCH_ALLOW_MAX if end > entryCnt { end = entryCnt } entriesToDelete := m3u8FileList[i*BATCH_ALLOW_MAX : end] batchDelete(client, entriesToDelete) } } fmt.Println("All deleted!") } else { CmdHelp(cmd) } }
func Stat(cmd string, params ...string) { if len(params) == 2 { bucket := params[0] key := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) entry, err := client.Stat(nil, bucket, key) if err != nil { log.Error("Stat error,", err) } else { printStat(bucket, key, entry) } } else { CmdHelp(cmd) } }
func TestAll(t *testing.T) { //先上传文件到空间做初始化准备 for i := 0; i < 10; i++ { key := "rsf_test_put_" + strconv.Itoa(i) err := upFile("rsf_api.go", bucketName, key) if err != nil { t.Fatal(err) } keys = append(keys, key) } defer func() { for _, k := range keys { rs.New(nil).Delete(nil, bucketName, k) } }() testList(t) testEof(t) }
func DeleteFile(fileName string) bool { entryPathes := []qrs.EntryPath{ qrs.EntryPath{ Bucket: BulketName, Key: fmt.Sprintf("%s.html", fileName), }, qrs.EntryPath{ Bucket: BulketName, Key: fmt.Sprintf("%s_code.html", fileName), }, } rs := qrs.New(nil) _, err := rs.BatchDelete(nil, entryPathes) if err != nil { //产生错误 revel.ERROR.Printf("rs.BatchMove failed:%v", err) return false } return true }
func Delete(cmd string, params ...string) { if len(params) == 2 { bucket := params[0] key := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) err := client.Delete(nil, bucket, key) if err != nil { log.Error("Delete error,", err) } else { fmt.Println("Done!") } } else { CmdHelp(cmd) } }
func testPutWithoutKey(t *testing.T, token string, extra *PutExtra) { var ret PutRet f, err := os.Open(testFile) if err != nil { t.Fatal(err) } defer f.Close() fi, err := f.Stat() if err != nil { t.Fatal(err) } err = PutWithoutKey(nil, &ret, token, f, fi.Size(), extra) if err != nil { t.Fatal(err) } defer rs.New(nil).Delete(nil, bucket, ret.Key) }
func Chgm(cmd string, params ...string) { if len(params) == 3 { bucket := params[0] key := params[1] newMimeType := params[2] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) err := client.ChangeMime(nil, bucket, key, newMimeType) if err != nil { log.Error("Change mimeType error,", err) } else { fmt.Println("Done!") } } else { CmdHelp(cmd) } }
func TestAll(t *testing.T) { testPut(t, key1) k1 := testPutWithoutKey(t) testPutFile(t, localFile, key2) k2 := testPutFileWithoutKey(t, localFile) testPut(t, key3) k3 := testPutWithoutKey2(t) //clear all keys rs.New(nil).Delete(nil, bucket, key1) rs.New(nil).Delete(nil, bucket, key2) rs.New(nil).Delete(nil, bucket, key3) rs.New(nil).Delete(nil, bucket, k1) rs.New(nil).Delete(nil, bucket, k2) rs.New(nil).Delete(nil, bucket, k3) }
func (this *Mkziper) Do(req UfopRequest) (result interface{}, contentType string, err error) { //set mkzip check criteria if this.maxFileCount <= 0 { this.maxFileCount = MKZIP_MAX_FILE_COUNT } if this.maxFileLength <= 0 { this.maxFileLength = MKZIP_MAX_FILE_LENGTH } //parse command bucket, encoding, zipFiles, pErr := this.parse(req.Cmd) if pErr != nil { err = pErr return } //check file count if len(zipFiles) > this.maxFileCount { err = errors.New("zip file count exceeds the limit") return } if len(zipFiles) > MKZIP_MAX_FILE_LIMIT { err = errors.New("only support items less than 1000") return } //check whether file in bucket and exceeds the limit statItems := make([]rs.EntryPath, 0) for _, zipFile := range zipFiles { entryPath := rs.EntryPath{ bucket, zipFile.key, } statItems = append(statItems, entryPath) } qclient := rs.New(this.mac) statRet, statErr := qclient.BatchStat(nil, statItems) if statErr != nil { err = errors.New(fmt.Sprintf("batch stat error, %s", statErr)) return } for _, ret := range statRet { if ret.Error != "" { err = errors.New(fmt.Sprintf("stat resource in bucket error, %s", ret.Error)) return } if ret.Data.Fsize > this.maxFileLength { err = errors.New(fmt.Sprintf("stat resource length exceeds the limit, %d", ret.Data.Fsize)) return } } //retrieve resource and create zip file var tErr error zipBuffer := new(bytes.Buffer) zipWriter := zip.NewWriter(zipBuffer) for _, zipFile := range zipFiles { //convert encoding fname := zipFile.alias if encoding == "gbk" { fname, tErr = utf82GBK(fname) if tErr != nil { err = errors.New(fmt.Sprintf("unsupported encoding gbk, %s", tErr)) return } } //create each zip file writer fw, fErr := zipWriter.Create(fname) if fErr != nil { err = errors.New(fmt.Sprintf("create zip file error, %s", fErr)) return } //read data and write resResp, respErr := http.Get(zipFile.url) if respErr != nil || resResp.StatusCode != 200 { if respErr != nil { err = errors.New("get zip file resource error, " + respErr.Error()) } else { err = errors.New(fmt.Sprintf("get zip file resource error, %s", resResp.Status)) if resResp.Body != nil { resResp.Body.Close() } } return } respData, readErr := ioutil.ReadAll(resResp.Body) if readErr != nil { err = errors.New(fmt.Sprintf("read zip file resource content error, %s", readErr)) return } resResp.Body.Close() _, writeErr := fw.Write(respData) if writeErr != nil { err = errors.New(fmt.Sprintf("write zip file content error, %s", writeErr)) return } } //close zip file if cErr := zipWriter.Close(); cErr != nil { err = errors.New(fmt.Sprintf("close zip file error, %s", cErr)) return } result = zipBuffer.Bytes() contentType = "application/zip" return }
func Fetch(job, filePath, bucket, accessKey, secretKey string, worker int, zone string) { //open file fh, openErr := os.Open(filePath) if openErr != nil { fmt.Println("Open resource file error,", openErr) return } defer fh.Close() //open leveldb proFile := fmt.Sprintf(".%s.job", job) ldb, lerr := leveldb.OpenFile(proFile, nil) if lerr != nil { fmt.Println("Open fetch progress file error,", lerr) return } defer ldb.Close() //fetch prepare switch zone { case "bc": conf.IO_HOST = "http://iovip-z1.qbox.me" case "aws": conf.IO_HOST = "http://iovip.gdipper.com" default: conf.IO_HOST = "http://iovip.qbox.me" } mac := digest.Mac{ accessKey, []byte(secretKey), } client := rs.New(&mac) wg := sync.WaitGroup{} var seq int = 0 //scan each line bReader := bufio.NewScanner(fh) bReader.Split(bufio.ScanLines) for bReader.Scan() { line := strings.TrimSpace(bReader.Text()) if line == "" { continue } items := strings.Split(line, "\t") if !(len(items) == 1 || len(items) == 2) { fmt.Println("Invalid resource line,", line) continue } resUrl := items[0] resKey := "" if len(items) == 1 { resUri, pErr := url.Parse(resUrl) if pErr != nil { fmt.Println("Invalid resource url", resUrl) continue } resKey = resUri.Path if strings.HasPrefix(resKey, "/") { resKey = resKey[1:] } } else if len(items) == 2 { resKey = items[1] } //check from leveldb whether it is done val, exists := ldb.Get([]byte(resUrl), nil) if exists == nil && string(val) == resKey { continue } //otherwise fetch it seq += 1 if seq%worker == 0 { wg.Wait() } wg.Add(1) go func() { defer wg.Done() fErr := client.Fetch(nil, bucket, resKey, resUrl) if fErr == nil { ldb.Put([]byte(resUrl), []byte(resKey), nil) } else { fmt.Println("Fetch", resUrl, " error due to", fErr) } }() } wg.Wait() }
func batchDemo(bucket, key, bucket1, key1, bucket2, key2, bucket3, key3, bucket4, key4 string) { // @gist rsBatchPre // 此操作前 请确保 accesskey和secretkey 已被正确赋值 var rsCli = rs.New(nil) var err error // @endgist // @gist rsEntryPathes entryPathes := []rs.EntryPath{ rs.EntryPath{ Bucket: bucket1, Key: key1, }, rs.EntryPath{ Bucket: bucket2, Key: key2, }, } // @endgist // @gist rsPathPairs // 每个复制操作都含有源文件和目标文件 entryPairs := []rs.EntryPathPair{ rs.EntryPathPair{ Src: rs.EntryPath{ Bucket: bucket1, Key: key1, }, Dest: rs.EntryPath{ Bucket: bucket2, Key: key2, }, }, rs.EntryPathPair{ Src: rs.EntryPath{ Bucket: bucket3, Key: key3, }, Dest: rs.EntryPath{ Bucket: bucket4, Key: key4, }, }, } // @endgist // @gist rsBatchStat var batchStatRets []rs.BatchStatItemRet batchStatRets, err = rsCli.BatchStat(nil, entryPathes) // []rs.BatchStatItemRet, error if err != nil { // 产生错误 log.Println("rs.BatchStat failed:", err) return } // 处理返回值 for _, item := range batchStatRets { log.Println(item) } // @endgist // @gist rsBatchCopy var batchCopyRets []rs.BatchItemRet batchCopyRets, err = rsCli.BatchCopy(nil, entryPairs) if err != nil { // 产生错误 log.Println("rs.BatchCopy failed:", err) return } for _, item := range batchCopyRets { // 遍历每个操作的返回结果 log.Println(item.Code, item.Error) } // @endgist // @gist rsBatchMove var batchMoveRets []rs.BatchItemRet batchMoveRets, err = rsCli.BatchMove(nil, entryPairs) if err != nil { // 产生错误 log.Println("rs.BatchMove failed:", err) return } for _, item := range batchMoveRets { // 遍历每个操作的返回结果 log.Println(item.Code, item.Error) } // @endgist // @gist rsBatchDelete var batchDeleteRets []rs.BatchItemRet batchDeleteRets, err = rsCli.BatchDelete(nil, entryPathes) if err != nil { // 产生错误 log.Println("rs.BatchDelete failed:", err) return } for _, item := range batchDeleteRets { // 遍历每个操作的返回结果 log.Println(item.Code, item.Error) } // @endgist // @gist rsBatchAdv ops := []string{ rs.URIStat(bucket, key1), rs.URICopy(bucket, key1, bucket, key2), // 复制key1到key2 rs.URIDelete(bucket, key1), // 删除key1 rs.URIMove(bucket, key2, bucket, key1), //将key2移动到key1 } rets := new([]rs.BatchItemRet) err = rsCli.Batch(nil, rets, ops) if err != nil { // 产生错误 log.Println("rs.Batch failed:", err) return } for _, ret := range *rets { log.Println(ret.Code, ret.Error) } // @endgist }
func QiniuUpload(threadCount int, uploadConfigFile string) { fp, err := os.Open(uploadConfigFile) if err != nil { log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err)) return } defer fp.Close() configData, err := ioutil.ReadAll(fp) if err != nil { log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err)) return } var uploadConfig UploadConfig err = json.Unmarshal(configData, &uploadConfig) if err != nil { log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err)) return } if _, err := os.Stat(uploadConfig.SrcDir); err != nil { log.Error("Upload config error for parameter `SrcDir`,", err) return } dirCache := DirCache{} currentUser, err := user.Current() if err != nil { log.Error("Failed to get current user", err) return } pathSep := string(os.PathSeparator) //create job id md5Hasher := md5.New() md5Hasher.Write([]byte(strings.TrimSuffix(uploadConfig.SrcDir, pathSep) + ":" + uploadConfig.Bucket)) jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil)) //local storage path storePath := filepath.Join(currentUser.HomeDir, ".qshell", "qupload", jobId) err = os.MkdirAll(storePath, 0775) if err != nil { log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err)) return } //cache file cacheFileName := filepath.Join(storePath, jobId+".cache") //leveldb folder leveldbFileName := filepath.Join(storePath, jobId+".ldb") totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName) ldb, err := leveldb.OpenFile(leveldbFileName, nil) if err != nil { log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err)) return } defer ldb.Close() //sync ufp, err := os.Open(cacheFileName) if err != nil { log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err)) return } defer ufp.Close() bScanner := bufio.NewScanner(ufp) bScanner.Split(bufio.ScanLines) var currentFileCount int64 = 0 var successFileCount int64 = 0 var failureFileCount int64 = 0 var skippedFileCount int64 = 0 ldbWOpt := opt.WriteOptions{ Sync: true, } upWorkGroup := sync.WaitGroup{} upCounter := 0 threadThreshold := threadCount + 1 //use host if not empty if uploadConfig.UpHost != "" { conf.UP_HOST = uploadConfig.UpHost } //set resume upload settings rio.SetSettings(&upSettings) mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)} for bScanner.Scan() { line := strings.TrimSpace(bScanner.Text()) items := strings.Split(line, "\t") if len(items) != 3 { log.Error(fmt.Sprintf("Invalid cache line `%s'", line)) continue } localFname := items[0] currentFileCount += 1 skip := false //check skip local file or folder if uploadConfig.SkipPrefixes != "" { //unpack skip prefix skipPrefixes := strings.Split(uploadConfig.SkipPrefixes, ",") for _, prefix := range skipPrefixes { if strings.HasPrefix(localFname, strings.TrimSpace(prefix)) { log.Debug(fmt.Sprintf("Skip by prefix '%s' for local file %s", strings.TrimSpace(prefix), localFname)) skip = true skippedFileCount += 1 break } } if skip { continue } } if uploadConfig.SkipSuffixes != "" { skipSuffixes := strings.Split(uploadConfig.SkipSuffixes, ",") for _, suffix := range skipSuffixes { if strings.HasSuffix(localFname, strings.TrimSpace(suffix)) { log.Debug(fmt.Sprintf("Skip by suffix '%s' for local file %s", strings.TrimSpace(suffix), localFname)) skip = true skippedFileCount += 1 break } } if skip { continue } } //pack the upload file key localFlmd, _ := strconv.Atoi(items[2]) uploadFileKey := localFname if uploadConfig.IgnoreDir { if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 { uploadFileKey = uploadFileKey[i+1:] } } if uploadConfig.KeyPrefix != "" { uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "") } //convert \ to / under windows if runtime.GOOS == "windows" { uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1) } localFilePath := filepath.Join(uploadConfig.SrcDir, localFname) fstat, err := os.Stat(localFilePath) if err != nil { log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", localFilePath, err)) return } fsize := fstat.Size() ldbKey := fmt.Sprintf("%s => %s", localFilePath, uploadFileKey) log.Info(fmt.Sprintf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount, float32(currentFileCount)*100/float32(totalFileCount))) rsClient := rs.New(&mac) //check exists if uploadConfig.CheckExists { rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey) if checkErr == nil { //compare hash localEtag, cErr := GetEtag(localFilePath) if cErr != nil { atomic.AddInt64(&failureFileCount, 1) log.Error("Calc local file hash failed,", cErr) continue } if rsEntry.Hash == localEtag { atomic.AddInt64(&skippedFileCount, 1) log.Debug(fmt.Sprintf("File %s already exists in bucket, ignore this upload", uploadFileKey)) continue } } else { if _, ok := checkErr.(*rpc.ErrorInfo); !ok { //not logic error, should be network error atomic.AddInt64(&failureFileCount, 1) continue } } } else { //check leveldb ldbFlmd, err := ldb.Get([]byte(ldbKey), nil) flmd, _ := strconv.Atoi(string(ldbFlmd)) //not exist, return ErrNotFound //check last modified if err == nil && localFlmd == flmd { log.Debug("Skip by local log for file", localFname) atomic.AddInt64(&skippedFileCount, 1) continue } } //worker upCounter += 1 if upCounter%threadThreshold == 0 { upWorkGroup.Wait() } upWorkGroup.Add(1) //start to upload go func() { defer upWorkGroup.Done() policy := rs.PutPolicy{} policy.Scope = uploadConfig.Bucket if uploadConfig.Overwrite { policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey policy.InsertOnly = 0 } policy.Expires = 24 * 3600 uptoken := policy.Token(&mac) if fsize > PUT_THRESHOLD { putRet := rio.PutRet{} err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, localFilePath, nil) if err != nil { atomic.AddInt64(&failureFileCount, 1) if pErr, ok := err.(*rpc.ErrorInfo); ok { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, pErr.Err)) } else { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, err)) } } else { atomic.AddInt64(&successFileCount, 1) perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt) if perr != nil { log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)) } } } else { putRet := fio.PutRet{} err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, localFilePath, nil) if err != nil { atomic.AddInt64(&failureFileCount, 1) if pErr, ok := err.(*rpc.ErrorInfo); ok { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, pErr.Err)) } else { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", localFilePath, uploadFileKey, err)) } } else { atomic.AddInt64(&successFileCount, 1) perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(localFlmd)), &ldbWOpt) if perr != nil { log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)) } } } }() } upWorkGroup.Wait() log.Info("-------Upload Done-------") log.Info("Total:\t", currentFileCount) log.Info("Success:\t", successFileCount) log.Info("Failure:\t", failureFileCount) log.Info("Skipped:\t", skippedFileCount) log.Info("-------------------------") }
// DeleteArchive deletes a archive from QiNiu. func DeleteArchive(key string) error { rsCli := rs.New(nil) return rsCli.Delete(nil, setting.BucketName, key) }
func (helper *QiniuHelper) Del(bucket string, key string) error { client := rs.New(nil) return client.Delete(nil, bucket, key) }
func (this *Mkzipper) Do(req ufop.UfopRequest) (result interface{}, resultType int, contentType string, err error) { //parse command bucket, encoding, zipFiles, pErr := this.parse(req.Cmd) if pErr != nil { err = pErr return } //check file count if len(zipFiles) > this.maxFileCount { err = errors.New("zip file count exceeds the limit") return } if len(zipFiles) > MKZIP_MAX_FILE_LIMIT { err = errors.New("only support items less than 1000") return } //check whether file in bucket and exceeds the limit statItems := make([]rs.EntryPath, 0) statUrls := make([]string, 0) for _, zipFile := range zipFiles { entryPath := rs.EntryPath{ bucket, zipFile.key, } statItems = append(statItems, entryPath) statUrls = append(statUrls, zipFile.url) } qclient := rs.New(this.mac) statRet, statErr := qclient.BatchStat(nil, statItems) if statErr != nil { if _, ok := statErr.(*rpc.ErrorInfo); !ok { err = errors.New(fmt.Sprintf("batch stat error, %s", statErr.Error())) return } } for index := 0; index < len(statRet); index++ { ret := statRet[index] if ret.Code != 200 { if ret.Code == 612 { err = errors.New(fmt.Sprintf("batch stat '%s' error, no such file or directory", statUrls[index])) } else if ret.Code == 631 { err = errors.New(fmt.Sprintf("batch stat '%s' error, no such bucket", statUrls[index])) } else { err = errors.New(fmt.Sprintf("batch stat '%s' error, %d", statUrls[index], ret.Code)) } return } } //retrieve resource and create zip file var tErr error zipBuffer := new(bytes.Buffer) zipWriter := zip.NewWriter(zipBuffer) for _, zipFile := range zipFiles { //convert encoding fname := zipFile.alias if encoding == "gbk" { fname, tErr = utils.Utf82Gbk(fname) if tErr != nil { err = errors.New(fmt.Sprintf("unsupported encoding gbk, %s", tErr)) return } } //create each zip file writer fw, fErr := zipWriter.Create(fname) if fErr != nil { err = errors.New(fmt.Sprintf("create zip file error, %s", fErr)) return } //read data and write resResp, respErr := http.Get(zipFile.url) if respErr != nil || resResp.StatusCode != 200 { if respErr != nil { err = errors.New("get zip file resource error, " + respErr.Error()) } else { err = errors.New(fmt.Sprintf("get zip file resource error, %s", resResp.Status)) if resResp.Body != nil { resResp.Body.Close() } } return } respData, readErr := ioutil.ReadAll(resResp.Body) if readErr != nil { err = errors.New(fmt.Sprintf("read zip file resource content error, %s", readErr)) return } resResp.Body.Close() _, writeErr := fw.Write(respData) if writeErr != nil { err = errors.New(fmt.Sprintf("write zip file content error, %s", writeErr)) return } } //close zip file if cErr := zipWriter.Close(); cErr != nil { err = errors.New(fmt.Sprintf("close zip file error, %s", cErr)) return } //write result result = zipBuffer.Bytes() resultType = ufop.RESULT_TYPE_OCTECT_BYTES contentType = "application/zip" return }
func QiniuUpload(threadCount int, uploadConfigFile string) { fp, err := os.Open(uploadConfigFile) if err != nil { log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err)) return } defer fp.Close() configData, err := ioutil.ReadAll(fp) if err != nil { log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err)) return } var uploadConfig UploadConfig err = json.Unmarshal(configData, &uploadConfig) if err != nil { log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err)) return } if _, err := os.Stat(uploadConfig.SrcDir); err != nil { log.Error("Upload config error for parameter `SrcDir`,", err) return } dirCache := DirCache{} currentUser, err := user.Current() if err != nil { log.Error("Failed to get current user", err) return } pathSep := string(os.PathSeparator) //create job id md5Hasher := md5.New() md5Hasher.Write([]byte(uploadConfig.SrcDir + ":" + uploadConfig.Bucket)) jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil)) //local storage path storePath := fmt.Sprintf("%s%s.qshell%squpload%s%s", currentUser.HomeDir, pathSep, pathSep, pathSep, jobId) err = os.MkdirAll(storePath, 0775) if err != nil { log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err)) return } //cache file cacheFileName := fmt.Sprintf("%s%s%s.cache", storePath, pathSep, jobId) //leveldb folder leveldbFileName := fmt.Sprintf("%s%s%s.ldb", storePath, pathSep, jobId) totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName) ldb, err := leveldb.OpenFile(leveldbFileName, nil) if err != nil { log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err)) return } defer ldb.Close() //sync ufp, err := os.Open(cacheFileName) if err != nil { log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err)) return } defer ufp.Close() bScanner := bufio.NewScanner(ufp) bScanner.Split(bufio.ScanLines) currentFileCount := 0 ldbWOpt := opt.WriteOptions{ Sync: true, } upWorkGroup := sync.WaitGroup{} upCounter := 0 threadThreshold := threadCount + 1 //use host if not empty if uploadConfig.UpHost != "" { conf.UP_HOST = uploadConfig.UpHost } //set settings rio.SetSettings(&upSettings) mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)} //check thread count for bScanner.Scan() { line := strings.TrimSpace(bScanner.Text()) items := strings.Split(line, "\t") if len(items) > 1 { cacheFname := items[0] cacheFlmd, _ := strconv.Atoi(items[2]) uploadFileKey := cacheFname if uploadConfig.IgnoreDir { if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 { uploadFileKey = uploadFileKey[i+1:] } } if uploadConfig.KeyPrefix != "" { uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "") } //convert \ to / under windows if runtime.GOOS == "windows" { uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1) } cacheFilePath := strings.Join([]string{uploadConfig.SrcDir, cacheFname}, pathSep) fstat, err := os.Stat(cacheFilePath) if err != nil { log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err)) return } fsize := fstat.Size() //check leveldb currentFileCount += 1 ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey) log.Debug(fmt.Sprintf("Checking %s ...", ldbKey)) //check last modified ldbFlmd, err := ldb.Get([]byte(ldbKey), nil) flmd, _ := strconv.Atoi(string(ldbFlmd)) //not exist, return ErrNotFound if err == nil && cacheFlmd == flmd { continue } fmt.Print("\033[2K\r") fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount, float32(currentFileCount)*100/float32(totalFileCount)) os.Stdout.Sync() rsClient := rs.New(&mac) //worker upCounter += 1 if upCounter%threadThreshold == 0 { upWorkGroup.Wait() } upWorkGroup.Add(1) go func() { defer upWorkGroup.Done() //check exists if uploadConfig.CheckExists { rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey) if checkErr == nil { //compare hash localEtag, cErr := GetEtag(cacheFilePath) if cErr != nil { log.Error("Calc local file hash failed,", cErr) return } if rsEntry.Hash == localEtag { log.Info("File already exists in bucket, ignore this upload") return } } else { if _, ok := checkErr.(*rpc.ErrorInfo); !ok { //not logic error, should be network error return } } } //upload policy := rs.PutPolicy{} policy.Scope = uploadConfig.Bucket if uploadConfig.Overwrite { policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey policy.InsertOnly = 0 } policy.Expires = 24 * 3600 uptoken := policy.Token(&mac) if fsize > PUT_THRESHOLD { putRet := rio.PutRet{} err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil) if err != nil { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err)) } else { perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt) if perr != nil { log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)) } } } else { putRet := fio.PutRet{} err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil) if err != nil { log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err)) } else { perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt) if perr != nil { log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr)) } } } }() } else { log.Error(fmt.Sprintf("Error cache line `%s'", line)) } } upWorkGroup.Wait() fmt.Println() fmt.Println("Upload done!") }
func BatchDelete(cmd string, params ...string) { //confirm rcode := CreateRandString(6) if rcode == "" { log.Error("Create confirm code failed") return } rcode2 := "" fmt.Print(fmt.Sprintf("\033[31m<DANGER>\033[0m Input \033[32m%s\033[0m to confirm operation: ", rcode)) fmt.Scanln(&rcode2) if rcode != rcode2 { fmt.Println("Task quit!") return } if len(params) == 2 { bucket := params[0] keyListFile := params[1] accountS.Get() mac := digest.Mac{ accountS.AccessKey, []byte(accountS.SecretKey), } client := rs.New(&mac) fp, err := os.Open(keyListFile) if err != nil { log.Error("Open key list file error", err) return } defer fp.Close() scanner := bufio.NewScanner(fp) scanner.Split(bufio.ScanLines) entries := make([]rs.EntryPath, 0) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) items := strings.Split(line, "\t") if len(items) > 0 { key := items[0] if key != "" { entry := rs.EntryPath{ bucket, key, } entries = append(entries, entry) } } //check 1000 limit if len(entries) == BATCH_ALLOW_MAX { batchDelete(client, entries) //reset slice entries = make([]rs.EntryPath, 0) } } //delete the last batch if len(entries) > 0 { batchDelete(client, entries) } fmt.Println("All deleted!") } else { CmdHelp(cmd) } }