func TestAll(t *testing.T) {
testPut(t, key1)
k1 := testPutWithoutKey(t)
testPutFile(t, localFile, key2)
k2 := testPutFileWithoutKey(t, localFile)
//clear all keys
rs.New(nil).Delete(nil, bucket, key1)
rs.New(nil).Delete(nil, bucket, key2)
rs.New(nil).Delete(nil, bucket, k1)
rs.New(nil).Delete(nil, bucket, k2)
}
func testXVar(t *testing.T, token string, extra *PutExtra) {
type Ret struct {
PutRet
X1 string `json:"x:1"`
}
var ret Ret
f, err := os.Open(testFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
err = Put(nil, &ret, token, testKey, f, fi.Size(), extra)
if err != nil {
t.Fatal(err)
}
defer rs.New(nil).Delete(nil, bucket, ret.Key)
if ret.X1 != "1" {
t.Fatal("test xVar failed:", ret.X1)
}
}
func Copy(cmd string, params ...string) {
if len(params) == 3 || len(params) == 4 {
srcBucket := params[0]
srcKey := params[1]
destBucket := params[2]
destKey := srcKey
if len(params) == 4 {
destKey = params[3]
}
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
err := client.Copy(nil, srcBucket, srcKey, destBucket, destKey)
if err != nil {
log.Error("Copy error,", err)
} else {
fmt.Println("Done!")
}
} else {
CmdHelp(cmd)
}
}
func copy(keySrc, keyDest string) {
client := rs.New(mac)
err := client.Copy(nil, Bucket, keySrc, Bucket_Copy, keyDest)
if err != nil {
//产生错误
log.Println("rs.Copy failed:", err)
return
}
}
//6.3 copy file
func copyFile(bucketSrc string, keySrc string, bucketDest string, keyDest string) {
var rsCli = rs.New(nil)
err := rsCli.Copy(nil, bucketSrc, keySrc, bucketDest, keyDest)
if err != nil {
log.Println("rs.Copy failed :", err)
return
} else {
log.Println("copy success to:", bucketDest)
}
}
func qiniuDeleteImage(fileName string) error {
var client rs.Client
client = rs.New(nil)
err := client.Delete(nil, models.QiniuScope, fileName)
if err != nil {
revel.ERROR.Println("rs.Delete failed:", err)
}
return err
}
func getInfo(key string) {
var ret rs.Entry
client := rs.New(mac)
ret, err := client.Stat(nil, Bucket, key)
if err != nil {
//产生错误
log.Println("rs.Stat failed:", err)
return
}
//处理返回值
log.Printf("ret:%+v", ret)
}
func Fetch(mac *digest.Mac, remoteResUrl, bucket, key string) (fetchResult FetchResult, err error) {
client := rs.New(mac)
entry := bucket
if key != "" {
entry += ":" + key
}
fetchUri := fmt.Sprintf("/fetch/%s/to/%s",
base64.URLEncoding.EncodeToString([]byte(remoteResUrl)),
base64.URLEncoding.EncodeToString([]byte(entry)))
err = client.Conn.Call(nil, &fetchResult, conf.IO_HOST+fetchUri)
return
}
func init() {
ACCESS_KEY = os.Getenv("QINIU_ACCESS_KEY")
SECRET_KEY = os.Getenv("QINIU_SECRET_KEY")
if ACCESS_KEY == "" || SECRET_KEY == "" {
panic("require ACCESS_KEY & SECRET_KEY")
}
bucket = os.Getenv("QINIU_TEST_BUCKET")
if bucket == "" {
panic("require QINIU_TEST_BUCKET")
}
rs.New(nil).Delete(nil, bucket, testKey)
}
//5.2delete file
func delFile2() {
bucket := "need10"
key := "132"
var rsCli = rs.New(nil)
err := rsCli.Delete(nil, bucket, key)
if err != nil {
//generate error
log.Println("rs.Copy failed:", err)
return
} else {
log.Println("success delete:", key)
}
}
func batchDel(keys []string) {
entryPaths := []rs.EntryPath{}
for _, key := range keys {
ep := rs.EntryPath{
Bucket: Bucket,
Key: key,
}
entryPaths = append(entryPaths, ep)
}
client := rs.New(mac)
client.BatchDelete(nil, entryPaths)
}
func testPutFile(t *testing.T, token string, extra *PutExtra) {
var ret PutRet
f, err := os.Open(testFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
err = PutFile(nil, &ret, token, testKey, testFile, extra)
if err != nil {
t.Fatal(err)
}
defer rs.New(nil).Delete(nil, bucket, ret.Key)
}
// 批量删除文件
func (this *QiniuController) DeleteFiles(bucketSrc string, keySrc []string) bool {
entryPathes := make([]rs.EntryPath, len(keySrc))
// 赋值
for k, _ := range entryPathes {
entryPathes[k].Bucket = bucketSrc
entryPathes[k].Key = keySrc[k]
}
rsCli := rs.New(nil)
_, err := rsCli.BatchDelete(nil, entryPathes)
return err == nil
}
func BatchCopy(cmd string, params ...string) {
if len(params) == 3 {
srcBucket := params[0]
destBucket := params[1]
srcDestKeyMapFile := params[2]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
fp, err := os.Open(srcDestKeyMapFile)
if err != nil {
log.Error("Open src dest key map file error")
return
}
defer fp.Close()
scanner := bufio.NewScanner(fp)
scanner.Split(bufio.ScanLines)
entries := make([]qshell.CopyEntryPath, 0)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
items := strings.Split(line, "\t")
if len(items) == 1 || len(items) == 2 {
srcKey := items[0]
destKey := srcKey
if len(items) == 2 {
destKey = items[1]
}
if srcKey != "" && destKey != "" {
entry := qshell.CopyEntryPath{srcBucket, destBucket, srcKey, destKey}
entries = append(entries, entry)
}
}
if len(entries) == BATCH_ALLOW_MAX {
batchCopy(client, entries)
entries = make([]qshell.CopyEntryPath, 0)
}
}
if len(entries) > 0 {
batchCopy(client, entries)
}
fmt.Println("All Copyed!")
} else {
CmdHelp(cmd)
}
}
func BatchDelete(cmd string, params ...string) {
if len(params) == 2 {
bucket := params[0]
keyListFile := params[1]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
fp, err := os.Open(keyListFile)
if err != nil {
log.Error("Open key list file error", err)
return
}
defer fp.Close()
scanner := bufio.NewScanner(fp)
scanner.Split(bufio.ScanLines)
entries := make([]rs.EntryPath, 0)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
items := strings.Split(line, "\t")
if len(items) > 0 {
key := items[0]
if key != "" {
entry := rs.EntryPath{
bucket, key,
}
entries = append(entries, entry)
}
}
//check 1000 limit
if len(entries) == BATCH_ALLOW_MAX {
batchDelete(client, entries)
//reset slice
entries = make([]rs.EntryPath, 0)
}
}
//delete the last batch
if len(entries) > 0 {
batchDelete(client, entries)
}
fmt.Println("All deleted!")
} else {
CmdHelp(cmd)
}
}
func rsDemo(bucket, key, bucketSrc, keySrc, bucketDest, keyDest string) {
// @gist rsPre
//此操作前 请确保 accesskey和secretkey 已被正确赋值
var rsCli = rs.New(nil)
var err error
// @endgist
// @gist rsStat
var ret rs.Entry
ret, err = rsCli.Stat(nil, bucket, key)
if err != nil {
// 产生错误
log.Println("rs.Stat failed:", err)
return
}
// 处理返回值
log.Println(ret)
// @endgist
// @gist rsCopy
err = rsCli.Copy(nil, bucketSrc, keySrc, bucketDest, keyDest)
if err != nil {
// 产生错误
log.Println("rs.Copy failed:", err)
return
}
// @endgist
// @gist rsMove
err = rsCli.Move(nil, bucketSrc, keySrc, bucketDest, keyDest)
if err != nil {
//产生错误
log.Println("rs.Copy failed:", err)
return
}
// @endgist
// @gist rsDelete
err = rsCli.Delete(nil, bucket, key)
if err != nil {
// 产生错误
log.Println("rs.Copy failed:", err)
return
}
// @endgist
}
//高级批处理功能
func highBatch() {
ops := []string{
rs.URIStat("needkane", "need"),
rs.URICopy("needkane", "kane3.jpg", "needcrystal", "kane"),
rs.URIDelete("needkane", "kane3.jpg"),
}
rets := new([]rs.BatchItemRet)
var rsCli = rs.New(nil)
err := rsCli.Batch(nil, rets, ops)
if err != nil {
log.Println("rs.Batch failed:", err)
return
}
for time, ret := range *rets {
log.Println(time, "::", ret.Code, ret.Error)
}
}
func (t *Task) Run() {
urls, err := UrlsToSync(t.SyncSrc)
if err != nil {
log.Error("Read Urls Failed:", err)
return
}
cl, base, err := t.Synced()
if err != nil {
log.Error("Read Log Failed", err)
return
}
left := namediff.Diff(urls, base)
mac := digest.Mac{
AccessKey: t.Ak,
SecretKey: []byte(t.Sk),
}
log.Info("Entry Fetched:", len(base))
log.Info("Entry ALL:", len(urls))
log.Info("Entry TO Fetch:", len(left))
c := rs.New(&mac)
failed := 0
for i := 0; i < len(left); i++ {
urlSrc := t.srcUrl(left[i])
key := destKey(left[i])
err := c.Fetch(nil, urlSrc, t.Bucket, key)
if err != nil {
log.Error("Fetch Url Failed:", err, urlSrc)
failed++
continue
}
cl.Put(left[i])
log.Info(urlSrc, "==>", t.Bucket, key)
}
if failed == 0 {
log.Info("Fetch Success")
} else {
log.Info(failed, "Files Failed")
}
return
}
func BatchRename(cmd string, params ...string) {
if len(params) == 2 {
bucket := params[0]
oldNewKeyMapFile := params[1]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
fp, err := os.Open(oldNewKeyMapFile)
if err != nil {
log.Error("Open old new key map file error")
return
}
defer fp.Close()
scanner := bufio.NewScanner(fp)
scanner.Split(bufio.ScanLines)
entries := make([]qshell.RenameEntryPath, 0)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
items := strings.Split(line, "\t")
if len(items) == 2 {
oldKey := items[0]
newKey := items[1]
if oldKey != "" && newKey != "" {
entry := qshell.RenameEntryPath{bucket, oldKey, newKey}
entries = append(entries, entry)
}
}
if len(entries) == BATCH_ALLOW_MAX {
batchRename(client, entries)
entries = make([]qshell.RenameEntryPath, 0)
}
}
if len(entries) > 0 {
batchRename(client, entries)
}
fmt.Println("All Renamed!")
} else {
CmdHelp(cmd)
}
}
func M3u8Delete(cmd string, params ...string) {
if len(params) == 2 || len(params) == 3 {
bucket := params[0]
m3u8Key := params[1]
isPrivate := false
if len(params) == 3 {
isPrivate, _ = strconv.ParseBool(params[2])
}
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
m3u8FileList, err := qshell.M3u8FileList(&mac, bucket, m3u8Key, isPrivate)
if err != nil {
log.Error(err)
return
}
client := rs.New(&mac)
entryCnt := len(m3u8FileList)
if entryCnt == 0 {
log.Error("no m3u8 slices found")
return
}
if entryCnt <= BATCH_ALLOW_MAX {
batchDelete(client, m3u8FileList)
} else {
batchCnt := entryCnt / BATCH_ALLOW_MAX
for i := 0; i < batchCnt; i++ {
end := (i + 1) * BATCH_ALLOW_MAX
if end > entryCnt {
end = entryCnt
}
entriesToDelete := m3u8FileList[i*BATCH_ALLOW_MAX : end]
batchDelete(client, entriesToDelete)
}
}
fmt.Println("All deleted!")
} else {
CmdHelp(cmd)
}
}
func TestAll(t *testing.T) {
//先上传文件到空间做初始化准备
for i := 0; i < 10; i++ {
key := "rsf_test_put_" + strconv.Itoa(i)
err := upFile("rsf_api.go", bucketName, key)
if err != nil {
t.Fatal(err)
}
keys = append(keys, key)
}
defer func() {
for _, k := range keys {
rs.New(nil).Delete(nil, bucketName, k)
}
}()
testList(t)
testEof(t)
}
func Stat(cmd string, params ...string) {
if len(params) == 2 {
bucket := params[0]
key := params[1]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
entry, err := client.Stat(nil, bucket, key)
if err != nil {
log.Error("Stat error,", err)
} else {
printStat(bucket, key, entry)
}
} else {
CmdHelp(cmd)
}
}
func testPutWithoutKey(t *testing.T, token string, extra *PutExtra) {
var ret PutRet
f, err := os.Open(testFile)
if err != nil {
t.Fatal(err)
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
err = PutWithoutKey(nil, &ret, token, f, fi.Size(), extra)
if err != nil {
t.Fatal(err)
}
defer rs.New(nil).Delete(nil, bucket, ret.Key)
}
func Delete(cmd string, params ...string) {
if len(params) == 2 {
bucket := params[0]
key := params[1]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
err := client.Delete(nil, bucket, key)
if err != nil {
log.Error("Delete error,", err)
} else {
fmt.Println("Done!")
}
} else {
CmdHelp(cmd)
}
}
func QiniuInit() error {
c := common.MongoCollection(FILE_DB, FILE_QINIU_FILE_TABLE)
index := mgo.Index{
Key: []string{"filename"},
Unique: true,
DropDups: true,
}
g_file_mgr = &FileMgr{
fileCh: make(chan *File, 100),
expiredTimer: time.Tick(time.Hour),
waitGroup: sync.NewWaitGroup(),
rsClient: rs.New(&digest.Mac{AccessKey: g_access_key, SecretKey: []byte(g_secret_key)}),
}
go g_file_mgr.run()
return errors.As(c.EnsureIndex(index))
}
// 批量移动文件
func (this *QiniuController) MoveFiles(bucketSrc string, keySrc []string, bucketDest string, keyDest []string) []rs.BatchItemRet {
// 每个复制操作都含有源文件和目标文件
entryPairs := make([]rs.EntryPathPair, len(keySrc))
for k, _ := range entryPairs {
entryPairs[k].Src.Bucket = bucketSrc
entryPairs[k].Src.Key = keySrc[k]
entryPairs[k].Dest.Bucket = bucketDest
entryPairs[k].Dest.Key = keyDest[k]
}
rsCli := rs.New(nil)
batchCopyRets, err := rsCli.BatchMove(nil, entryPairs)
if err != nil {
return nil
}
return batchCopyRets
}
// 批量获取图片信息
func (this *QiniuController) GetInfos(bucketSrc string, keySrc []string) []rs.BatchStatItemRet {
entryPathes := make([]rs.EntryPath, len(keySrc))
// 赋值
for k, _ := range entryPathes {
entryPathes[k].Bucket = bucketSrc
entryPathes[k].Key = keySrc[k]
}
var batchStatRets []rs.BatchStatItemRet
rsCli := rs.New(nil)
batchStatRets, err := rsCli.BatchStat(nil, entryPathes)
if err != nil {
return nil
}
return batchStatRets
}
func Chgm(cmd string, params ...string) {
if len(params) == 3 {
bucket := params[0]
key := params[1]
newMimeType := params[2]
accountS.Get()
mac := digest.Mac{
accountS.AccessKey,
[]byte(accountS.SecretKey),
}
client := rs.New(&mac)
err := client.ChangeMime(nil, bucket, key, newMimeType)
if err != nil {
log.Error("Change mimeType error,", err)
} else {
fmt.Println("Done!")
}
} else {
CmdHelp(cmd)
}
}
func batchCopyBucketQiniu(keys []string, srcBucket, descBucket string) error {
entryPairs := []rs.EntryPathPair{}
for _, key := range keys {
epp := rs.EntryPathPair{
Src: rs.EntryPath{
Bucket: srcBucket,
Key: key,
},
Dest: rs.EntryPath{
Bucket: descBucket,
Key: key,
},
}
entryPairs = append(entryPairs, epp)
}
client := rs.New(mac)
_, err := client.BatchCopy(nil, entryPairs)
return err
}
func getBatchInfo() {
entryPathes := []rs.EntryPath{
rs.EntryPath{
Bucket: "needkane",
Key: "kane3.jpg",
},
rs.EntryPath{
Bucket: "needcrystal",
Key: "needtoo",
},
}
var batchStatRests []rs.BatchStatItemRet
var rsCli = rs.New(nil)
batchStatRests, err := rsCli.BatchStat(nil, entryPathes)
if err != nil {
log.Println("rs.BatchStat failed", err)
return
}
for time, item := range batchStatRests {
log.Println(time, ";;;;", item)
}
}
