func upFile(localFile, bucketName, key string) error {
policy := rs.PutPolicy{
Scope: bucketName + ":" + key,
}
return qio.PutFile(nil, nil, policy.Token(nil), key, localFile, nil)
}
func TestPrivateImageView(t *testing.T) {
//首先上传一个图片 用于测试
policy := rs.PutPolicy{
Scope: bucket + ":" + key,
}
err := io.PutFile(nil, nil, policy.Token(nil), key, localFile, nil)
if err != nil {
t.Errorf("TestPrivateImageView failed: %v", err)
return
}
rawUrl := makeUrl(key)
iv := ImageView{
Mode: 2,
Height: 250,
Quality: 80,
}
imageViewUrl := iv.MakeRequest(rawUrl)
p := rs.GetPolicy{}
imageViewUrlWithToken := p.MakeRequest(imageViewUrl, nil)
resp, err := http.DefaultClient.Get(imageViewUrlWithToken)
if err != nil {
t.Errorf("TestPrivateImageView failed: %v", err)
return
}
defer resp.Body.Close()
if (resp.StatusCode / 100) != 2 {
t.Errorf("TestPrivateImageView failed: resp.StatusCode = %v", resp.StatusCode)
return
}
}
func UploadFile(dir string, read io.Reader) (string, error) {
ACCESS_KEY = Webconfig.UploadConfig.QiniuAccessKey
SECRET_KEY = Webconfig.UploadConfig.QiniuSecretKey
extra := &qiniu_io.PutExtra{
// Bucket: "messagedream",
MimeType: "",
}
var policy = rs.PutPolicy{
Scope: "messagedream",
}
filename := strings.Replace(UUID(), "-", "", -1) + ".jpg"
body, _ := ioutil.ReadAll(read)
h := md5.New()
h.Write(body)
key := "static/upload/" + dir + "/" + filename
ret := new(qiniu_io.PutRet)
buf := bytes.NewBuffer(body)
return key, qiniu_io.Put(nil, ret, policy.Token(nil), key, buf, extra)
}
func handleUpload(w http.ResponseWriter, req *http.Request) {
policy := rs.PutPolicy{Scope: BUCKET, ReturnUrl: "http://localhost:8765/uploaded", EndUser: "******"}
token := policy.Token(nil)
log.Println("token:", token)
uploadForm := fmt.Sprintf(uploadFormFmt, token)
w.Write([]byte(uploadForm))
}
func handleUploadWithKeyAndCustomField(w http.ResponseWriter, req *http.Request) {
policy := rs.PutPolicy{Scope: BUCKET, ReturnUrl: "http://localhost:8765/uploaded"}
token := policy.Token()
log.Println("token:", token)
uploadForm := fmt.Sprintf(uploadWithkeyAndCustomFieldFmt, token)
w.Write([]byte(uploadForm))
}
// URL: /upload/image
// 编辑器上传图片，接收后上传到七牛
func uploadImageHandler(handler *Handler) {
file, header, err := handler.Request.FormFile("editormd-image-file")
if err != nil {
panic(err)
return
}
defer file.Close()
// 检查是否是jpg或png文件
uploadFileType := header.Header["Content-Type"][0]
filenameExtension := ""
if uploadFileType == "image/jpeg" {
filenameExtension = ".jpg"
} else if uploadFileType == "image/png" {
filenameExtension = ".png"
} else if uploadFileType == "image/gif" {
filenameExtension = ".gif"
}
if filenameExtension == "" {
handler.ResponseWriter.Header().Set("Content-Type", "text/html")
handler.ResponseWriter.Write(editorMdUploadImageResult(0, "", "不支持的文件格式，请上传 jpg/png/gif 图片"))
return
}
// 上传到七牛
// 文件名：32位uuid+后缀组成
filename := strings.Replace(uuid.NewUUID().String(), "-", "", -1) + filenameExtension
key := "upload/image/" + filename
ret := new(qiniuIo.PutRet)
var policy = rs.PutPolicy{
Scope: "gopher",
}
err = qiniuIo.Put(
nil,
ret,
policy.Token(nil),
key,
file,
nil,
)
if err != nil {
panic(err)
handler.ResponseWriter.Header().Set("Content-Type", "text/html")
handler.ResponseWriter.Write(editorMdUploadImageResult(0, "", "图片上传到七牛失败"))
return
}
handler.ResponseWriter.Header().Set("Content-Type", "text/html")
handler.ResponseWriter.Write(editorMdUploadImageResult(1, "http://7xj1eq.com1.z0.glb.clouddn.com/"+key, ""))
}
func ResumablePut(cmd string, params ...string) {
if len(params) == 3 || len(params) == 4 || len(params) == 5 {
bucket := params[0]
key := params[1]
localFile := params[2]
mimeType := ""
upHost := "http://upload.qiniu.com"
if len(params) == 4 {
param := params[3]
if strings.HasPrefix(param, "http") {
upHost = param
} else {
mimeType = param
}
}
if len(params) == 5 {
mimeType = params[3]
upHost = params[4]
}
accountS.Get()
mac := digest.Mac{accountS.AccessKey, []byte(accountS.SecretKey)}
policy := rs.PutPolicy{}
policy.Scope = bucket
putExtra := rio.PutExtra{}
if mimeType != "" {
putExtra.MimeType = mimeType
}
conf.UP_HOST = upHost
progressHandler := ProgressHandler{
BlockIndices: make([]int, 0),
BlockProgresses: make(map[int]float32),
}
putExtra.Notify = progressHandler.Notify
putExtra.NotifyErr = progressHandler.NotifyErr
uptoken := policy.Token(&mac)
putRet := rio.PutRet{}
startTime := time.Now()
fStat, statErr := os.Stat(localFile)
if statErr != nil {
log.Error("Local file error", statErr)
return
}
fsize := fStat.Size()
err := rio.PutFile(nil, &putRet, uptoken, key, localFile, &putExtra)
if err != nil {
log.Error("Put file error", err)
} else {
fmt.Println("\r\nPut file", localFile, "=>", bucket, ":", putRet.Key, "(", putRet.Hash, ")", "success!")
}
lastNano := time.Now().UnixNano() - startTime.UnixNano()
lastTime := fmt.Sprintf("%.2f", float32(lastNano)/1e9)
avgSpeed := fmt.Sprintf("%.1f", float32(fsize)*1e6/float32(lastNano))
fmt.Println("Last time:", lastTime, "s, Average Speed:", avgSpeed, "KB/s")
} else {
CmdHelp(cmd)
}
}
func (this *AdminController) QiniuTokens() {
bucket := models.Appconf.String("qiniu::bucket")
putPolicy := rs.PutPolicy{
Scope: bucket,
}
this.Data["json"] = putPolicy.Token(nil)
this.ServeJson()
}
// generate qiniu token
func qntoken(key string) string {
scope := defaultBulket + ":" + key
log.Infof("qiniu scrope: %s", scope)
policy := rs.PutPolicy{
Expires: uint32(time.Now().Unix() + 3600),
Scope: scope,
}
return policy.Token(nil)
}
func upFile(bucket string, key string, localPath string) error {
var ret qiniuio.PutRet
policy := rs.PutPolicy{
Scope: bucket + ":" + key,
}
err := qiniuio.PutFile(nil, &ret, policy.Token(mac), key, localPath, nil)
log.Printf("ret : %+v", ret)
return err
}
func generateQiniuUpToken(scope, accessKey, secretKey string, sizeLimit int64) string {
mac := qauth.Mac{AccessKey: accessKey, SecretKey: []byte(secretKey)}
policy := qrs.PutPolicy{}
policy.Scope = scope
policy.ReturnBody = `{"key": $(key), "mimeType": $(mimeType), "fsize": $(fsize)}`
policy.FsizeLimit = sizeLimit
return policy.Token(&mac)
}
func handleUpload(w http.ResponseWriter, req *http.Request) {
policy := rs.PutPolicy{
Scope: BUCKET,
EndUser: "******",
SaveKey: "$(sha1)",
}
token := policy.Token(nil)
log.Println("token:", token)
uploadForm := fmt.Sprintf(uploadFormFmt, token)
w.Write([]byte(uploadForm))
}
func qiniuUploadImage(file *multipart.File, fileName string) (error, qio.PutRet) {
var ret qio.PutRet
var policy = rs.PutPolicy{
Scope: models.QiniuScope,
}
err := qio.Put(nil, &ret, policy.Token(nil), fileName, *file, nil)
if err != nil {
revel.ERROR.Println("io.Put failed:", err)
}
return err, ret
}
func main() {
ACCESS_KEY = gopher.Config.QiniuAccessKey
SECRET_KEY = gopher.Config.QiniuSecretKey
extra := &qiniu_io.PutExtra{
Bucket: "gopher",
MimeType: "",
CustomMeta: "",
CallbackParams: "",
}
var policy = rs.PutPolicy{
Scope: "gopher",
}
c := gopher.DB.C("users")
var users []gopher.User
c.Find(nil).All(&users)
for _, user := range users {
url := webhelpers.Gravatar(user.Email, 256)
resp, err := http.Get(url)
if err != nil {
fmt.Println("get gravatar image error:", url, err.Error())
return
}
filename := strings.Replace(uuid.NewUUID().String(), "-", "", -1) + ".jpg"
body, _ := ioutil.ReadAll(resp.Body)
h := md5.New()
h.Write(body)
md5Str := fmt.Sprintf("%x", h.Sum(nil))
if md5Str != "ac83818c6d5b6aca4b6f796b6d3cb338" {
// 不是默认头像，上传
key := "avatar/" + filename
ret := new(qiniu_io.PutRet)
buf := bytes.NewBuffer(body)
err = qiniu_io.Put(nil, ret, policy.Token(), key, buf, extra)
if err == nil {
c.Update(bson.M{"_id": user.Id_}, bson.M{"$set": bson.M{"avatar": filename}})
fmt.Printf("upload %s's avatar success: %s\n", user.Email, filename)
} else {
fmt.Printf("upload %s' avatar error: %s\n", user.Email, err.Error())
}
}
resp.Body.Close()
}
}
func GetToken() (string, string) {
key := uuid.NewV4().String()
putPolicy := rs.PutPolicy{
Scope: g_bucket_name + ":" + key,
CallbackUrl: CALLBACK_URL,
CallbackBody: g_callback_body,
ReturnUrl: g_return_url,
ReturnBody: g_return_body,
}
return putPolicy.Token(nil), key
}
func uptoken(bucketName string) string {
putPolicy := rs.PutPolicy{
Scope: bucketName,
//CallbackUrl: callbackUrl,
//CallbackBody:callbackBody,
//ReturnUrl: returnUrl,
//ReturnBody: returnBody,
//AsyncOps: asyncOps,
//EndUser: endUser,
//Expires: expires,
}
return putPolicy.Token(mac)
}
func getInfo(w http.ResponseWriter, r *http.Request) {
userInfo := make(map[string]string)
policy := rs.PutPolicy{
Scope: BUCKET,
EndUser: "******",
SaveKey: "$(sha1)",
}
token := policy.Token(nil)
userInfo["token"] = token
s := strings.Split(r.RemoteAddr, ":")
s[0] = "58.20.0.17"
userInfo["ip"] = s[0]
resp, _ := http.Get("http://ip.taobao.com/service/getIpInfo.php?ip=" + s[0])
defer resp.Body.Close()
by, _ := ioutil.ReadAll(resp.Body)
var ret map[string]interface{}
json.Unmarshal(by, &ret)
var data = ret["data"]
dataMap, ok := data.(map[string]interface{})
if ok {
var country = dataMap["country"]
countryInfo, ok := country.(string)
if ok {
userInfo["country"] = countryInfo
}
var region = dataMap["region"]
regionInfo, ok := region.(string)
if ok {
userInfo["region"] = regionInfo
}
var city = dataMap["city"]
cityInfo, ok := city.(string)
if ok {
userInfo["city"] = cityInfo
}
var isp = dataMap["isp"]
ispInfo, ok := isp.(string)
if ok {
userInfo["isp"] = ispInfo
}
}
for _, v := range r.Header["User-Agent"] {
userInfo["useragent"] = v
}
getInfoTmpl.ExecuteTemplate(w, "getinfo", userInfo)
}
//GET upload access token
func uptoken(bucketName string) string {
putPolicy := rs.PutPolicy{
Scope: bucketName,
//CallbackUrl: "127.0.0.1",
//CallbackBody:callbackBody,
//ReturnUrl: "127.0.0.1",
//ReturnBody: returnBody,
//AsyncOps: asyncOps,
//EndUser: endUser,
//Expires: expires,
PersistentOps: "vframe/jpg/offset/1|saveas/bmVlZGthbmU6MTN2Mg==",
}
return putPolicy.Token(nil)
}
func UploadFile(localFile string, destName string) (addr string, err error) {
policy := new(rs.PutPolicy)
policy.Scope = Bulket
uptoken := policy.Token(nil)
var ret io.PutRet
var extra = new(io.PutExtra)
err = io.PutFile(nil, &ret, uptoken, destName, localFile, extra)
if err != nil {
return
}
addr = "http://" + Bulket + ".qiniudn.com/" + destName
return
}
func uptoken(bucketName string, uid string) string {
//body := "x:uid=" + uid + "&key=$(etag)&size=$(fsize)" // + "&gentime=" + string(time.Now().Unix())
body := "uid=$(x:uid)&audiotitle=$(x:audiotitle)&key=$(etag)&size=$(fsize)" + "&gentime=" + string(time.Now().Unix())
putPolicy := rs.PutPolicy{
Scope: bucketName,
CallbackUrl: "http://www.oohoohoo.com/recqiniu?", //http://<your domain>/recqiniu
CallbackBody: body, //gae body eg:test=$(x:test)&key=$(etag)&size=$(fsize)&uid=$(endUser)
//ReturnUrl: returnUrl,
//ReturnBody: returnBody,
//AsyncOps: asyncOps,
EndUser: uid, //uid
Expires: 3600 * 24 * 7, // 1week?
}
return putPolicy.Token(nil)
}
// 上传到七牛，并返回文件名
func uploadAvatarToQiniu(file io.ReadCloser, contentType string) (filename string, err error) {
isValidateType := false
for _, imgType := range []string{"image/png", "image/jpeg"} {
if imgType == contentType {
isValidateType = true
break
}
}
if !isValidateType {
return "", errors.New("文件类型错误")
}
filenameExtension := ".jpg"
if contentType == "image/png" {
filenameExtension = ".png"
}
// 文件名：32位uuid，不带减号和后缀组成
filename = strings.Replace(uuid.NewUUID().String(), "-", "", -1) + filenameExtension
ACCESS_KEY = Config.QiniuAccessKey
SECRET_KEY = Config.QiniuSecretKey
key := "avatar/" + filename
ret := new(qiniu_io.PutRet)
var policy = rs.PutPolicy{
Scope: "gopher",
}
err = qiniu_io.Put(
nil,
ret,
policy.Token(nil),
key,
file,
nil,
)
if err != nil {
return "", err
}
return filename, nil
}
// 生成上传token
func (this *QiniuController) CreateUpToken() {
ok, data := func() (bool, interface{}) {
member := &models.Member{}
var session interface{}
if session = this.GetSession("WOKUID"); session == nil {
return false, "未登录"
}
if ok := member.FindOne(session.(string)); !ok {
return false, "用户不存在"
}
//查询用户所在job组
job := &models.Job{}
job.FindOne(member.Type)
//判断用户今日上传容量是否达到上限
if member.UploadSize > int64(job.UploadSize)*1024*1024 {
return false, "今日上传已达到限额"
}
// 生成上传到temp目录的token
putPolicy := rs.PutPolicy{
Scope: "woku",
FsizeLimit: 500 * 1024, //最大500kb
Expires: uint32(time.Now().Unix()) + 60,
ReturnBody: `
{
"etag": "$(etag)",
"ext": "$(ext)",
"name": "temp/$(year)/$(mon)/$(day)/$(etag)$(ext)",
"type": "` + this.GetString("type") + `"
}`,
SaveKey: "temp/$(year)/$(mon)/$(day)/$(etag)$(ext)",
MimeLimit: "image/*",
}
return true, putPolicy.Token(nil)
}()
this.Data["json"] = map[string]interface{}{
"ok": ok,
"data": data,
}
this.ServeJson()
}
func InitQiniu() {
conf.ACCESS_KEY = config.Config["qiniu_access_key"]
conf.SECRET_KEY = config.Config["qiniu_secret_key"]
putPolicy := rs.PutPolicy{
Scope: config.Config["qiniu_bucket_name"],
// CallbackUrl: callbackUrl,
// CallbackBody: callbackBody,
// ReturnUrl: returnUrl,
// ReturnBody: returnBody,
// AsyncOps: asyncOps,
// EndUser: endUser,
// Expires: expires,
}
uptoken = putPolicy.Token(nil)
}
//上传文件,检查返回的hash和需要的hash是否一致
func UploadFileWithHash(ctx *Context, localPath string, remotePath string, expectHash string) (err error) {
var ret qiniuIo.PutRet
var extra = &qiniuIo.PutExtra{
CheckCrc: 1,
}
putPolicy := rs.PutPolicy{
Scope: ctx.bucket + ":" + remotePath,
}
uptoken := putPolicy.Token(nil)
err = qiniuIo.PutFile(nil, &ret, uptoken, remotePath, localPath, extra)
//fmt.Println(localPath,remotePath,err)
if err != nil {
return
}
if ret.Hash != expectHash {
return fmt.Errorf("[UploadFileWithHash][remotePath:%s] ret.Hash:[%s]!=expectHash[%s] ", remotePath, ret.Hash, expectHash)
}
return
}
func TestAll(t *testing.T) {
policy := rs.PutPolicy{
Scope: bucket,
}
token := policy.Token(nil)
params := map[string]string{"x:1": "1"}
extra := &PutExtra{
ChunkSize: 128,
MimeType: "text/plain",
Notify: blockNotify,
Params: params,
}
testPut(t, token, nil)
testPutWithoutKey(t, token, extra)
testPutFile(t, token, extra)
testPutFileWithoutKey(t, token, extra)
testXVar(t, token, extra)
}
// Deploys a site to QiniuCloudStorage.
func (s *Site) DeployToQiniu(key, secret, bucket string) error {
q6cfg.ACCESS_KEY = key
q6cfg.SECRET_KEY = secret
// walks _site directory and uploads file to QiniuCloudStorage
walker := func(fn string, fi os.FileInfo, err error) error {
if fi.IsDir() {
return nil
}
rel, _ := filepath.Rel(s.Dest, fn)
logf(MsgUploadFile, rel)
if err != nil {
return err
}
key := filepath.ToSlash(rel)
policy := q6rs.PutPolicy{
Scope: bucket + ":" + key,
Expires: 60,
}
uptoken := policy.Token()
ret := new(q6io.PutRet)
extra := &q6io.PutExtra{MimeType: mime.TypeByExtension(filepath.Ext(rel)), Bucket: bucket}
// try to upload the file ... sometimes this fails due to QiniuCloudStorage
// issues. If so, we'll re-try
if err := q6io.PutFile(nil, ret, uptoken, key, fn, extra); err != nil {
time.Sleep(100 * time.Millisecond) // sleep so that we don't immediately retry
return q6io.PutFile(nil, ret, uptoken, key, fn, extra)
}
// file upload was a success, return nil
return nil
}
return filepath.Walk(s.Dest, walker)
}
func qiniucloudsave(file string) (url string, err error) {
var key string
//get the filename from the file , eg,get "1.txt" from /home/liugenping/1.txt
for _, key = range strings.Split(file, "/") {
}
url = "http://" + g_qiniuEndpoint + "/" + key
putPolicy := rs.PutPolicy{Scope: g_qiniuBucket}
uptoken := putPolicy.Token(nil)
var ret io.PutRet
var extra = &io.PutExtra{}
err = io.PutFile(nil, &ret, uptoken, key, file, extra)
if err != nil {
return "", err
} else {
return url, nil
}
}
// URL: /profile/avatar
// 修改头像,提交到七牛云存储
func changeAvatarHandler(w http.ResponseWriter, r *http.Request) {
user, ok := currentUser(r)
if !ok {
http.Redirect(w, r, "/signin?next=/profile/avatar", http.StatusFound)
return
}
if r.Method == "POST" {
formFile, formHeader, err := r.FormFile("file")
if err != nil {
fmt.Println("changeAvatarHandler:", err.Error())
renderTemplate(w, r, "account/avatar.html", map[string]interface{}{
"user": user,
"error": "请选择图片上传",
})
return
}
defer formFile.Close()
// 检查是否是jpg或png文件
uploadFileType := formHeader.Header["Content-Type"][0]
isValidateType := false
for _, imgType := range []string{"image/png", "image/jpeg"} {
if imgType == uploadFileType {
isValidateType = true
break
}
}
if !isValidateType {
fmt.Println("upload image type error:", uploadFileType)
// 提示错误
renderTemplate(w, r, "account/avatar.html", map[string]interface{}{
"user": user,
"error": "文件类型错误，请选择jpg/png图片上传。",
})
return
}
// 检查文件尺寸是否在500K以内
fileSize := formFile.(Sizer).Size()
if fileSize > 500*1024 {
// > 500K
fmt.Printf("upload image size > 500K: %dK\n", fileSize/1024)
renderTemplate(w, r, "account/avatar.html", map[string]interface{}{
"user": user,
"error": "图片大小大于500K，请选择500K以内图片上传。",
})
return
}
extra := &qiniu_io.PutExtra{
Bucket: "gopher",
MimeType: "",
CustomMeta: "",
CallbackParams: "",
}
ACCESS_KEY = Config.QiniuAccessKey
SECRET_KEY = Config.QiniuSecretKey
filenameExtension := ".jpg"
if uploadFileType == "image/png" {
filenameExtension = ".png"
}
// 文件名：32位uuid，不带减号和后缀组成
filename := strings.Replace(uuid.NewUUID().String(), "-", "", -1) + filenameExtension
key := "avatar/" + filename
ret := new(qiniu_io.PutRet)
var policy = rs.PutPolicy{
Scope: "gopher",
}
err = qiniu_io.Put(nil, ret, policy.Token(), key, formFile, extra)
if err != nil {
fmt.Println("upload to qiniu failed:", err.Error())
renderTemplate(w, r, "account/avatar.html", map[string]interface{}{
"user": user,
"error": "上传失败，请反馈错误",
})
return
}
// 存储远程文件名
c := DB.C("users")
c.Update(bson.M{"_id": user.Id_}, bson.M{"$set": bson.M{"avatar": filename}})
http.Redirect(w, r, "/profile#avatar", http.StatusFound)
return
}
renderTemplate(w, r, "account/avatar.html", map[string]interface{}{"user": user})
}
func getInfo(w http.ResponseWriter, r *http.Request) {
userInfo := make(map[string]string)
policy := rs.PutPolicy{
Scope: BUCKET,
EndUser: "******",
SaveKey: "$(sha1)",
}
token := policy.Token(nil)
fmt.Println(token)
fmt.Printf("%T", token)
userInfo["token"] = token
s := strings.Split(r.RemoteAddr, ":")
s[0] = "58.20.0.17"
userInfo["ip"] = s[0]
resp, _ := http.Get("http://ip.taobao.com/service/getIpInfo.php?ip=" + s[0])
defer resp.Body.Close()
by, _ := ioutil.ReadAll(resp.Body)
var ret map[string]interface{}
json.Unmarshal(by, &ret)
var data = ret["data"]
dataMap, ok := data.(map[string]interface{})
if ok {
var country = dataMap["country"]
countryInfo, ok := country.(string)
if ok {
userInfo["country"] = countryInfo
}
var region = dataMap["region"]
regionInfo, ok := region.(string)
if ok {
userInfo["region"] = regionInfo
}
var city = dataMap["city"]
cityInfo, ok := city.(string)
if ok {
userInfo["city"] = cityInfo
}
var isp = dataMap["isp"]
ispInfo, ok := isp.(string)
if ok {
userInfo["isp"] = ispInfo
}
}
for _, v := range r.Header["User-Agent"] {
userInfo["useragent"] = v
}
if r.Method != "GET" {
r.ParseForm()
email := r.Form["email"][0]
phone := r.Form["phone"][0]
description := r.Form["description"][0]
userInfo["email"] = email
userInfo["phone"] = phone
userInfo["description"] = description
// postsupportvalue := new {
// "Title": {"PiliWebTest"},
// "Type": {"Pili"},
// "Content": {userInfo["ip"] + userInfo["country"] + userInfo["region"] + userInfo["isp"] + userInfo["useragent"] + userInfo["phone"] + userInfo["description"]},
// "Email": {userInfo["email"]},
// "Source": {"Pili"},
// "Service":{"none"},
// "Attachment": {["none"]},
// "TitlePrefix": {"Pili"}
// }
// resp, err := http.PostForm("https://portal.qiniu.com/support/new",postsupportvalue)
}
getInfoTmpl.ExecuteTemplate(w, "getinfo", userInfo)
}
func QiniuUpload(threadCount int, uploadConfigFile string) {
fp, err := os.Open(uploadConfigFile)
if err != nil {
log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err))
return
}
defer fp.Close()
configData, err := ioutil.ReadAll(fp)
if err != nil {
log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err))
return
}
var uploadConfig UploadConfig
err = json.Unmarshal(configData, &uploadConfig)
if err != nil {
log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err))
return
}
if _, err := os.Stat(uploadConfig.SrcDir); err != nil {
log.Error("Upload config error for parameter `SrcDir`,", err)
return
}
dirCache := DirCache{}
currentUser, err := user.Current()
if err != nil {
log.Error("Failed to get current user", err)
return
}
pathSep := string(os.PathSeparator)
jobId := base64.URLEncoding.EncodeToString([]byte(uploadConfig.SrcDir + ":" + uploadConfig.Bucket))
storePath := fmt.Sprintf("%s%s.qshell%squpload%s%s", currentUser.HomeDir, pathSep, pathSep, pathSep, jobId)
err = os.MkdirAll(storePath, 0775)
if err != nil {
log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err))
return
}
cacheFileName := fmt.Sprintf("%s%s%s.cache", storePath, pathSep, jobId)
leveldbFileName := fmt.Sprintf("%s%s%s.ldb", storePath, pathSep, jobId)
totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName)
ldb, err := leveldb.OpenFile(leveldbFileName, nil)
if err != nil {
log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err))
return
}
defer ldb.Close()
//sync
ufp, err := os.Open(cacheFileName)
if err != nil {
log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err))
return
}
defer ufp.Close()
bScanner := bufio.NewScanner(ufp)
bScanner.Split(bufio.ScanLines)
currentFileCount := 0
ldbWOpt := opt.WriteOptions{
Sync: true,
}
upWorkGroup := sync.WaitGroup{}
upCounter := 0
threadThreshold := threadCount + 1
//use host if not empty
if uploadConfig.UpHost != "" {
conf.UP_HOST = uploadConfig.UpHost
}
//set settings
rio.SetSettings(&upSettings)
mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}
//check thread count
for bScanner.Scan() {
line := strings.TrimSpace(bScanner.Text())
items := strings.Split(line, "\t")
if len(items) > 1 {
cacheFname := items[0]
cacheFlmd, _ := strconv.Atoi(items[2])
uploadFileKey := cacheFname
if uploadConfig.IgnoreDir {
if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
uploadFileKey = uploadFileKey[i+1:]
}
}
if uploadConfig.KeyPrefix != "" {
uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
}
//convert \ to / under windows
if runtime.GOOS == "windows" {
uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
}
cacheFilePath := strings.Join([]string{uploadConfig.SrcDir, cacheFname}, pathSep)
fstat, err := os.Stat(cacheFilePath)
if err != nil {
log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err))
return
}
fsize := fstat.Size()
//check leveldb
currentFileCount += 1
ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey)
log.Debug(fmt.Sprintf("Checking %s ...", ldbKey))
//check last modified
ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
flmd, _ := strconv.Atoi(string(ldbFlmd))
//not exist, return ErrNotFound
if err == nil && cacheFlmd == flmd {
continue
}
fmt.Print("\033[2K\r")
fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount,
float32(currentFileCount)*100/float32(totalFileCount))
os.Stdout.Sync()
rsClient := rs.New(&mac)
//worker
upCounter += 1
if upCounter%threadThreshold == 0 {
upWorkGroup.Wait()
}
upWorkGroup.Add(1)
go func() {
defer upWorkGroup.Done()
//check exists
if uploadConfig.CheckExists {
rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
if checkErr != nil {
log.Error(fmt.Sprintf("Stat `%s' error due to `%s'", uploadFileKey, checkErr))
return
} else if rsEntry.Fsize == fsize {
log.Debug("File already exists in bucket, ignore this upload")
return
}
}
//upload
policy := rs.PutPolicy{}
policy.Scope = uploadConfig.Bucket
if uploadConfig.Overwrite {
policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey
policy.InsertOnly = 0
}
policy.Expires = 24 * 3600
uptoken := policy.Token(&mac)
if fsize > PUT_THRESHOLD {
putRet := rio.PutRet{}
err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
if err != nil {
log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
} else {
perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt)
if perr != nil {
log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
}
}
} else {
putRet := fio.PutRet{}
err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
if err != nil {
log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
} else {
perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt)
if perr != nil {
log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
}
}
}
}()
} else {
log.Error(fmt.Sprintf("Error cache line `%s'", line))
}
}
upWorkGroup.Wait()
fmt.Println()
fmt.Println("Upload done!")
}