func MkdirForDataItem(repo, item, dpname, itemdesc string) (err error) {
dpconn, dptype := GetDataPoolDpconnAndDptype(dpname)
if len(dpconn) != 0 {
datapoolOpt, e := dpdriver.New(dptype)
if e != nil {
return e
}
err = datapoolOpt.CheckItemLocation(dpname, dpconn, itemdesc)
return err
} else {
return errors.New(fmt.Sprintf("dpconn is not found for datapool %s", dpname))
}
return nil
}
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
logq.LogPutqueue(l)
r.ParseForm()
sRepoName := ps.ByName("repo")
sDataItem := ps.ByName("dataitem")
sTag := ps.ByName("tag")
log.Info(sRepoName, sDataItem, sTag)
jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
var irpdmid, idpid int
var stagdetail, itemdesc, dpconn, dpname, dptype string
msg := &ds.MsgResp{}
msg.Msg = "OK."
irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
if len(itemdesc) == 0 {
itemdesc = sRepoName + "_" + sDataItem
}
log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)
stagdetail = GetTagDetail(irpdmid, sTag)
log.Debug("tagdetail", stagdetail)
if len(stagdetail) == 0 {
l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
logq.LogPutqueue(l)
http.Error(rw, sTag+" not found", http.StatusNotFound)
return
}
dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)
datapool, err := dpdriver.New(dptype)
if err != nil {
WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
return
}
filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
log.Println("filename:", filepathname)
if exists := isFileExists(filepathname); !exists {
l := log.Error(filepathname, "not found")
logq.LogPutqueue(l)
putToJobQueue(jobtag, filepathname, "N/A", -1)
msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
resp, _ := json.Marshal(msg)
respStr := string(resp)
rw.WriteHeader(http.StatusNotFound)
fmt.Fprintln(rw, respStr)
return
}
tokenValid := false
retmsg := ""
token := r.Form.Get("token")
username := r.Form.Get("username")
log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
if len(token) > 0 && len(username) > 0 {
log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
logq.LogPutqueue(l)
}
log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
//rw.Header().Set("Source-FileName", stagdetail)
bmd5, err := ComputeMd5(filepathname)
strmd5 := fmt.Sprintf("%x", bmd5)
if err != nil {
log.Error(filepathname, err, bmd5, strmd5)
} else {
rw.Header().Set("X-Source-MD5", strmd5)
}
rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))
l = log.Info("transfering", filepathname, bmd5, strmd5)
logq.LogPutqueue(l)
jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
http.ServeFile(rw, r, filepathname)
updateJobQueue(jobid, "transfered", 0)
return
}
func pubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub tag)")
pub := ds.PubPara{}
if CheckLength(w, pub.Comment, MaxCommentLength) == false {
return
}
reqBody, _ := ioutil.ReadAll(r.Body)
if err := json.Unmarshal(reqBody, &pub); err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "pub tag error while unmarshal reqBody")
return
}
if len(pub.Detail) == 0 {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "tag detail is not found")
return
}
repo := ps.ByName("repo")
item := ps.ByName("item")
tag := ps.ByName("tag")
log.Println("repo", repo, "item", item, "tag", tag)
if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) {
return
}
//get DpFullPath and check whether repo/dataitem has been published
dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag)
log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc)
if err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorTagAlreadyExist, err.Error())
return
}
splits := strings.Split(pub.Detail, "/")
fileName := splits[len(splits)-1]
//DpItemFullPath := dpconn + "/" + itemDesc
//DestFullPathFileName := DpItemFullPath + "/" + fileName
datapool, err := dpdriver.New(dptype)
if err != nil {
l := log.Error(err.Error())
logq.LogPutqueue(l)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorDatapoolNotExits, err.Error())
return
}
exist, size, errc := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName)
if errc != nil && exist == false {
//errlog := fmt.Sprintf("File %v not found", DestFullPathFileName)
l := log.Error(errc.Error())
logq.LogPutqueue(l)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorFileNotExist, errc.Error())
return
}
/*if isFileExists(DestFullPathFileName) == false {
errlog := fmt.Sprintf("File %v not found", DestFullPathFileName)
l := log.Error(errlog)
logq.LogPutqueue(l)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorFileNotExist, errlog)
return
}
if size, err := GetFileSize(DestFullPathFileName); err != nil {
l := log.Errorf("Get %s size error, %v", DestFullPathFileName, err)
logq.LogPutqueue(l)
} else {
pub.Comment += SizeToStr(size)
//fmt.Sprintf(" Size:%v ", size)
}*/
if size > 0 {
pub.Comment += SizeToStr(size)
}
body, e := json.Marshal(&struct {
Comment string `json:"comment"`
}{pub.Comment})
if e != nil {
s := "Pub tag error while marshal struct"
log.Println(s)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorMarshal, s)
return
}
err = InsertPubTagToDb(repo, item, tag, fileName, pub.Comment)
if err != nil {
log.Error("Insert tag to db error.")
return
}
log.Println("daemon: connecting to ", DefaultServer+r.URL.Path)
req, err := http.NewRequest("POST", DefaultServer+r.URL.Path, bytes.NewBuffer(body))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
s := "Pub tag service unavailable"
HttpNoData(w, http.StatusServiceUnavailable, cmd.ErrorServiceUnavailable, s)
return
}
defer resp.Body.Close()
//Get server result
rbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(rbody))
if resp.StatusCode == http.StatusOK {
if dptype == "file" {
//AddtoMonitor(DestFullPathFileName, repo+"/"+item+":"+tag) //do not monitor temporarily
}
HttpNoData(w, http.StatusOK, cmd.ResultOK, "OK")
g_DaemonRole = PUBLISHER
} else {
err = rollbackInsertPubTagToDb(repo, item, tag)
if err != nil {
log.Error("rollbackInsertPubTagToDb error :", err)
return
}
result := ds.Result{}
err = json.Unmarshal(rbody, &result)
if err != nil {
s := "Pub dataitem error while unmarshal server response"
log.Println(s)
HttpNoData(w, resp.StatusCode, cmd.ErrorUnmarshal, s)
return
}
log.Println(resp.StatusCode, result.Msg)
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
}
return
}
func newPubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub tag)")
repo := ps.ByName("repo")
item := ps.ByName("item")
tag := ps.ByName("tag")
log.Println("repo", repo, "item", item, "tag", tag) //log
paras := ds.PubPara{}
reqbody, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("ioutil.ReadAll err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorIOUtil, "Internal Error.", nil)
return
}
err = json.Unmarshal(reqbody, ¶s)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "Internal Error.", nil)
return
}
if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) {
JsonResult(w, http.StatusOK, cmd.ErrorOutMaxLength, "repo or item or tag length out of max length.", nil)
return
}
isExist, err := CheckItemExist(repo, item)
if isExist == false {
if paras.Datapool != "" && paras.ItemDesc != "" {
isExist := CheckDataPoolExist(paras.Datapool)
if isExist == false {
JsonResult(w, http.StatusOK, cmd.ErrorDatapoolNotExits, "datapool not exist.", nil)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Println("http.NewRequest err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("http.DefaultClient.Do err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
result := ds.Result{}
itemInfo := ds.ItemInfo{}
result.Data = &itemInfo
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
log.Println(string(respbody))
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
return
}
err = InsertItemToDb(repo, item, paras.Datapool, paras.ItemDesc, itemInfo.Optime)
if err != nil {
log.Println("InsertItemToDb err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
JsonResult(w, http.StatusOK, cmd.ErrorItemNotExist, "item not been published.", nil)
return
}
}
//-------------------------------------------这是分割线-----------------------------------------------------
dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag)
log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc) //log
if err != nil {
log.Println("CheckTagAndGetDpPath err:", err)
//rollbackInsertPubItem(w, repo, item)
msg := fmt.Sprintf("Tag '%s' already exist.", tag)
JsonResult(w, http.StatusOK, cmd.ErrorTagAlreadyExist, msg, nil)
return
}
splits := strings.Split(paras.Detail, "/")
fileName := splits[len(splits)-1]
datapool, err := dpdriver.New(dptype)
if err != nil {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
exist, size, err := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName)
if err != nil && exist == false {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
if size > 0 {
paras.Comment += SizeToStr(size)
}
body, e := json.Marshal(&struct {
Comment string `json:"comment"`
}{paras.Comment})
if e != nil {
s := "Pub tag error while marshal struct"
log.Println(s)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
//fmt.Println("Marshal err:", err)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item + "/" + tag
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("DefaultClient.Do err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
//Get server result
result := ds.Result{}
respbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(respbody)) //log
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
err = InsertPubTagToDb(repo, item, tag, fileName, paras.Comment)
if err != nil {
l := log.Error("Insert tag to db error.")
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
g_DaemonRole = PUBLISHER
} else if resp.StatusCode == http.StatusUnauthorized {
JsonResult(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "no login.", nil)
//rollbackInsertPubItem(w, repo, item)
return
} else {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
log.Println(string(respbody))
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
}
return
}
/*download routine, supports resuming broken downloads.*/
func download(uri string, p ds.DsPull, w http.ResponseWriter, c chan int) (int64, error) {
log.Printf("we are going to download %s, save to dp=%s,name=%s\n", uri, p.Datapool, p.DestName)
var out *os.File
var err error
var destfilename, tmpdestfilename, tmpdir, dpconn, dptype string
dpconn, dptype = GetDataPoolDpconnAndDptype(p.Datapool)
if len(dpconn) == 0 {
err = fmt.Errorf("dpconn is null! datapool:%s ", p.Datapool)
return ErrLogAndResp(c, w, http.StatusBadRequest, cmd.ErrorNoRecord, err)
}
//New a datapool object
datapool, err := dpdriver.New(dptype)
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
}
destfilename, tmpdir, tmpdestfilename = datapool.GetDestFileName(dpconn, p.ItemDesc, p.DestName)
os.MkdirAll(tmpdir, 0777)
log.Info("open tmp destfile name:", tmpdestfilename)
out, err = os.OpenFile(tmpdestfilename, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorOpenFile, err)
}
stat, err := out.Stat()
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorStatFile, err)
}
out.Seek(stat.Size(), 0)
req, err := http.NewRequest("GET", uri, nil)
req.Header.Set("User-Agent", "go-downloader")
/* Set download starting position with 'Range' in HTTP header*/
req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
log.Printf("%v bytes had already been downloaded.\n", stat.Size())
log.Debug(EnvDebug("http_proxy", false))
resp, err := http.DefaultClient.Do(req)
/*Save response body to file only when HTTP 2xx received. TODO*/
if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
log.Error("http error", err)
if resp != nil {
body, _ := ioutil.ReadAll(resp.Body)
l := log.Error("http status code:", resp.StatusCode, "response Body:", string(body), err)
logq.LogPutqueue(l)
struMsg := &ds.MsgResp{}
json.Unmarshal(body, struMsg)
msg := struMsg.Msg
if resp.StatusCode == 416 {
msg = tmpdestfilename + " has already been downloaded."
}
r, _ := buildResp(resp.StatusCode, msg, nil)
w.WriteHeader(resp.StatusCode)
w.Write(r)
} else {
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorOtherError, err.Error())
}
filesize := stat.Size()
out.Close()
if filesize == 0 {
os.Remove(tmpdestfilename)
}
c <- -1
return 0, err
}
defer resp.Body.Close()
HttpNoData(w, http.StatusOK, cmd.ResultOK, strret)
//write channel
c <- 1
jobtag := p.Repository + "/" + p.Dataitem + ":" + p.Tag
srcsize, err := strconv.ParseInt(resp.Header.Get("X-Source-FileSize"), DECIMAL_BASE, INT_SIZE_64)
md5str := resp.Header.Get("X-Source-MD5")
status := "downloading"
log.Info("pull tag:", jobtag, tmpdestfilename, status, srcsize)
jobid := putToJobQueue(jobtag, tmpdestfilename, status, srcsize)
n, err := io.Copy(out, resp.Body)
if err != nil {
out.Close()
bl := log.Error(err)
logq.LogPutqueue(bl)
dlsize, e := GetFileSize(tmpdestfilename)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
status = "failed"
updateJobQueue(jobid, status, dlsize)
return 0, err
}
out.Close()
status = "downloaded"
if len(md5str) > 0 {
bmd5, err := ComputeMd5(tmpdestfilename)
bmd5str := fmt.Sprintf("%x", bmd5)
log.Debug("md5", md5str, tmpdestfilename, bmd5str)
if err != nil {
log.Error(tmpdestfilename, err, bmd5)
} else if md5str != bmd5str {
l := log.Errorf("check md5 code error! src md5:%v, local md5:%v", md5str, bmd5str)
logq.LogPutqueue(l)
status = "md5 error"
updateJobQueue(jobid, status, 0)
return n, nil
}
}
log.Printf("%d bytes downloaded.", n)
if err := MoveFromTmp(tmpdestfilename, destfilename); err != nil {
status = "MoveFromTmp error"
}
dlsize, e := GetFileSize(destfilename)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
status = datapool.StoreFile(status, destfilename, dpconn, p.Datapool, p.ItemDesc, p.DestName)
updateJobQueue(jobid, status, dlsize)
tagComment := GetTagComment(p.Repository, p.Dataitem, p.Tag)
InsertTagToDb(true, p, tagComment)
return n, nil
}