func Pub(needlogin bool, args []string) (err error) {
//fmt.Println(args)
//return
if len(args) < 2 {
//fmt.Println(ErrMsgArgument)
pubUsage()
return errors.New("args len error!")
}
pub := ds.PubPara{}
//var largs []string = args
var repo, item, tag, argfi, argse string
f := mflag.NewFlagSet("datahub pub", mflag.ContinueOnError)
//f.StringVar(&pub.Datapool, []string{"-datapool", "p"}, "", "datapool name")
f.StringVar(&pub.Accesstype, []string{"-accesstype", "t"}, "private", "dataitem accesstype: private or public")
f.StringVar(&pub.Comment, []string{"-comment", "m"}, "", "comments")
//f.StringVar(&pub.Detail, []string{"-detail", "d"}, "", "tag detail ,for example file name")
f.StringVar(&pub.SupplyStyle, []string{"-supplystyle", "s"}, "batch", "dataitem supplystyle: batch , flow or api")
f.StringVar(&pub.Ch_itemname, []string{"-chinese", "ch"}, "", "dataitem's Chinese name")
f.Usage = pubUsage
if len(args) > 2 {
if err = f.Parse(args[2:]); err != nil {
fmt.Println("Error : parse parameter error.", err)
return err
}
}
//if pub.Ch_itemname == "" {
// fmt.Println("DataHub: Dataitem's Chinese name cannot be empty.")
// pubUsage()
// return
//}
if len(args[0]) == 0 || len(args[1]) == 0 {
fmt.Println(ErrMsgArgument)
pubUsage()
return errors.New("need item or tag error!")
}
argfi = strings.Trim(args[0], "/")
//deal arg[0]
sp := strings.Split(argfi, "/")
if len(sp) != 2 {
fmt.Println(ErrMsgArgument)
pubUsage()
return errors.New("invalid repo/item")
}
repo = sp[0]
sptag := strings.Split(sp[1], ":")
l := len(sptag)
if l == 1 {
item = sptag[0]
argse = strings.Trim(args[1], "/")
se := strings.Split(argse, "://")
if len(se) == 2 && len(se[1]) > 0 {
pub.Datapool = se[0]
pub.ItemDesc = strings.Trim(se[1], "/")
err = PubItem(repo, item, pub, args)
} else {
fmt.Println("DataHub : Please input a valid datapool and path.")
err = errors.New("Error : Please input a valid datapool and path.")
}
} else if l == 2 {
item = sptag[0]
tag = sptag[1]
pub.Detail = args[1]
if len(args) == 2 || (len(args) == 3 && strings.Contains(args[2], "-")) {
PubTag(repo, item, tag, pub, args)
} else {
if len(strings.Split(args[2], ":")) != 2 || strings.Split(args[2], ":")[0] == "" {
fmt.Printf("DataHub : Invalid argument.\nSee '%s --help'.\n", f.Name())
return
}
datapool := strings.Split(args[2], ":")[0]
pub.Datapool = datapool
if len(strings.Split(strings.Split(args[2], ":")[1], "//")) != 2 || strings.Split(strings.Split(args[2], ":")[1], "//")[1] == "" {
fmt.Printf("DataHub : Invalid argument.\nSee '%s --help'.\n", f.Name())
return
}
itemDesc := strings.Split(strings.Split(args[2], ":")[1], "//")[1]
pub.ItemDesc = itemDesc
PubTag(repo, item, tag, pub, args)
}
} else {
fmt.Printf("DataHub : Invalid argument.\nSee '%s --help'.\n", f.Name())
return errors.New("Invalid argument.")
}
return err
}
func PubItem(repo, item string, p ds.PubPara, args []string) (err error) {
if p.Ch_itemname == "" {
p.Ch_itemname = item
}
url := repo + "/" + item
if len(p.Accesstype) == 0 {
p.Accesstype = PRIVATE
}
p.Accesstype = strings.ToLower(p.Accesstype)
if p.Accesstype != PRIVATE && p.Accesstype != PUBLIC {
fmt.Println("Error : Invalid accesstype, e.g accesstype=public, private")
return
}
if len(p.SupplyStyle) == 0 {
p.SupplyStyle = BATCH
}
p.SupplyStyle = strings.ToLower(p.SupplyStyle)
if p.SupplyStyle != BATCH && p.SupplyStyle != FLOW && p.SupplyStyle != API {
fmt.Println("Error : Invalid supplystyle, e.g supplystyle=batch, flow, api")
return
}
if len(p.Datapool) == 0 {
fmt.Println("DataHub : Publishing dataitem requires a parameter \"--datapool=???\" .")
return
}
jsonData, err := json.Marshal(p)
if err != nil {
fmt.Println("Error : Marshal pubdata error while publishing dateitem.")
return err
}
resp, err := commToDaemon("POST", "/repositories/"+url, jsonData)
if err != nil {
fmt.Println("Error :", err)
return err
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode == http.StatusOK {
result := ds.Result{}
err = json.Unmarshal(body, &result)
if err != nil {
fmt.Println("Error : Pub error.", err) //todo add http code
return err
} else {
if result.Code == 0 {
fmt.Println("DataHub : Successed in publishing.")
} else {
fmt.Printf("Error : %v\n", result.Msg)
}
}
} else if resp.StatusCode == http.StatusUnauthorized {
if err = Login(false, nil); err == nil {
Pub(true, args)
} else {
fmt.Println(err)
}
} else {
result := ds.Result{}
err = json.Unmarshal(body, &result)
if err != nil {
fmt.Println("Error : Pub error.", err)
return err
}
if result.Code == ServerErrResultCode1008 {
fmt.Printf("Error : Dataitem '%s' already exists.\n", item)
} else if result.Code == ServerErrResultCode4010 {
fmt.Printf("Error : Datapool '%s' not found.\n", p.Datapool)
} else if result.Code == ServerErrResultCode1011 {
fmt.Println("Error : Only 50 items should be included within each repository.")
} else {
fmt.Println("Error :", result.Msg)
}
}
//err = pubResp(url, jsonData, args)
return err
}
func newPubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub tag)")
repo := ps.ByName("repo")
item := ps.ByName("item")
tag := ps.ByName("tag")
log.Println("repo", repo, "item", item, "tag", tag) //log
paras := ds.PubPara{}
reqbody, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("ioutil.ReadAll err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorIOUtil, "Internal Error.", nil)
return
}
err = json.Unmarshal(reqbody, ¶s)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "Internal Error.", nil)
return
}
if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) {
JsonResult(w, http.StatusOK, cmd.ErrorOutMaxLength, "repo or item or tag length out of max length.", nil)
return
}
isExist, err := CheckItemExist(repo, item)
if isExist == false {
if paras.Datapool != "" && paras.ItemDesc != "" {
isExist := CheckDataPoolExist(paras.Datapool)
if isExist == false {
JsonResult(w, http.StatusOK, cmd.ErrorDatapoolNotExits, "datapool not exist.", nil)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Println("http.NewRequest err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("http.DefaultClient.Do err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
result := ds.Result{}
itemInfo := ds.ItemInfo{}
result.Data = &itemInfo
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
log.Println(string(respbody))
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
return
}
err = InsertItemToDb(repo, item, paras.Datapool, paras.ItemDesc, itemInfo.Optime)
if err != nil {
log.Println("InsertItemToDb err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
JsonResult(w, http.StatusOK, cmd.ErrorItemNotExist, "item not been published.", nil)
return
}
}
//-------------------------------------------这是分割线-----------------------------------------------------
dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag)
log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc) //log
if err != nil {
log.Println("CheckTagAndGetDpPath err:", err)
//rollbackInsertPubItem(w, repo, item)
msg := fmt.Sprintf("Tag '%s' already exist.", tag)
JsonResult(w, http.StatusOK, cmd.ErrorTagAlreadyExist, msg, nil)
return
}
splits := strings.Split(paras.Detail, "/")
fileName := splits[len(splits)-1]
datapool, err := dpdriver.New(dptype)
if err != nil {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
exist, size, err := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName)
if err != nil && exist == false {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
if size > 0 {
paras.Comment += SizeToStr(size)
}
body, e := json.Marshal(&struct {
Comment string `json:"comment"`
}{paras.Comment})
if e != nil {
s := "Pub tag error while marshal struct"
log.Println(s)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
//fmt.Println("Marshal err:", err)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item + "/" + tag
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("DefaultClient.Do err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
//Get server result
result := ds.Result{}
respbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(respbody)) //log
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
err = InsertPubTagToDb(repo, item, tag, fileName, paras.Comment)
if err != nil {
l := log.Error("Insert tag to db error.")
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
g_DaemonRole = PUBLISHER
} else if resp.StatusCode == http.StatusUnauthorized {
JsonResult(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "no login.", nil)
//rollbackInsertPubItem(w, repo, item)
return
} else {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
log.Println(string(respbody))
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
}
return
}
func pubItemHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub dataitem)")
repo := ps.ByName("repo")
item := ps.ByName("item")
if CheckLength(w, repo, MaxRepoLength) == false {
return
}
if CheckLength(w, item, MaxItemLength) == false {
return
}
if len(loginAuthStr) == 0 {
HttpNoData(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "Unlogin")
return
}
pub := ds.PubPara{}
pub.ItemDesc = strings.Trim(pub.ItemDesc, "/")
if strings.Contains(pub.ItemDesc, "/") == true {
log.Println("The path of item can't contain '/'.", pub.ItemDesc)
w.Write([]byte(`{"msg":"The path of item can't contain '/'."}`))
return
}
if CheckLength(w, pub.Comment, MaxCommentLength) == false {
return
}
reqBody, _ := ioutil.ReadAll(r.Body)
if err := json.Unmarshal(reqBody, &pub); err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "pub dataitem error while unmarshal reqBody")
return
}
if CheckDataPoolExist(pub.Datapool) == false {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal,
fmt.Sprintf("Datapool %s not found", pub.Datapool))
return
}
//ToDO check item dir exist
priceplans := []PricePlan{}
meta, sample, priceplans := GetMetaAndSampleAndPricePlan(pub.Datapool, pub.ItemDesc)
icpub := ic{AccessType: pub.Accesstype,
Comment: pub.Comment,
Meta: meta,
Sample: sample,
Ch_itemname: pub.Ch_itemname}
//{"itemaccesstype":"private","comment":"","meta":" ","label":{"sys":{"supply_style":"batch"}}}
isys := Sys{Supplystyle: pub.SupplyStyle}
icpub.Slabel = Label{Ssys: isys}
if len(priceplans) > 0 {
log.Info(priceplans)
icpub.PricePlans = priceplans
}
body, err := json.Marshal(icpub)
if err != nil {
s := "pub dataitem error while marshal icpub struct"
log.Println(s)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorMarshal, s)
return
}
log.Println(string(body))
log.Println("daemon: connecting to", DefaultServer+r.URL.Path)
req, err := http.NewRequest("POST", DefaultServer+r.URL.Path, bytes.NewBuffer(body))
req.Header.Set("Authorization", loginAuthStr)
resp, err := http.DefaultClient.Do(req)
if err != nil {
s := "Pub dataitem service unavailable"
HttpNoData(w, http.StatusServiceUnavailable, cmd.ErrorServiceUnavailable, s)
return
}
defer resp.Body.Close()
//Get server result
rbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(rbody))
if resp.StatusCode == http.StatusOK {
err := MkdirForDataItem(repo, item, pub.Datapool, pub.ItemDesc)
if err != nil {
RollBackItem(repo, item)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
fmt.Sprintf("Mkdir error! %s", err.Error()))
} else {
createTime := time.Now().String()
err = InsertItemToDb(repo, item, pub.Datapool, pub.ItemDesc, createTime)
if err != nil {
RollBackItem(repo, item)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
"Insert dataitem to datapool error, please check it immediately!")
} else {
HttpNoData(w, http.StatusOK, cmd.ResultOK, "OK")
g_DaemonRole = PUBLISHER
}
}
} else {
result := ds.Result{}
err = json.Unmarshal(rbody, &result)
if err != nil {
s := "Pub dataitem error while unmarshal server response"
log.Println(s, err)
HttpNoData(w, resp.StatusCode, cmd.ErrorUnmarshal, s)
return
}
log.Println(resp.StatusCode, result.Msg)
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
}
return
}
func pubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub tag)")
pub := ds.PubPara{}
if CheckLength(w, pub.Comment, MaxCommentLength) == false {
return
}
reqBody, _ := ioutil.ReadAll(r.Body)
if err := json.Unmarshal(reqBody, &pub); err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "pub tag error while unmarshal reqBody")
return
}
if len(pub.Detail) == 0 {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "tag detail is not found")
return
}
repo := ps.ByName("repo")
item := ps.ByName("item")
tag := ps.ByName("tag")
log.Println("repo", repo, "item", item, "tag", tag)
if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) {
return
}
//get DpFullPath and check whether repo/dataitem has been published
dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag)
log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc)
if err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorTagAlreadyExist, err.Error())
return
}
splits := strings.Split(pub.Detail, "/")
fileName := splits[len(splits)-1]
//DpItemFullPath := dpconn + "/" + itemDesc
//DestFullPathFileName := DpItemFullPath + "/" + fileName
datapool, err := dpdriver.New(dptype)
if err != nil {
l := log.Error(err.Error())
logq.LogPutqueue(l)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorDatapoolNotExits, err.Error())
return
}
exist, size, errc := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName)
if errc != nil && exist == false {
//errlog := fmt.Sprintf("File %v not found", DestFullPathFileName)
l := log.Error(errc.Error())
logq.LogPutqueue(l)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorFileNotExist, errc.Error())
return
}
/*if isFileExists(DestFullPathFileName) == false {
errlog := fmt.Sprintf("File %v not found", DestFullPathFileName)
l := log.Error(errlog)
logq.LogPutqueue(l)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorFileNotExist, errlog)
return
}
if size, err := GetFileSize(DestFullPathFileName); err != nil {
l := log.Errorf("Get %s size error, %v", DestFullPathFileName, err)
logq.LogPutqueue(l)
} else {
pub.Comment += SizeToStr(size)
//fmt.Sprintf(" Size:%v ", size)
}*/
if size > 0 {
pub.Comment += SizeToStr(size)
}
body, e := json.Marshal(&struct {
Comment string `json:"comment"`
}{pub.Comment})
if e != nil {
s := "Pub tag error while marshal struct"
log.Println(s)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorMarshal, s)
return
}
err = InsertPubTagToDb(repo, item, tag, fileName, pub.Comment)
if err != nil {
log.Error("Insert tag to db error.")
return
}
log.Println("daemon: connecting to ", DefaultServer+r.URL.Path)
req, err := http.NewRequest("POST", DefaultServer+r.URL.Path, bytes.NewBuffer(body))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
s := "Pub tag service unavailable"
HttpNoData(w, http.StatusServiceUnavailable, cmd.ErrorServiceUnavailable, s)
return
}
defer resp.Body.Close()
//Get server result
rbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(rbody))
if resp.StatusCode == http.StatusOK {
if dptype == "file" {
//AddtoMonitor(DestFullPathFileName, repo+"/"+item+":"+tag) //do not monitor temporarily
}
HttpNoData(w, http.StatusOK, cmd.ResultOK, "OK")
g_DaemonRole = PUBLISHER
} else {
err = rollbackInsertPubTagToDb(repo, item, tag)
if err != nil {
log.Error("rollbackInsertPubTagToDb error :", err)
return
}
result := ds.Result{}
err = json.Unmarshal(rbody, &result)
if err != nil {
s := "Pub dataitem error while unmarshal server response"
log.Println(s)
HttpNoData(w, resp.StatusCode, cmd.ErrorUnmarshal, s)
return
}
log.Println(resp.StatusCode, result.Msg)
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
}
return
}