func getTagStatus(reponame, itemname, tagname string) (string, error) {
uri := "/daemon/" + reponame + "/" + itemname + "/" + tagname
resp, err := commToDaemon("get", uri, nil)
if err != nil {
return "", err
}
result := ds.Result{}
tagStatus := ds.TagStatus{}
result.Data = &tagStatus
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
return "", err
}
} else if resp.StatusCode == http.StatusUnauthorized {
return "", errors.New("Not login.")
} else {
return "", errors.New(string(respbody))
}
return tagStatus.Status, nil
}
func deleteItemsAccordingToHeartbeat(body []byte) {
log.Debug("deleteItemsAccordingToHeartbeat() BEGIN:", string(body))
result := ds.Result{}
itemEvent := &Event{}
result.Data = itemEvent
itemsdelete := []ItemDel{}
itemEvent.Data = &itemsdelete
if err := json.Unmarshal(body, &result); err == nil {
log.Debug("items delete:", itemsdelete)
for _, v := range itemsdelete {
log.Debugf("delete item according to heartbeat: %v/%v\n", v.Repname, v.Itemname)
err := delTagsForDelItem(v.Repname, v.Itemname)
if err != nil {
log.Error(err)
return
}
err = delItem(v.Repname, v.Itemname)
if err != nil {
log.Error(err)
return
}
log.Infof("Delete data item %v/%v according to heartbeat successfully.\n", v.Repname, v.Itemname)
}
} else {
log.Warn("Unmarshal error:", err)
}
}
func getItemStatus(reponame, itemname string) (string, error) {
uri := "/repositories/" + reponame + "/" + itemname
resp, err := commToDaemon("get", uri, nil)
if err != nil {
return "", err
}
result := ds.Result{}
itemInfo := ds.ItemInfo{}
result.Data = &itemInfo
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
return "", err
}
} else {
return "", errors.New(string(respbody))
//showResponse(resp)
}
uri = "/heartbeat/status/" + itemInfo.Create_user
resp, err = commToDaemon("get", uri, nil)
if err != nil {
return "", err
}
itemStatus := ds.ItemStatus{}
result.Data = &itemStatus
respbody, err = ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
return "", err
}
} else {
showResponse(resp)
}
return itemStatus.Status, nil
}
func repoResp(icmd int, respbody []byte, repo, item, tag string) {
//fmt.Println(string(respbody))
result := ds.Result{Code: ResultOK}
if icmd == Repos {
repos := []ds.Repositories{}
result.Data = &repos
err := json.Unmarshal(respbody, &result)
if err != nil {
panic(err)
}
fmt.Printf("%-16s\n", "REPOSITORY") //, "UPDATETIME", "COMMENT")
for _, v := range repos {
fmt.Printf("%-16s\n", v.RepositoryName) //, v.Optime, v.Comment)
}
} else if icmd == ReposReponame {
onerepo := ds.Repository{}
result.Data = &onerepo
err := json.Unmarshal(respbody, &result)
if err != nil {
panic(err)
}
fmt.Printf("REPOSITORY/DATAITEM\n")
for _, v := range onerepo.DataItems {
fmt.Printf("%s/%s\n", repo, v)
}
} else if icmd == ReposReponameDataItem {
itemStatus, err := getItemStatus(repo, item)
if err != nil {
fmt.Println("Error :", err)
return
}
var tagStatus string
repoitemtags := ds.Data{}
result.Data = &repoitemtags
err = json.Unmarshal(respbody, &result)
if err != nil {
fmt.Println("Error :", err)
return
}
abnormalTags, err := getTagStatusOfItem(repo, item)
if err != nil {
fmt.Println("Error :", err)
return
}
repoitemname := repo + "/" + item
ctag := []string{"REPOSITORY/ITEM:TAG"}
cupdatetime := []string{"UPDATETIME"}
ccomment := []string{"COMMENT"}
cstatus := []string{"STATUS"}
if itemStatus == "offline" {
for _, v := range repoitemtags.Taglist {
//fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, v.Tag, v.Optime, v.Comment, "ABNORMAL")
ctag = append(ctag, repo+"/"+item+":"+v.Tag)
cupdatetime = append(cupdatetime, v.Optime)
ccomment = append(ccomment, v.Comment)
cstatus = append(cstatus, "ABNORMAL")
}
} else {
for _, v := range repoitemtags.Taglist {
repoitemtag := repoitemname + ":" + v.Tag
tagStatus = judgeTag(abnormalTags, repoitemtag)
//fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, v.Tag, v.Optime, v.Comment, tagStatus)
ctag = append(ctag, repo+"/"+item+":"+v.Tag)
cupdatetime = append(cupdatetime, v.Optime)
ccomment = append(ccomment, v.Comment)
cstatus = append(cstatus, tagStatus)
}
}
utils.PrintFmt(ctag, cupdatetime, ccomment, cstatus)
} else if icmd == ReposReponameDataItemTag {
itemStatus, err := getItemStatus(repo, item)
if err != nil {
fmt.Println("Error :", err)
return
}
tagStatus, err := getTagStatus(repo, item, tag)
if err != nil {
fmt.Println("Error :", err)
return
}
onetag := ds.Tag{}
result.Data = &onetag
err = json.Unmarshal(respbody, &result)
if err != nil {
fmt.Println("Error :", err)
return
}
//n, _ := fmt.Printf("%s\t%s\t%s\t\t%s\n", "REPOSITORY/ITEM:TAG", "UPDATETIME", "COMMENT", "STATUS")
ctag := []string{"REPOSITORY/ITEM:TAG"}
cupdatetime := []string{"UPDATETIME"}
ccomment := []string{"COMMENT"}
cstatus := []string{"STATUS"}
if itemStatus == "offline" {
//fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, tag, onetag.Optime, onetag.Comment, "ABNORMAL")
ctag = append(ctag, repo+"/"+item+":"+tag)
cupdatetime = append(cupdatetime, onetag.Optime)
ccomment = append(ccomment, onetag.Comment)
cstatus = append(cstatus, "ABNORMAL")
} else {
//fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, tag, onetag.Optime, onetag.Comment, tegStatus)
ctag = append(ctag, repo+"/"+item+":"+tag)
cupdatetime = append(cupdatetime, onetag.Optime)
ccomment = append(ccomment, onetag.Comment)
cstatus = append(cstatus, tagStatus)
}
utils.PrintFmt(ctag, cupdatetime, ccomment, cstatus)
}
}
func GetPulledRepoInfo(dpName, repoName string, offset int64, limit int) ([]ds.PulledItemInfo, error) {
var pulledItemInfo ds.PulledItemInfo
pulledItemInfos := make([]ds.PulledItemInfo, 0)
sql := fmt.Sprintf(`SELECT DPID FROM DH_DP WHERE DPNAME = '%s' AND STATUS = 'A';`, dpName)
row, err := g_ds.QueryRow(sql)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
}
var dpid int
row.Scan(&dpid)
sql = fmt.Sprintf(`SELECT DATAITEM ,ITEMDESC
FROM DH_DP_RPDM_MAP
WHERE DPID = %d AND REPOSITORY = '%s' AND PUBLISH = 'N' AND STATUS = 'A'
ORDER BY RPDMID
LIMIT %v OFFSET %v;`, dpid, repoName, limit, offset)
rows, err := g_ds.QueryRows(sql)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
}
result := ds.Result{}
pages := ds.ResultPages{}
orderInfoSlice := []ds.OrderInfo{}
pages.Results = &orderInfoSlice
result.Data = &pages
for rows.Next() {
rows.Scan(&pulledItemInfo.ItemName, &pulledItemInfo.Location)
path := "/api/subscriptions/pull/" + repoName + "/" + pulledItemInfo.ItemName
resp, err := commToServerGetRsp("get", path, nil)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusUnauthorized {
pulledItemInfo.SignTime = nil
log.Debug("resp.StatusCode == http.StatusUnauthorized")
} else if resp.StatusCode != http.StatusOK {
err = errors.New("request subscriptions api failed.")
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
} else {
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
} else {
err = json.Unmarshal(respbody, &result)
if err != nil {
err = errors.New("unmarshal failed.")
l := log.Error(err)
logq.LogPutqueue(l)
return nil, err
}
if len(orderInfoSlice) > 0 {
//Already order by signtime desc. Get the first member.
pulledItemInfo.SignTime = &orderInfoSlice[0].Signtime
log.Debug("pulledItemInfo.SignTime:", pulledItemInfo.SignTime)
}
}
}
pulledItemInfos = append(pulledItemInfos, pulledItemInfo)
}
log.Debug(pulledItemInfos)
return pulledItemInfos, err
}
func newPubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub tag)")
repo := ps.ByName("repo")
item := ps.ByName("item")
tag := ps.ByName("tag")
log.Println("repo", repo, "item", item, "tag", tag) //log
paras := ds.PubPara{}
reqbody, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("ioutil.ReadAll err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorIOUtil, "Internal Error.", nil)
return
}
err = json.Unmarshal(reqbody, ¶s)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "Internal Error.", nil)
return
}
if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) {
JsonResult(w, http.StatusOK, cmd.ErrorOutMaxLength, "repo or item or tag length out of max length.", nil)
return
}
isExist, err := CheckItemExist(repo, item)
if isExist == false {
if paras.Datapool != "" && paras.ItemDesc != "" {
isExist := CheckDataPoolExist(paras.Datapool)
if isExist == false {
JsonResult(w, http.StatusOK, cmd.ErrorDatapoolNotExits, "datapool not exist.", nil)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Println("http.NewRequest err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("http.DefaultClient.Do err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
respbody, err := ioutil.ReadAll(resp.Body)
if err != nil {
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
result := ds.Result{}
itemInfo := ds.ItemInfo{}
result.Data = &itemInfo
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
log.Println(string(respbody))
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("json.Unmarshal err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
return
}
err = InsertItemToDb(repo, item, paras.Datapool, paras.ItemDesc, itemInfo.Optime)
if err != nil {
log.Println("InsertItemToDb err:", err)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
} else {
JsonResult(w, http.StatusOK, cmd.ErrorItemNotExist, "item not been published.", nil)
return
}
}
//-------------------------------------------这是分割线-----------------------------------------------------
dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag)
log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc) //log
if err != nil {
log.Println("CheckTagAndGetDpPath err:", err)
//rollbackInsertPubItem(w, repo, item)
msg := fmt.Sprintf("Tag '%s' already exist.", tag)
JsonResult(w, http.StatusOK, cmd.ErrorTagAlreadyExist, msg, nil)
return
}
splits := strings.Split(paras.Detail, "/")
fileName := splits[len(splits)-1]
datapool, err := dpdriver.New(dptype)
if err != nil {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
exist, size, err := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName)
if err != nil && exist == false {
l := log.Error(err.Error())
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
if size > 0 {
paras.Comment += SizeToStr(size)
}
body, e := json.Marshal(&struct {
Comment string `json:"comment"`
}{paras.Comment})
if e != nil {
s := "Pub tag error while marshal struct"
log.Println(s)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
//fmt.Println("Marshal err:", err)
return
}
url := DefaultServer + "/api/repositories/" + repo + "/" + item + "/" + tag
log.Println("daemon: connecting to ", url) //log
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Println("DefaultClient.Do err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
defer resp.Body.Close()
//Get server result
result := ds.Result{}
respbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(respbody)) //log
if resp.StatusCode == http.StatusOK {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
err = InsertPubTagToDb(repo, item, tag, fileName, paras.Comment)
if err != nil {
l := log.Error("Insert tag to db error.")
logq.LogPutqueue(l)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
g_DaemonRole = PUBLISHER
} else if resp.StatusCode == http.StatusUnauthorized {
JsonResult(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "no login.", nil)
//rollbackInsertPubItem(w, repo, item)
return
} else {
err = json.Unmarshal(respbody, &result)
if err != nil {
log.Println("Unmarshal err:", err)
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil)
return
}
log.Println(string(respbody))
//rollbackInsertPubItem(w, repo, item)
JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil)
}
return
}
func HeartBeat() {
getEp := false
g_DaemonRole = GetDaemonRoleByPubRecord()
for {
if len(DaemonID) == 0 {
log.Warn("No daemonid. You'd better start datahub with the parameter \"--token\".")
return
}
heartbeatbody := Beatbody{Daemonid: DaemonID, Role: g_DaemonRole}
if getEp == false && len(EntryPoint) == 0 {
EntryPoint = getEntryPoint()
getEp = true
}
if len(EntryPoint) != 0 {
heartbeatbody.Entrypoint = append(heartbeatbody.Entrypoint, EntryPoint)
}
logQueue := logq.LogGetqueue()
if len(logQueue) > 0 {
heartbeatbody.Log = logQueue
}
errortags := checkErrortagsmap(&Errortagsmap)
if len(errortags) != 0 {
heartbeatbody.Errortag = errortags
}
heartbeatbody.CliEntrypoint = CLIEntrypoint + "?daemonauth=" + DaemonAuthrization
jsondata, err := json.Marshal(heartbeatbody)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
}
url := DefaultServerAPI + "/heartbeat"
log.Trace("connecting to", url, string(jsondata))
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsondata))
/*
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
*/
resp, err := http.DefaultClient.Do(req)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
time.Sleep(10 * time.Second)
continue
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
log.Tracef("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body)
result := ds.Result{}
result.Data = &Event{}
if err := json.Unmarshal(body, &result); err == nil {
if result.Code == 0 {
EntryPointStatus = "available"
} else {
EntryPointStatus = "not available"
}
}
if result.Data != nil && result.Data.(*Event).Type == TYPE_ITEM_DEL {
deleteItemsAccordingToHeartbeat(body)
}
time.Sleep(heartbeatTimeout)
}
}
func GetMessages() {
log.Info("start GetMessages from messages server")
var sleepInterval int
var srtInterval string
var e error
url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0"
for AutoPull == true {
if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 {
sleepInterval, e = strconv.Atoi(srtInterval)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
} else {
sleepInterval = 600
}
time.Sleep(time.Duration(sleepInterval) * time.Second)
log.Debug("connecting to", url)
req, err := http.NewRequest("GET", url, nil)
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
continue
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode == http.StatusOK {
log.Debugf("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body)
result := ds.Result{}
Pages := ds.ResultPages{}
MessagesSlice := []Messages{}
Pages.Results = &MessagesSlice
result.Data = &Pages
if err := json.Unmarshal(body, &result); err == nil {
if result.Code == 0 {
log.Debug(result)
for _, v := range MessagesSlice {
if v.Type == "item_event" && v.Data.Event == TAGADDED {
InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD)
}
}
} else {
l := log.Error("Get Messages errror:", result.Code)
logq.LogPutqueue(l)
}
} else {
log.Error(err)
}
} else if resp.StatusCode == http.StatusUnauthorized {
log.Debug("not login", http.StatusUnauthorized)
urllogin := DefaultServerAPI + "/"
reql, err := http.NewRequest("GET", urllogin, nil)
if len(loginBasicAuthStr) > 0 {
reql.Header.Set("Authorization", loginBasicAuthStr)
log.Info("user name:", gstrUsername)
} else {
log.Warn("not login")
continue
}
respl, err := http.DefaultClient.Do(reql)
if err != nil {
log.Error(err)
continue
}
defer respl.Body.Close()
result := &ds.Result{}
log.Println("login return", respl.StatusCode)
if respl.StatusCode == 200 {
body, _ := ioutil.ReadAll(respl.Body)
log.Println(string(body))
result.Data = &tk{}
if err = json.Unmarshal(body, result); err != nil {
log.Error(err)
log.Println(respl.StatusCode, string(body))
continue
} else {
loginAuthStr = "Token " + result.Data.(*tk).Token
loginLogged = true
log.Println(loginAuthStr)
}
}
}
}
}