func getTagStatus(reponame, itemname, tagname string) (string, error) { uri := "/daemon/" + reponame + "/" + itemname + "/" + tagname resp, err := commToDaemon("get", uri, nil) if err != nil { return "", err } result := ds.Result{} tagStatus := ds.TagStatus{} result.Data = &tagStatus respbody, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode == http.StatusOK { err = json.Unmarshal(respbody, &result) if err != nil { return "", err } } else if resp.StatusCode == http.StatusUnauthorized { return "", errors.New("Not login.") } else { return "", errors.New(string(respbody)) } return tagStatus.Status, nil }
func deleteItemsAccordingToHeartbeat(body []byte) { log.Debug("deleteItemsAccordingToHeartbeat() BEGIN:", string(body)) result := ds.Result{} itemEvent := &Event{} result.Data = itemEvent itemsdelete := []ItemDel{} itemEvent.Data = &itemsdelete if err := json.Unmarshal(body, &result); err == nil { log.Debug("items delete:", itemsdelete) for _, v := range itemsdelete { log.Debugf("delete item according to heartbeat: %v/%v\n", v.Repname, v.Itemname) err := delTagsForDelItem(v.Repname, v.Itemname) if err != nil { log.Error(err) return } err = delItem(v.Repname, v.Itemname) if err != nil { log.Error(err) return } log.Infof("Delete data item %v/%v according to heartbeat successfully.\n", v.Repname, v.Itemname) } } else { log.Warn("Unmarshal error:", err) } }
func getItemStatus(reponame, itemname string) (string, error) { uri := "/repositories/" + reponame + "/" + itemname resp, err := commToDaemon("get", uri, nil) if err != nil { return "", err } result := ds.Result{} itemInfo := ds.ItemInfo{} result.Data = &itemInfo respbody, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode == http.StatusOK { err = json.Unmarshal(respbody, &result) if err != nil { return "", err } } else { return "", errors.New(string(respbody)) //showResponse(resp) } uri = "/heartbeat/status/" + itemInfo.Create_user resp, err = commToDaemon("get", uri, nil) if err != nil { return "", err } itemStatus := ds.ItemStatus{} result.Data = &itemStatus respbody, err = ioutil.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode == http.StatusOK { err = json.Unmarshal(respbody, &result) if err != nil { return "", err } } else { showResponse(resp) } return itemStatus.Status, nil }
func repoResp(icmd int, respbody []byte, repo, item, tag string) { //fmt.Println(string(respbody)) result := ds.Result{Code: ResultOK} if icmd == Repos { repos := []ds.Repositories{} result.Data = &repos err := json.Unmarshal(respbody, &result) if err != nil { panic(err) } fmt.Printf("%-16s\n", "REPOSITORY") //, "UPDATETIME", "COMMENT") for _, v := range repos { fmt.Printf("%-16s\n", v.RepositoryName) //, v.Optime, v.Comment) } } else if icmd == ReposReponame { onerepo := ds.Repository{} result.Data = &onerepo err := json.Unmarshal(respbody, &result) if err != nil { panic(err) } fmt.Printf("REPOSITORY/DATAITEM\n") for _, v := range onerepo.DataItems { fmt.Printf("%s/%s\n", repo, v) } } else if icmd == ReposReponameDataItem { itemStatus, err := getItemStatus(repo, item) if err != nil { fmt.Println("Error :", err) return } var tagStatus string repoitemtags := ds.Data{} result.Data = &repoitemtags err = json.Unmarshal(respbody, &result) if err != nil { fmt.Println("Error :", err) return } abnormalTags, err := getTagStatusOfItem(repo, item) if err != nil { fmt.Println("Error :", err) return } repoitemname := repo + "/" + item ctag := []string{"REPOSITORY/ITEM:TAG"} cupdatetime := []string{"UPDATETIME"} ccomment := []string{"COMMENT"} cstatus := []string{"STATUS"} if itemStatus == "offline" { for _, v := range repoitemtags.Taglist { //fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, v.Tag, v.Optime, v.Comment, "ABNORMAL") ctag = append(ctag, repo+"/"+item+":"+v.Tag) cupdatetime = append(cupdatetime, v.Optime) ccomment = append(ccomment, v.Comment) cstatus = append(cstatus, "ABNORMAL") } } else { for _, v := range repoitemtags.Taglist { repoitemtag := repoitemname + ":" + v.Tag tagStatus = judgeTag(abnormalTags, repoitemtag) //fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, v.Tag, v.Optime, v.Comment, tagStatus) ctag = append(ctag, repo+"/"+item+":"+v.Tag) cupdatetime = append(cupdatetime, v.Optime) ccomment = append(ccomment, v.Comment) cstatus = append(cstatus, tagStatus) } } utils.PrintFmt(ctag, cupdatetime, ccomment, cstatus) } else if icmd == ReposReponameDataItemTag { itemStatus, err := getItemStatus(repo, item) if err != nil { fmt.Println("Error :", err) return } tagStatus, err := getTagStatus(repo, item, tag) if err != nil { fmt.Println("Error :", err) return } onetag := ds.Tag{} result.Data = &onetag err = json.Unmarshal(respbody, &result) if err != nil { fmt.Println("Error :", err) return } //n, _ := fmt.Printf("%s\t%s\t%s\t\t%s\n", "REPOSITORY/ITEM:TAG", "UPDATETIME", "COMMENT", "STATUS") ctag := []string{"REPOSITORY/ITEM:TAG"} cupdatetime := []string{"UPDATETIME"} ccomment := []string{"COMMENT"} cstatus := []string{"STATUS"} if itemStatus == "offline" { //fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, tag, onetag.Optime, onetag.Comment, "ABNORMAL") ctag = append(ctag, repo+"/"+item+":"+tag) cupdatetime = append(cupdatetime, onetag.Optime) ccomment = append(ccomment, onetag.Comment) cstatus = append(cstatus, "ABNORMAL") } else { //fmt.Printf("%s/%s:%s\t%s\t%s\t%s\n", repo, item, tag, onetag.Optime, onetag.Comment, tegStatus) ctag = append(ctag, repo+"/"+item+":"+tag) cupdatetime = append(cupdatetime, onetag.Optime) ccomment = append(ccomment, onetag.Comment) cstatus = append(cstatus, tagStatus) } utils.PrintFmt(ctag, cupdatetime, ccomment, cstatus) } }
func GetPulledRepoInfo(dpName, repoName string, offset int64, limit int) ([]ds.PulledItemInfo, error) { var pulledItemInfo ds.PulledItemInfo pulledItemInfos := make([]ds.PulledItemInfo, 0) sql := fmt.Sprintf(`SELECT DPID FROM DH_DP WHERE DPNAME = '%s' AND STATUS = 'A';`, dpName) row, err := g_ds.QueryRow(sql) if err != nil { l := log.Error(err) logq.LogPutqueue(l) return nil, err } var dpid int row.Scan(&dpid) sql = fmt.Sprintf(`SELECT DATAITEM ,ITEMDESC FROM DH_DP_RPDM_MAP WHERE DPID = %d AND REPOSITORY = '%s' AND PUBLISH = 'N' AND STATUS = 'A' ORDER BY RPDMID LIMIT %v OFFSET %v;`, dpid, repoName, limit, offset) rows, err := g_ds.QueryRows(sql) if err != nil { l := log.Error(err) logq.LogPutqueue(l) return nil, err } result := ds.Result{} pages := ds.ResultPages{} orderInfoSlice := []ds.OrderInfo{} pages.Results = &orderInfoSlice result.Data = &pages for rows.Next() { rows.Scan(&pulledItemInfo.ItemName, &pulledItemInfo.Location) path := "/api/subscriptions/pull/" + repoName + "/" + pulledItemInfo.ItemName resp, err := commToServerGetRsp("get", path, nil) if err != nil { l := log.Error(err) logq.LogPutqueue(l) return nil, err } defer resp.Body.Close() if resp.StatusCode == http.StatusUnauthorized { pulledItemInfo.SignTime = nil log.Debug("resp.StatusCode == http.StatusUnauthorized") } else if resp.StatusCode != http.StatusOK { err = errors.New("request subscriptions api failed.") l := log.Error(err) logq.LogPutqueue(l) return nil, err } else { respbody, err := ioutil.ReadAll(resp.Body) if err != nil { l := log.Error(err) logq.LogPutqueue(l) return nil, err } else { err = json.Unmarshal(respbody, &result) if err != nil { err = errors.New("unmarshal failed.") l := log.Error(err) logq.LogPutqueue(l) return nil, err } if len(orderInfoSlice) > 0 { //Already order by signtime desc. Get the first member. pulledItemInfo.SignTime = &orderInfoSlice[0].Signtime log.Debug("pulledItemInfo.SignTime:", pulledItemInfo.SignTime) } } } pulledItemInfos = append(pulledItemInfos, pulledItemInfo) } log.Debug(pulledItemInfos) return pulledItemInfos, err }
func newPubTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { log.Println(r.URL.Path, "(pub tag)") repo := ps.ByName("repo") item := ps.ByName("item") tag := ps.ByName("tag") log.Println("repo", repo, "item", item, "tag", tag) //log paras := ds.PubPara{} reqbody, err := ioutil.ReadAll(r.Body) if err != nil { log.Println("ioutil.ReadAll err:", err) JsonResult(w, http.StatusBadRequest, cmd.ErrorIOUtil, "Internal Error.", nil) return } err = json.Unmarshal(reqbody, ¶s) if err != nil { log.Println("json.Unmarshal err:", err) JsonResult(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "Internal Error.", nil) return } if !CheckLength(w, repo, MaxRepoLength) || !CheckLength(w, item, MaxItemLength) || !CheckLength(w, tag, MaxTagLength) { JsonResult(w, http.StatusOK, cmd.ErrorOutMaxLength, "repo or item or tag length out of max length.", nil) return } isExist, err := CheckItemExist(repo, item) if isExist == false { if paras.Datapool != "" && paras.ItemDesc != "" { isExist := CheckDataPoolExist(paras.Datapool) if isExist == false { JsonResult(w, http.StatusOK, cmd.ErrorDatapoolNotExits, "datapool not exist.", nil) return } url := DefaultServer + "/api/repositories/" + repo + "/" + item log.Println("daemon: connecting to ", url) //log req, err := http.NewRequest("GET", url, nil) if err != nil { log.Println("http.NewRequest err:", err) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } resp, err := http.DefaultClient.Do(req) if err != nil { log.Println("http.DefaultClient.Do err:", err) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } defer resp.Body.Close() respbody, err := ioutil.ReadAll(resp.Body) if err != nil { JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } result := ds.Result{} itemInfo := ds.ItemInfo{} result.Data = &itemInfo if resp.StatusCode == http.StatusOK { err = json.Unmarshal(respbody, &result) if err != nil { log.Println("json.Unmarshal err:", err) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } } else { log.Println(string(respbody)) err = json.Unmarshal(respbody, &result) if err != nil { log.Println("json.Unmarshal err:", err) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil) return } err = InsertItemToDb(repo, item, paras.Datapool, paras.ItemDesc, itemInfo.Optime) if err != nil { log.Println("InsertItemToDb err:", err) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } } else { JsonResult(w, http.StatusOK, cmd.ErrorItemNotExist, "item not been published.", nil) return } } //-------------------------------------------这是分割线----------------------------------------------------- dpconn, dptype, itemDesc, err := CheckTagAndGetDpPath(repo, item, tag) log.Println("CheckTagAndGetDpPath ret:", dpconn, dptype, itemDesc) //log if err != nil { log.Println("CheckTagAndGetDpPath err:", err) //rollbackInsertPubItem(w, repo, item) msg := fmt.Sprintf("Tag '%s' already exist.", tag) JsonResult(w, http.StatusOK, cmd.ErrorTagAlreadyExist, msg, nil) return } splits := strings.Split(paras.Detail, "/") fileName := splits[len(splits)-1] datapool, err := dpdriver.New(dptype) if err != nil { l := log.Error(err.Error()) logq.LogPutqueue(l) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } exist, size, err := datapool.CheckDataAndGetSize(dpconn, itemDesc, fileName) if err != nil && exist == false { l := log.Error(err.Error()) logq.LogPutqueue(l) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } if size > 0 { paras.Comment += SizeToStr(size) } body, e := json.Marshal(&struct { Comment string `json:"comment"` }{paras.Comment}) if e != nil { s := "Pub tag error while marshal struct" log.Println(s) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) //fmt.Println("Marshal err:", err) return } url := DefaultServer + "/api/repositories/" + repo + "/" + item + "/" + tag log.Println("daemon: connecting to ", url) //log req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) if len(loginAuthStr) > 0 { req.Header.Set("Authorization", loginAuthStr) } resp, err := http.DefaultClient.Do(req) if err != nil { log.Println("DefaultClient.Do err:", err) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } defer resp.Body.Close() //Get server result result := ds.Result{} respbody, _ := ioutil.ReadAll(resp.Body) log.Println(resp.StatusCode, string(respbody)) //log if resp.StatusCode == http.StatusOK { err = json.Unmarshal(respbody, &result) if err != nil { log.Println("Unmarshal err:", err) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } err = InsertPubTagToDb(repo, item, tag, fileName, paras.Comment) if err != nil { l := log.Error("Insert tag to db error.") logq.LogPutqueue(l) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil) g_DaemonRole = PUBLISHER } else if resp.StatusCode == http.StatusUnauthorized { JsonResult(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "no login.", nil) //rollbackInsertPubItem(w, repo, item) return } else { err = json.Unmarshal(respbody, &result) if err != nil { log.Println("Unmarshal err:", err) //rollbackInsertPubItem(w, repo, item) JsonResult(w, http.StatusBadRequest, cmd.InternalError, "Internal Error.", nil) return } log.Println(string(respbody)) //rollbackInsertPubItem(w, repo, item) JsonResult(w, resp.StatusCode, result.Code, result.Msg, nil) } return }
func HeartBeat() { getEp := false g_DaemonRole = GetDaemonRoleByPubRecord() for { if len(DaemonID) == 0 { log.Warn("No daemonid. You'd better start datahub with the parameter \"--token\".") return } heartbeatbody := Beatbody{Daemonid: DaemonID, Role: g_DaemonRole} if getEp == false && len(EntryPoint) == 0 { EntryPoint = getEntryPoint() getEp = true } if len(EntryPoint) != 0 { heartbeatbody.Entrypoint = append(heartbeatbody.Entrypoint, EntryPoint) } logQueue := logq.LogGetqueue() if len(logQueue) > 0 { heartbeatbody.Log = logQueue } errortags := checkErrortagsmap(&Errortagsmap) if len(errortags) != 0 { heartbeatbody.Errortag = errortags } heartbeatbody.CliEntrypoint = CLIEntrypoint + "?daemonauth=" + DaemonAuthrization jsondata, err := json.Marshal(heartbeatbody) if err != nil { l := log.Error(err) logq.LogPutqueue(l) } url := DefaultServerAPI + "/heartbeat" log.Trace("connecting to", url, string(jsondata)) req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsondata)) /* if len(loginAuthStr) > 0 { req.Header.Set("Authorization", loginAuthStr) } */ resp, err := http.DefaultClient.Do(req) if err != nil { l := log.Error(err) logq.LogPutqueue(l) time.Sleep(10 * time.Second) continue } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) log.Tracef("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body) result := ds.Result{} result.Data = &Event{} if err := json.Unmarshal(body, &result); err == nil { if result.Code == 0 { EntryPointStatus = "available" } else { EntryPointStatus = "not available" } } if result.Data != nil && result.Data.(*Event).Type == TYPE_ITEM_DEL { deleteItemsAccordingToHeartbeat(body) } time.Sleep(heartbeatTimeout) } }
func GetMessages() { log.Info("start GetMessages from messages server") var sleepInterval int var srtInterval string var e error url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0" for AutoPull == true { if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 { sleepInterval, e = strconv.Atoi(srtInterval) if e != nil { l := log.Error(e) logq.LogPutqueue(l) } } else { sleepInterval = 600 } time.Sleep(time.Duration(sleepInterval) * time.Second) log.Debug("connecting to", url) req, err := http.NewRequest("GET", url, nil) if len(loginAuthStr) > 0 { req.Header.Set("Authorization", loginAuthStr) } resp, err := http.DefaultClient.Do(req) if err != nil { l := log.Error(err) logq.LogPutqueue(l) continue } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) if resp.StatusCode == http.StatusOK { log.Debugf("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body) result := ds.Result{} Pages := ds.ResultPages{} MessagesSlice := []Messages{} Pages.Results = &MessagesSlice result.Data = &Pages if err := json.Unmarshal(body, &result); err == nil { if result.Code == 0 { log.Debug(result) for _, v := range MessagesSlice { if v.Type == "item_event" && v.Data.Event == TAGADDED { InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD) } } } else { l := log.Error("Get Messages errror:", result.Code) logq.LogPutqueue(l) } } else { log.Error(err) } } else if resp.StatusCode == http.StatusUnauthorized { log.Debug("not login", http.StatusUnauthorized) urllogin := DefaultServerAPI + "/" reql, err := http.NewRequest("GET", urllogin, nil) if len(loginBasicAuthStr) > 0 { reql.Header.Set("Authorization", loginBasicAuthStr) log.Info("user name:", gstrUsername) } else { log.Warn("not login") continue } respl, err := http.DefaultClient.Do(reql) if err != nil { log.Error(err) continue } defer respl.Body.Close() result := &ds.Result{} log.Println("login return", respl.StatusCode) if respl.StatusCode == 200 { body, _ := ioutil.ReadAll(respl.Body) log.Println(string(body)) result.Data = &tk{} if err = json.Unmarshal(body, result); err != nil { log.Error(err) log.Println(respl.StatusCode, string(body)) continue } else { loginAuthStr = "Token " + result.Data.(*tk).Token loginLogged = true log.Println(loginAuthStr) } } } } }