func deleteItemsAccordingToHeartbeat(body []byte) {
log.Debug("deleteItemsAccordingToHeartbeat() BEGIN:", string(body))
result := ds.Result{}
itemEvent := &Event{}
result.Data = itemEvent
itemsdelete := []ItemDel{}
itemEvent.Data = &itemsdelete
if err := json.Unmarshal(body, &result); err == nil {
log.Debug("items delete:", itemsdelete)
for _, v := range itemsdelete {
log.Debugf("delete item according to heartbeat: %v/%v\n", v.Repname, v.Itemname)
err := delTagsForDelItem(v.Repname, v.Itemname)
if err != nil {
log.Error(err)
return
}
err = delItem(v.Repname, v.Itemname)
if err != nil {
log.Error(err)
return
}
log.Infof("Delete data item %v/%v according to heartbeat successfully.\n", v.Repname, v.Itemname)
}
} else {
log.Warn("Unmarshal error:", err)
}
}
func CheckLength(w http.ResponseWriter, data string, length int) bool {
if len(data) > length {
m := fmt.Sprintf("length of %s can't over %d bytes", data, length)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorOverLength, m)
log.Warn(m)
return false
}
return true
}
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
logq.LogPutqueue(l)
r.ParseForm()
sRepoName := ps.ByName("repo")
sDataItem := ps.ByName("dataitem")
sTag := ps.ByName("tag")
log.Info(sRepoName, sDataItem, sTag)
jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
var irpdmid, idpid int
var stagdetail, itemdesc, dpconn, dpname, dptype string
msg := &ds.MsgResp{}
msg.Msg = "OK."
irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
if len(itemdesc) == 0 {
itemdesc = sRepoName + "_" + sDataItem
}
log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)
stagdetail = GetTagDetail(irpdmid, sTag)
log.Debug("tagdetail", stagdetail)
if len(stagdetail) == 0 {
l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
logq.LogPutqueue(l)
http.Error(rw, sTag+" not found", http.StatusNotFound)
return
}
dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)
datapool, err := dpdriver.New(dptype)
if err != nil {
WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
return
}
filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
log.Println("filename:", filepathname)
if exists := isFileExists(filepathname); !exists {
l := log.Error(filepathname, "not found")
logq.LogPutqueue(l)
putToJobQueue(jobtag, filepathname, "N/A", -1)
msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
resp, _ := json.Marshal(msg)
respStr := string(resp)
rw.WriteHeader(http.StatusNotFound)
fmt.Fprintln(rw, respStr)
return
}
tokenValid := false
retmsg := ""
token := r.Form.Get("token")
username := r.Form.Get("username")
log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
if len(token) > 0 && len(username) > 0 {
log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
logq.LogPutqueue(l)
}
log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
//rw.Header().Set("Source-FileName", stagdetail)
bmd5, err := ComputeMd5(filepathname)
strmd5 := fmt.Sprintf("%x", bmd5)
if err != nil {
log.Error(filepathname, err, bmd5, strmd5)
} else {
rw.Header().Set("X-Source-MD5", strmd5)
}
rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))
l = log.Info("transfering", filepathname, bmd5, strmd5)
logq.LogPutqueue(l)
jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
http.ServeFile(rw, r, filepathname)
updateJobQueue(jobid, "transfered", 0)
return
}
func HeartBeat() {
getEp := false
g_DaemonRole = GetDaemonRoleByPubRecord()
for {
if len(DaemonID) == 0 {
log.Warn("No daemonid. You'd better start datahub with the parameter \"--token\".")
return
}
heartbeatbody := Beatbody{Daemonid: DaemonID, Role: g_DaemonRole}
if getEp == false && len(EntryPoint) == 0 {
EntryPoint = getEntryPoint()
getEp = true
}
if len(EntryPoint) != 0 {
heartbeatbody.Entrypoint = append(heartbeatbody.Entrypoint, EntryPoint)
}
logQueue := logq.LogGetqueue()
if len(logQueue) > 0 {
heartbeatbody.Log = logQueue
}
errortags := checkErrortagsmap(&Errortagsmap)
if len(errortags) != 0 {
heartbeatbody.Errortag = errortags
}
heartbeatbody.CliEntrypoint = CLIEntrypoint + "?daemonauth=" + DaemonAuthrization
jsondata, err := json.Marshal(heartbeatbody)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
}
url := DefaultServerAPI + "/heartbeat"
log.Trace("connecting to", url, string(jsondata))
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsondata))
/*
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
*/
resp, err := http.DefaultClient.Do(req)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
time.Sleep(10 * time.Second)
continue
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
log.Tracef("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body)
result := ds.Result{}
result.Data = &Event{}
if err := json.Unmarshal(body, &result); err == nil {
if result.Code == 0 {
EntryPointStatus = "available"
} else {
EntryPointStatus = "not available"
}
}
if result.Data != nil && result.Data.(*Event).Type == TYPE_ITEM_DEL {
deleteItemsAccordingToHeartbeat(body)
}
time.Sleep(heartbeatTimeout)
}
}
func GetMessages() {
log.Info("start GetMessages from messages server")
var sleepInterval int
var srtInterval string
var e error
url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0"
for AutoPull == true {
if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 {
sleepInterval, e = strconv.Atoi(srtInterval)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
} else {
sleepInterval = 600
}
time.Sleep(time.Duration(sleepInterval) * time.Second)
log.Debug("connecting to", url)
req, err := http.NewRequest("GET", url, nil)
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
continue
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode == http.StatusOK {
log.Debugf("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body)
result := ds.Result{}
Pages := ds.ResultPages{}
MessagesSlice := []Messages{}
Pages.Results = &MessagesSlice
result.Data = &Pages
if err := json.Unmarshal(body, &result); err == nil {
if result.Code == 0 {
log.Debug(result)
for _, v := range MessagesSlice {
if v.Type == "item_event" && v.Data.Event == TAGADDED {
InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD)
}
}
} else {
l := log.Error("Get Messages errror:", result.Code)
logq.LogPutqueue(l)
}
} else {
log.Error(err)
}
} else if resp.StatusCode == http.StatusUnauthorized {
log.Debug("not login", http.StatusUnauthorized)
urllogin := DefaultServerAPI + "/"
reql, err := http.NewRequest("GET", urllogin, nil)
if len(loginBasicAuthStr) > 0 {
reql.Header.Set("Authorization", loginBasicAuthStr)
log.Info("user name:", gstrUsername)
} else {
log.Warn("not login")
continue
}
respl, err := http.DefaultClient.Do(reql)
if err != nil {
log.Error(err)
continue
}
defer respl.Body.Close()
result := &ds.Result{}
log.Println("login return", respl.StatusCode)
if respl.StatusCode == 200 {
body, _ := ioutil.ReadAll(respl.Body)
log.Println(string(body))
result.Data = &tk{}
if err = json.Unmarshal(body, result); err != nil {
log.Error(err)
log.Println(respl.StatusCode, string(body))
continue
} else {
loginAuthStr = "Token " + result.Data.(*tk).Token
loginLogged = true
log.Println(loginAuthStr)
}
}
}
}
}