コード例 #1
0
ファイル: heartbeat.go プロジェクト: asiainfoLDP/datahub
func deleteItemsAccordingToHeartbeat(body []byte) {
	log.Debug("deleteItemsAccordingToHeartbeat() BEGIN:", string(body))
	result := ds.Result{}
	itemEvent := &Event{}
	result.Data = itemEvent
	itemsdelete := []ItemDel{}
	itemEvent.Data = &itemsdelete

	if err := json.Unmarshal(body, &result); err == nil {
		log.Debug("items delete:", itemsdelete)
		for _, v := range itemsdelete {
			log.Debugf("delete item according to heartbeat: %v/%v\n", v.Repname, v.Itemname)
			err := delTagsForDelItem(v.Repname, v.Itemname)
			if err != nil {
				log.Error(err)
				return
			}

			err = delItem(v.Repname, v.Itemname)
			if err != nil {
				log.Error(err)
				return
			}
			log.Infof("Delete data item %v/%v according to heartbeat successfully.\n", v.Repname, v.Itemname)
		}
	} else {
		log.Warn("Unmarshal error:", err)
	}
}
コード例 #2
0
ファイル: pub_handler.go プロジェクト: asiainfoLDP/datahub
func CheckLength(w http.ResponseWriter, data string, length int) bool {
	if len(data) > length {
		m := fmt.Sprintf("length of %s can't over %d bytes", data, length)
		HttpNoData(w, http.StatusBadRequest, cmd.ErrorOverLength, m)
		log.Warn(m)
		return false
	}

	return true
}
コード例 #3
0
ファイル: p2pserver.go プロジェクト: asiainfoLDP/datahub
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
	logq.LogPutqueue(l)

	r.ParseForm()
	sRepoName := ps.ByName("repo")
	sDataItem := ps.ByName("dataitem")
	sTag := ps.ByName("tag")

	log.Info(sRepoName, sDataItem, sTag)
	jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
	var irpdmid, idpid int
	var stagdetail, itemdesc, dpconn, dpname, dptype string
	msg := &ds.MsgResp{}
	msg.Msg = "OK."

	irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
	if len(itemdesc) == 0 {
		itemdesc = sRepoName + "_" + sDataItem
	}
	log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)

	stagdetail = GetTagDetail(irpdmid, sTag)
	log.Debug("tagdetail", stagdetail)
	if len(stagdetail) == 0 {
		l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
		logq.LogPutqueue(l)
		http.Error(rw, sTag+" not found", http.StatusNotFound)
		return
	}

	dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
	log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)

	datapool, err := dpdriver.New(dptype)
	if err != nil {
		WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
		return
	}

	filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
	//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
	log.Println("filename:", filepathname)
	if exists := isFileExists(filepathname); !exists {
		l := log.Error(filepathname, "not found")
		logq.LogPutqueue(l)
		putToJobQueue(jobtag, filepathname, "N/A", -1)
		msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
		resp, _ := json.Marshal(msg)
		respStr := string(resp)
		rw.WriteHeader(http.StatusNotFound)
		fmt.Fprintln(rw, respStr)
		return
	}

	tokenValid := false
	retmsg := ""
	token := r.Form.Get("token")
	username := r.Form.Get("username")

	log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
	if len(token) > 0 && len(username) > 0 {
		log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
			"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
		logq.LogPutqueue(l)
	}
	log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
	//rw.Header().Set("Source-FileName", stagdetail)
	bmd5, err := ComputeMd5(filepathname)
	strmd5 := fmt.Sprintf("%x", bmd5)
	if err != nil {
		log.Error(filepathname, err, bmd5, strmd5)
	} else {
		rw.Header().Set("X-Source-MD5", strmd5)
	}
	rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))

	l = log.Info("transfering", filepathname, bmd5, strmd5)
	logq.LogPutqueue(l)

	jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
	http.ServeFile(rw, r, filepathname)
	updateJobQueue(jobid, "transfered", 0)

	return
}
コード例 #4
0
ファイル: heartbeat.go プロジェクト: asiainfoLDP/datahub
func HeartBeat() {
	getEp := false

	g_DaemonRole = GetDaemonRoleByPubRecord()

	for {
		if len(DaemonID) == 0 {
			log.Warn("No daemonid. You'd better start datahub with the parameter \"--token\".")
			return
		}

		heartbeatbody := Beatbody{Daemonid: DaemonID, Role: g_DaemonRole}
		if getEp == false && len(EntryPoint) == 0 {
			EntryPoint = getEntryPoint()
			getEp = true
		}
		if len(EntryPoint) != 0 {
			heartbeatbody.Entrypoint = append(heartbeatbody.Entrypoint, EntryPoint)
		}

		logQueue := logq.LogGetqueue()
		if len(logQueue) > 0 {
			heartbeatbody.Log = logQueue
		}

		errortags := checkErrortagsmap(&Errortagsmap)
		if len(errortags) != 0 {
			heartbeatbody.Errortag = errortags
		}

		heartbeatbody.CliEntrypoint = CLIEntrypoint + "?daemonauth=" + DaemonAuthrization

		jsondata, err := json.Marshal(heartbeatbody)
		if err != nil {
			l := log.Error(err)
			logq.LogPutqueue(l)
		}
		url := DefaultServerAPI + "/heartbeat"
		log.Trace("connecting to", url, string(jsondata))
		req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsondata))
		/*
			if len(loginAuthStr) > 0 {
				req.Header.Set("Authorization", loginAuthStr)
			}
		*/
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			l := log.Error(err)
			logq.LogPutqueue(l)
			time.Sleep(10 * time.Second)
			continue
		}

		defer resp.Body.Close()

		body, _ := ioutil.ReadAll(resp.Body)
		log.Tracef("HeartBeat http statuscode:%v,  http body:%s", resp.StatusCode, body)

		result := ds.Result{}
		result.Data = &Event{}
		if err := json.Unmarshal(body, &result); err == nil {
			if result.Code == 0 {
				EntryPointStatus = "available"
			} else {
				EntryPointStatus = "not available"
			}
		}
		if result.Data != nil && result.Data.(*Event).Type == TYPE_ITEM_DEL {
			deleteItemsAccordingToHeartbeat(body)
		}

		time.Sleep(heartbeatTimeout)
	}
}
コード例 #5
0
ファイル: heartbeat.go プロジェクト: asiainfoLDP/datahub
func GetMessages() {
	log.Info("start GetMessages from messages server")
	var sleepInterval int
	var srtInterval string
	var e error
	url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0"
	for AutoPull == true {

		if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 {
			sleepInterval, e = strconv.Atoi(srtInterval)
			if e != nil {
				l := log.Error(e)
				logq.LogPutqueue(l)
			}
		} else {
			sleepInterval = 600
		}

		time.Sleep(time.Duration(sleepInterval) * time.Second)
		log.Debug("connecting to", url)
		req, err := http.NewRequest("GET", url, nil)

		if len(loginAuthStr) > 0 {
			req.Header.Set("Authorization", loginAuthStr)
		}

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			l := log.Error(err)
			logq.LogPutqueue(l)

			continue
		}
		defer resp.Body.Close()

		body, _ := ioutil.ReadAll(resp.Body)

		if resp.StatusCode == http.StatusOK {
			log.Debugf("HeartBeat http statuscode:%v,  http body:%s", resp.StatusCode, body)

			result := ds.Result{}
			Pages := ds.ResultPages{}
			MessagesSlice := []Messages{}
			Pages.Results = &MessagesSlice
			result.Data = &Pages

			if err := json.Unmarshal(body, &result); err == nil {
				if result.Code == 0 {
					log.Debug(result)
					for _, v := range MessagesSlice {
						if v.Type == "item_event" && v.Data.Event == TAGADDED {
							InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD)
						}
					}
				} else {
					l := log.Error("Get Messages errror:", result.Code)
					logq.LogPutqueue(l)
				}
			} else {
				log.Error(err)
			}

		} else if resp.StatusCode == http.StatusUnauthorized {
			log.Debug("not login", http.StatusUnauthorized)
			urllogin := DefaultServerAPI + "/"
			reql, err := http.NewRequest("GET", urllogin, nil)
			if len(loginBasicAuthStr) > 0 {
				reql.Header.Set("Authorization", loginBasicAuthStr)
				log.Info("user name:", gstrUsername)
			} else {
				log.Warn("not login")
				continue
			}

			respl, err := http.DefaultClient.Do(reql)
			if err != nil {
				log.Error(err)
				continue
			}
			defer respl.Body.Close()

			result := &ds.Result{}
			log.Println("login return", respl.StatusCode)
			if respl.StatusCode == 200 {
				body, _ := ioutil.ReadAll(respl.Body)
				log.Println(string(body))

				result.Data = &tk{}
				if err = json.Unmarshal(body, result); err != nil {
					log.Error(err)
					log.Println(respl.StatusCode, string(body))
					continue
				} else {
					loginAuthStr = "Token " + result.Data.(*tk).Token
					loginLogged = true
					log.Println(loginAuthStr)
				}
			}
		}
	}
}