func startP2PServer() {
p2pListener, err := net.Listen("tcp", ":35800")
if err != nil {
log.Fatal(err)
}
p2psl, err = tcpNew(p2pListener)
if err != nil {
log.Fatal(err)
}
P2pRouter := httprouter.New()
P2pRouter.GET("/", sayhello)
P2pRouter.GET("/pull/:repo/:dataitem/:tag", p2p_pull)
P2pRouter.GET("/health", p2pHealthyCheckHandler)
p2pserver := http.Server{Handler: P2pRouter}
wg.Add(1)
defer wg.Done()
log.Info("p2p server start")
p2pserver.Serve(p2psl)
log.Info("p2p server stop")
}
func commToServer(method, path string, buffer []byte, w http.ResponseWriter) (body []byte, err error) {
//Trace()
s := log.Info("daemon: connecting to", DefaultServer+path)
logq.LogPutqueue(s)
req, err := http.NewRequest(strings.ToUpper(method), DefaultServer+path, bytes.NewBuffer(buffer))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
//req.Header.Set("User", "admin")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Error(err)
d := ds.Result{Code: cmd.ErrorServiceUnavailable, Msg: err.Error()}
body, e := json.Marshal(d)
if e != nil {
log.Error(e)
return body, e
}
w.WriteHeader(http.StatusServiceUnavailable)
w.Write(body)
return body, err
}
defer resp.Body.Close()
w.WriteHeader(resp.StatusCode)
body, err = ioutil.ReadAll(resp.Body)
w.Write(body)
log.Info(resp.StatusCode, string(body))
return
}
func delTagsForDelItem(reponame, itemname string) error {
log.Println("Begin to remove tags for remove item from db")
sqlrpdmid := fmt.Sprintf(`SELECT RPDMID FROM DH_DP_RPDM_MAP WHERE REPOSITORY='%s' AND DATAITEM='%s' AND STATUS='A';`, reponame, itemname)
row, err := g_ds.QueryRow(sqlrpdmid)
if err != nil {
l := log.Error("select rpdmid from DH_DP_RPDM_MAP error:", err)
logq.LogPutqueue(l)
return err
}
var rpdmId int
row.Scan(&rpdmId)
if rpdmId == 0 {
log.Debug(reponame, itemname, "not exist.")
return nil
}
sqldeltag := fmt.Sprintf(`UPDATE DH_RPDM_TAG_MAP SET STATUS='N' WHERE RPDMID=%d`, rpdmId)
_, err = g_ds.Update(sqldeltag)
log.Info("sqldeltag", sqldeltag)
if err != nil {
l := log.Error("delete tag error:", err)
logq.LogPutqueue(l)
return err
}
return nil
}
func (s3 *s3driver) CheckDataAndGetSize(dpconn, itemlocation, fileName string) (exist bool, size int64, err error) {
bucket := getAwsInfoFromDpconn(dpconn)
destFullPathFileName := bucket + "/" + itemlocation + "/" + fileName
log.Info(destFullPathFileName)
AWS_REGION = Env("AWS_REGION", false)
svc := s3aws.New(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
result, err := svc.ListObjects(&s3aws.ListObjectsInput{Bucket: aws.String(bucket),
Prefix: aws.String(itemlocation + "/" + fileName)})
if err != nil {
log.Error("Failed to list objects", err)
return exist, size, err
}
exist = false
for _, v := range result.Contents {
log.Infof("Tag:%s, key:%s, size:%v\n", aws.StringValue(v.ETag), aws.StringValue(v.Key), aws.Int64Value(v.Size))
if aws.StringValue(v.Key) == fileName {
size = aws.Int64Value(v.Size)
exist = true
}
}
return
}
func (hdfs *hdfsdriver) StoreFile(status, filename, dpconn, dp, itemlocation, destfile string) string {
log.Infof("Begin to upload %v to %v\n", filename, dp)
client, err := getClient(dpconn)
if err != nil {
log.Error("Failed to get a client", err)
status = "put to hdfs err"
return status
}
defer client.Close()
err = client.MkdirAll("/"+itemlocation, 1777)
if err != nil {
log.Error("Failed to mkdirall in hdfs", err)
status = "put to hdfs err"
return status
}
hdfsfile := "/" + itemlocation + "/" + destfile
err = client.CopyToRemote(filename, hdfsfile)
if err != nil {
log.Error("Failed to CopyToRemote", err)
status = "put to hdfs err"
return status
}
status = "put to hdfs ok"
log.Info("Successfully uploaded to", itemlocation, "in hdfs")
return status
}
func (hdfs *hdfsdriver) CheckDataAndGetSize(dpconn, itemlocation, fileName string) (exist bool, size int64, err error) {
destFullPathFileName := "/" + itemlocation + "/" + fileName
log.Info(destFullPathFileName)
exist = false
client, err := getClient(dpconn)
if err != nil {
log.Error("Failed to get a client", err)
return
}
defer client.Close()
fileinfo, _ := client.Stat(destFullPathFileName)
if fileinfo != nil {
exist = true
cs, _ := client.GetContentSummary(destFullPathFileName)
size = cs.Size()
} else {
err = errors.New("文件不存在")
return
}
return
}
func UpdateStatMsgTagadded(ID, Status int) (err error) {
log.Info("update MSG_TAGADDED status")
sql := fmt.Sprintf(`UPDATE MSG_TAGADDED SET STATUS=%d
WHERE ID=%d;`, Status, ID)
_, err = g_ds.Update(sql)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return
}
return
}
func getDaemonid() (id string) {
log.Println("TODO get daemonid from db.")
s := `SELECT DAEMONID FROM DH_DAEMON;`
row, e := g_ds.QueryRow(s)
if e != nil {
l := log.Error(s, "error.", e)
logq.LogPutqueue(l)
return
}
row.Scan(&id)
log.Info("daemon id is", id)
return id
}
func testserver() {
var tsl = new(StoppabletcpListener)
Listener, err := net.Listen("tcp", ":35888")
if err != nil {
log.Fatal(err)
}
tsl, err = tcpNew(Listener)
if err != nil {
log.Fatal(err)
}
tRouter := httprouter.New()
tRouter.GET("/", sayhello)
http.Handle("/", tRouter)
server := http.Server{Handler: tRouter}
log.Info("p2p server start")
server.Serve(tsl)
log.Info("p2p server stop")
}
func epPostHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
reqBody, _ := ioutil.ReadAll(r.Body)
ep := cmd.FormatEp{}
if err := json.Unmarshal(reqBody, &ep); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
EntryPoint = ep.Ep
saveEntryPoint(EntryPoint)
msg := ds.MsgResp{Msg: "OK. your entrypoint is: " + EntryPoint + "\n" + "The entrypoint is setted successfully. You can check whether it is available in one minute."}
log.Info(msg.Msg)
resp, _ := json.Marshal(&msg)
fmt.Fprintln(w, string(resp))
return
}
func commToServerGetRsp(method, path string, buffer []byte) (resp *http.Response, err error) {
s := log.Info("daemon: connecting to", DefaultServer+path)
logq.LogPutqueue(s)
req, err := http.NewRequest(strings.ToUpper(method), DefaultServer+path, bytes.NewBuffer(buffer))
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
log.Error(err)
return resp, err
}
return resp, nil
}
func PullTagAutomatic() {
for {
time.Sleep(30 * time.Second)
//log.Debug("AutomaticPullList.Len()", AutomaticPullList.Len())
var Tags map[int]string
for e := AutomaticPullList.Front(); e != nil; e = e.Next() {
v := e.Value.(ds.DsPull)
log.Info("PullTagAutomatic begin", v.Repository, v.Dataitem)
Tags = GetTagFromMsgTagadded(v.Repository, v.Dataitem, NOTREAD)
log.Println("Tags ", Tags)
go PullItemAutomatic(Tags, v)
}
}
}
func (s3 *s3driver) StoreFile(status, filename, dpconn, dp, itemlocation, destfile string) string {
bucket := getAwsInfoFromDpconn(dpconn)
//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
AWS_REGION = Env("AWS_REGION", false)
file, err := os.Open(filename)
if err != nil {
l := log.Error("Failed to open file", err)
logq.LogPutqueue(l)
status = "put to s3 err"
return status
}
log.Infof("Begin to upload %v to %v\n", filename, dp)
// Not required, but you could zip the file before uploading it
// using io.Pipe read/writer to stream gzip'd file contents.
reader, writer := io.Pipe()
go func() {
gw := gzip.NewWriter(writer)
io.Copy(gw, file)
file.Close()
gw.Close()
writer.Close()
//updateJobQueueStatus(jobid, "puttos3ok")
}()
uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
//uploader := s3manager.NewUploader(session.New(aws.NewConfig()))
result, err := uploader.Upload(&s3manager.UploadInput{
Body: reader,
Bucket: aws.String(bucket),
Key: aws.String( /*dp + "/" + */ itemlocation + "/" + destfile + ".gz"),
})
if err != nil {
log.Error("Failed to upload", err)
status = "put to s3 err"
return status
}
status = "put to s3 ok"
log.Info("Successfully uploaded to", result.Location)
return status
}
func CheckHealthClock() {
log.Debug("--------->BEGIN")
checkHealth(&Errortagsmap)
timer := time.NewTicker(10 * time.Minute)
for {
select {
case <-timer.C:
now := time.Now()
if now.Hour()%6 == 0 {
log.Info("Time:", now)
checkHealth(&Errortagsmap)
}
}
}
log.Debug("---------->END")
}
func AutomaticPullRmqueue(p ds.DsPull) (exist bool) {
exist = false
pullmu.Lock()
defer pullmu.Unlock()
var next *list.Element
for e := AutomaticPullList.Front(); e != nil; e = next {
v := e.Value.(ds.DsPull)
if v.Repository == p.Repository && v.Dataitem == p.Dataitem {
exist = true
AutomaticPullList.Remove(e)
log.Info(v, "removed from the queue.")
break
} else {
next = e.Next()
}
}
return
}
func (s3 *s3driver) GetFileTobeSend(dpconn, dpname, itemlocation, tagdetail string) (filepathname string) {
bucket := getAwsInfoFromDpconn(dpconn)
e := os.MkdirAll(gDpPath+"/"+bucket+"/"+itemlocation, 0777)
if e != nil {
log.Error(e)
return
}
filepathname = gDpPath + "/" + bucket + "/" + itemlocation + "/" + tagdetail
if true == isFileExists(filepathname) {
return
}
//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
AWS_REGION = Env("AWS_REGION", false)
file, err := os.Create(filepathname)
if err != nil {
log.Error("Failed to create file", err)
return ""
}
defer file.Close()
downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
numBytes, err := downloader.Download(file,
&s3aws.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String( /*dpname + "/" + */ itemlocation + "/" + tagdetail),
})
if err != nil {
log.Info("Failed to download file.", err)
os.Remove(filepathname)
return
}
log.Println("Downloaded file", file.Name(), numBytes, "bytes")
return
}
func checkHealth(errorTagsMap *map[string]string) {
localfiles := make([]string, 0)
alllocalfiles := make([]string, 0)
localfilepath := GetLocalfilePath()
for _, path := range localfilepath {
localfiles = ScanLocalFile(path)
for _, localfile := range localfiles {
alllocalfiles = append(alllocalfiles, localfile)
}
}
log.Info(alllocalfiles)
var tagDetails map[string]string
tagDetails = make(map[string]string)
err := GetAllTagDetails(&tagDetails)
if err != nil {
log.Error(err)
}
var i int
for file, tag := range tagDetails {
for i = 0; i < len(alllocalfiles); i++ {
//log.Info("--------->tag:", tag)
//log.Info("--------->tagfile:",file)
//log.Info("--------->localfile:",localfiles[i])
if file == alllocalfiles[i] {
break
}
}
if i >= len(alllocalfiles) {
(*errorTagsMap)[file] = tag
}
}
//for errortagfile, errortag := range *errorTagsMap {
// log.Info("------->errortag:", errortag, "-------->", errortagfile)
//}
}
func judgeRepoOrItemExist(repository, dataitem string) (exist bool, msg string, err error) {
path := "/api/repositories/" + repository + "/" + dataitem
exist = false
resp, err := commToServerGetRsp("get", path, nil)
if err != nil {
log.Error(err)
//HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
err = errors.New("unkown error")
return
}
result := ds.Response{}
respbody, _ := ioutil.ReadAll(resp.Body)
unmarshalerr := json.Unmarshal(respbody, &result)
if unmarshalerr != nil {
log.Error(unmarshalerr)
//HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
return
}
log.Info(string(respbody))
if resp.StatusCode == http.StatusBadRequest && result.Code == cmd.ServerErrResultCode1009 {
//HttpNoData(w, http.StatusBadRequest, cmd.RepoOrItemNotExist, result.Msg)
msg = result.Msg
return
} else if resp.StatusCode == http.StatusOK && result.Code == cmd.ServerErrResultCodeOk {
exist = true
msg = result.Msg
return
}
return
}
func judgeTagExist(repository, dataitem, tag string) (exist bool, msg string, err error) {
path := "/api/repositories/" + repository + "/" + dataitem + "/" + tag
exist = false
resp, err := commToServerGetRsp("get", path, nil)
defer resp.Body.Close()
if err != nil {
log.Error(err)
return
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
err = errors.New("unkown error")
return
}
result := ds.Response{}
respbody, _ := ioutil.ReadAll(resp.Body)
unmarshalerr := json.Unmarshal(respbody, &result)
if unmarshalerr != nil {
log.Error(unmarshalerr)
return
}
log.Info(string(respbody))
if resp.StatusCode == http.StatusBadRequest && result.Code == cmd.ServerErrResultCode1009 {
msg = result.Msg
return
} else if resp.StatusCode == http.StatusOK && result.Code == cmd.ServerErrResultCodeOk {
exist = true
msg = result.Msg
return
}
return
}
func repoDelTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
if len(loginAuthStr) == 0 {
HttpNoData(w, http.StatusUnauthorized, cmd.ErrorServiceUnavailable, " ")
return
}
repository := ps.ByName("repo")
dataitem := ps.ByName("item")
tag := ps.ByName("tag")
reqBody, _ := ioutil.ReadAll(r.Body)
if strings.Contains(tag, "*") {
tagsname, err := getBatchDelTagsName(repository, dataitem, tag)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag.")
return
}
if len(tagsname) == 0 {
log.Println("没有匹配的tag")
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "No match tag.")
return
}
successflag := true
for _, tagname := range tagsname {
if successflag {
_, err := delTag(repository, dataitem, tagname)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag")
return
}
path := "/api/repositories/" + repository + "/" + dataitem + "/" + tagname
resp, err := commToServerGetRsp("delete", path, reqBody)
if err != nil {
log.Error(err)
HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer error")
return
}
defer resp.Body.Close()
result := ds.Response{}
respbody, err := ioutil.ReadAll(resp.Body)
unmarshalerr := json.Unmarshal(respbody, &result)
if unmarshalerr != nil {
log.Error(unmarshalerr)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
return
}
if resp.StatusCode == http.StatusOK && result.Code == 0 {
continue
} else if resp.StatusCode == http.StatusOK && result.Code != 0 {
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
successflag = false
break
} else {
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
successflag = false
break
}
}
}
if successflag {
log.Info("批量删除tag成功")
HttpNoData(w, http.StatusOK, cmd.ResultOK, "ok.")
}
} else {
tagid, err := delTag(repository, dataitem, tag)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag")
return
}
reqBody, _ := ioutil.ReadAll(r.Body)
path := "/api/repositories/" + repository + "/" + dataitem + "/" + tag
resp, err := commToServerGetRsp("delete", path, reqBody)
if err != nil {
log.Error(err)
HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer error")
return
}
defer resp.Body.Close()
result := ds.Response{}
respbody, err := ioutil.ReadAll(resp.Body)
unmarshalerr := json.Unmarshal(respbody, &result)
if unmarshalerr != nil {
log.Error(unmarshalerr)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
return
}
if resp.StatusCode == http.StatusOK && result.Code == 0 {
HttpNoData(w, http.StatusOK, cmd.ResultOK, result.Msg)
log.Info("Msg :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
} else if resp.StatusCode == http.StatusOK && result.Code != 0 {
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
rollbackDelTag(tagid)
} else {
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
}
}
}
func GetMessages() {
log.Info("start GetMessages from messages server")
var sleepInterval int
var srtInterval string
var e error
url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0"
for AutoPull == true {
if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 {
sleepInterval, e = strconv.Atoi(srtInterval)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
} else {
sleepInterval = 600
}
time.Sleep(time.Duration(sleepInterval) * time.Second)
log.Debug("connecting to", url)
req, err := http.NewRequest("GET", url, nil)
if len(loginAuthStr) > 0 {
req.Header.Set("Authorization", loginAuthStr)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
continue
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode == http.StatusOK {
log.Debugf("HeartBeat http statuscode:%v, http body:%s", resp.StatusCode, body)
result := ds.Result{}
Pages := ds.ResultPages{}
MessagesSlice := []Messages{}
Pages.Results = &MessagesSlice
result.Data = &Pages
if err := json.Unmarshal(body, &result); err == nil {
if result.Code == 0 {
log.Debug(result)
for _, v := range MessagesSlice {
if v.Type == "item_event" && v.Data.Event == TAGADDED {
InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD)
}
}
} else {
l := log.Error("Get Messages errror:", result.Code)
logq.LogPutqueue(l)
}
} else {
log.Error(err)
}
} else if resp.StatusCode == http.StatusUnauthorized {
log.Debug("not login", http.StatusUnauthorized)
urllogin := DefaultServerAPI + "/"
reql, err := http.NewRequest("GET", urllogin, nil)
if len(loginBasicAuthStr) > 0 {
reql.Header.Set("Authorization", loginBasicAuthStr)
log.Info("user name:", gstrUsername)
} else {
log.Warn("not login")
continue
}
respl, err := http.DefaultClient.Do(reql)
if err != nil {
log.Error(err)
continue
}
defer respl.Body.Close()
result := &ds.Result{}
log.Println("login return", respl.StatusCode)
if respl.StatusCode == 200 {
body, _ := ioutil.ReadAll(respl.Body)
log.Println(string(body))
result.Data = &tk{}
if err = json.Unmarshal(body, result); err != nil {
log.Error(err)
log.Println(respl.StatusCode, string(body))
continue
} else {
loginAuthStr = "Token " + result.Data.(*tk).Token
loginLogged = true
log.Println(loginAuthStr)
}
}
}
}
}
func pubItemHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path, "(pub dataitem)")
repo := ps.ByName("repo")
item := ps.ByName("item")
if CheckLength(w, repo, MaxRepoLength) == false {
return
}
if CheckLength(w, item, MaxItemLength) == false {
return
}
if len(loginAuthStr) == 0 {
HttpNoData(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "Unlogin")
return
}
pub := ds.PubPara{}
pub.ItemDesc = strings.Trim(pub.ItemDesc, "/")
if strings.Contains(pub.ItemDesc, "/") == true {
log.Println("The path of item can't contain '/'.", pub.ItemDesc)
w.Write([]byte(`{"msg":"The path of item can't contain '/'."}`))
return
}
if CheckLength(w, pub.Comment, MaxCommentLength) == false {
return
}
reqBody, _ := ioutil.ReadAll(r.Body)
if err := json.Unmarshal(reqBody, &pub); err != nil {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "pub dataitem error while unmarshal reqBody")
return
}
if CheckDataPoolExist(pub.Datapool) == false {
HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal,
fmt.Sprintf("Datapool %s not found", pub.Datapool))
return
}
//ToDO check item dir exist
priceplans := []PricePlan{}
meta, sample, priceplans := GetMetaAndSampleAndPricePlan(pub.Datapool, pub.ItemDesc)
icpub := ic{AccessType: pub.Accesstype,
Comment: pub.Comment,
Meta: meta,
Sample: sample,
Ch_itemname: pub.Ch_itemname}
//{"itemaccesstype":"private","comment":"","meta":" ","label":{"sys":{"supply_style":"batch"}}}
isys := Sys{Supplystyle: pub.SupplyStyle}
icpub.Slabel = Label{Ssys: isys}
if len(priceplans) > 0 {
log.Info(priceplans)
icpub.PricePlans = priceplans
}
body, err := json.Marshal(icpub)
if err != nil {
s := "pub dataitem error while marshal icpub struct"
log.Println(s)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorMarshal, s)
return
}
log.Println(string(body))
log.Println("daemon: connecting to", DefaultServer+r.URL.Path)
req, err := http.NewRequest("POST", DefaultServer+r.URL.Path, bytes.NewBuffer(body))
req.Header.Set("Authorization", loginAuthStr)
resp, err := http.DefaultClient.Do(req)
if err != nil {
s := "Pub dataitem service unavailable"
HttpNoData(w, http.StatusServiceUnavailable, cmd.ErrorServiceUnavailable, s)
return
}
defer resp.Body.Close()
//Get server result
rbody, _ := ioutil.ReadAll(resp.Body)
log.Println(resp.StatusCode, string(rbody))
if resp.StatusCode == http.StatusOK {
err := MkdirForDataItem(repo, item, pub.Datapool, pub.ItemDesc)
if err != nil {
RollBackItem(repo, item)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
fmt.Sprintf("Mkdir error! %s", err.Error()))
} else {
createTime := time.Now().String()
err = InsertItemToDb(repo, item, pub.Datapool, pub.ItemDesc, createTime)
if err != nil {
RollBackItem(repo, item)
HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
"Insert dataitem to datapool error, please check it immediately!")
} else {
HttpNoData(w, http.StatusOK, cmd.ResultOK, "OK")
g_DaemonRole = PUBLISHER
}
}
} else {
result := ds.Result{}
err = json.Unmarshal(rbody, &result)
if err != nil {
s := "Pub dataitem error while unmarshal server response"
log.Println(s, err)
HttpNoData(w, resp.StatusCode, cmd.ErrorUnmarshal, s)
return
}
log.Println(resp.StatusCode, result.Msg)
HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
}
return
}
/*UpdateSql04To05() Temporarily not use*/
func UpdateSql04To05() (err error) {
//UPDATE DH_DP
TrimRightDpconn := `update DH_DP set DPCONN =substr(DPCONN,0,length(DPCONN)) where DPCONN like '%/';`
_, err = g_ds.Update(TrimRightDpconn)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
UpDhDp := `UPDATE DH_DP SET DPCONN=DPCONN||"/"||DPNAME;`
_, err = g_ds.Update(UpDhDp)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
//UPDATE DH_DP_RPDM_MAP
RenameDpRpdmMap := "ALTER TABLE DH_DP_RPDM_MAP RENAME TO OLD_DH_DP_RPDM_MAP;"
_, err = g_ds.Exec(RenameDpRpdmMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
_, err = g_ds.Create(ds.Create_dh_dp_repo_ditem_map)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
InsertDpRpdmMap := `INSERT INTO DH_DP_RPDM_MAP(RPDMID, REPOSITORY, DATAITEM, DPID, ITEMDESC
, PUBLISH, CREATE_TIME, STATUS)
SELECT RPDMID, REPOSITORY, DATAITEM, DPID, REPOSITORY||"/"||DATAITEM,
PUBLISH, CREATE_TIME, 'A' FROM OLD_DH_DP_RPDM_MAP;`
DropOldDpRpdmMap := `DROP TABLE OLD_DH_DP_RPDM_MAP;`
_, err = g_ds.Insert(InsertDpRpdmMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
_, err = g_ds.Drop(DropOldDpRpdmMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
//UPDATE DH_RPDM_TAG_MAP
RenameTagMap := "ALTER TABLE DH_RPDM_TAG_MAP RENAME TO OLD_DH_RPDM_TAG_MAP;"
_, err = g_ds.Exec(RenameTagMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
_, err = g_ds.Create(ds.Create_dh_repo_ditem_tag_map)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
InsertTagMap := `INSERT INTO DH_RPDM_TAG_MAP(TAGID, TAGNAME, RPDMID, DETAIL, CREATE_TIME, STATUS)
SELECT NULL, TAGNAME, RPDMID, DETAIL, CREATE_TIME, 'A' FROM OLD_DH_RPDM_TAG_MAP;`
DropOldTagMap := `DROP TABLE OLD_DH_RPDM_TAG_MAP;`
_, err = g_ds.Insert(InsertTagMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
_, err = g_ds.Drop(DropOldTagMap)
if err != nil {
l := log.Error(err)
logq.LogPutqueue(l)
return err
}
log.Info("update db successfully!")
return
}
func RunDaemon() {
//fmt.Println("Run daemon..")
// Daemonizing echo server application.
switch isDaemon, err := daemonigo.Daemonize(); {
case !isDaemon:
return
case err != nil:
log.Fatal("main(): could not start daemon, reason -> %s", err.Error())
}
//fmt.Printf("server := http.Server{}\n")
if false == isDirExists(g_strDpPath) {
err := os.MkdirAll(g_strDpPath, 0755)
if err != nil {
log.Printf("mkdir %s error! %v ", g_strDpPath, err)
}
}
DaemonAuthrization = utils.Getguid()
log.Println("DaemonAuthrization", DaemonAuthrization)
dbinit()
if len(DaemonID) == 40 {
log.Println("daemonid", DaemonID)
saveDaemonID(DaemonID)
} else {
log.Println("get daemonid from db")
DaemonID = getDaemonid()
}
LoadJobFromDB()
os.Chdir(g_strDpPath)
originalListener, err := net.Listen("tcp", DaemonCliServer)
if err != nil {
log.Fatal(err)
} //else {
// if err = os.Chmod(cmd.UnixSock, os.ModePerm); err != nil {
// l := log.Error(err)
// logq.LogPutqueue(l)
// }
// }
sl, err = tcpNew(originalListener)
if err != nil {
panic(err)
}
router := httprouter.New()
router.GET("/", serverFileHandler)
router.POST("/api/datapools", dpPostOneHandler)
router.GET("/api/datapools", dpGetAllHandler)
router.GET("/api/datapools/:dpname", dpGetOneHandler)
router.DELETE("/api/datapools/:dpname", dpDeleteOneHandler)
router.GET("/api/ep", epGetHandler)
router.POST("/api/ep", epPostHandler)
router.DELETE("/api/ep", epDeleteHandler)
router.GET("/api/repositories/:repo/:item/:tag", repoTagHandler)
router.GET("/api/repositories/:repo/:item", repoItemHandler)
router.GET("/api/repositories/:repo", repoRepoNameHandler)
router.GET("/api/repositories", repoHandler)
router.GET("/api/repositories/:repo/:item/:tag/judge", judgeTagExistHandler)
router.DELETE("/api/repositories/:repo/:item", repoDelOneItemHandler)
router.DELETE("/api/repositories/:repo/:item/:tag", repoDelTagHandler)
router.GET("/api/subscriptions/dataitems", subsHandler)
router.GET("/api/subscriptions/pull/:repo/:item", subsHandler)
router.POST("/api/repositories/:repo/:item", pubItemHandler)
router.POST("/api/repositories/:repo/:item/:tag", newPubTagHandler)
router.POST("/api/subscriptions/:repo/:item/pull", pullHandler)
router.GET("/api/job", jobHandler)
router.GET("/api/job/:id", jobDetailHandler)
router.DELETE("/api/job/:id", jobRmHandler)
router.DELETE("/api/job", jobRmAllHandler)
router.GET("/api/daemon/:repo/:item/:tag", tagStatusHandler)
router.GET("/api/daemon/:repo/:item", tagOfItemStatusHandler)
router.GET("/api/heartbeat/status/:user", userStatusHandler)
http.Handle("/", router)
http.HandleFunc("/api/stop", stopHttp)
http.HandleFunc("/api/users/auth", loginHandler)
http.HandleFunc("/api/users/logout", logoutHandler)
router.GET("/api/users/whoami", whoamiHandler)
router.GET("/api/pulled/:repo/:item", itemPulledHandler)
router.GET("/api/datapool/published/:dpname", publishedOfDatapoolHandler)
router.GET("/api/datapool/pulled/:dpname", pulledOfDatapoolHandler)
router.GET("/api/datapool/published/:dpname/:repo", publishedOfRepoHandler)
router.GET("/api/datapool/pulled/:dpname/:repo", pulledOfRepoHandler)
router.POST("/api/datapool/check", checkDpConnectHandler)
router.GET("/api/datapool/other/:dpname", dpGetOtherDataHandler)
router.GET("/api/datapool/pulled/:dpname/:repo/:item", pulledTagOfItemHandler)
router.GET("/api/datapool/published/:dpname/:repo/:item", publishedTagOfItemHandler)
//router.POST("/api/datapool/newpublishtag", newPublishTagHandler)
router.NotFound = &mux{}
server := http.Server{}
go func() {
stop := make(chan os.Signal)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
select {
case signal := <-stop:
log.Printf("Got signal:%v", signal)
}
sl.Stop()
if len(DaemonID) > 0 {
p2psl.Stop()
}
}()
if len(DaemonID) > 0 {
go startP2PServer()
go HeartBeat()
go CheckHealthClock()
//go datapoolMonitor() //Temporarily not use
go GetMessages()
go PullTagAutomatic()
} else {
l := log.Error("no daemonid specificed.")
logq.LogPutqueue(l)
fmt.Println("You don't have a daemonid specificed.")
}
/*
wg.Add(1)
defer wg.Done()
*/
log.Info("starting daemon listener...")
server.Serve(sl)
log.Info("Stopping daemon listener...")
if len(DaemonID) > 0 {
wg.Wait()
}
daemonigo.UnlockPidFile()
g_ds.Db.Close()
log.Info("daemon exit....")
log.CloseLogFile()
}
func repoDelOneItemHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
if len(loginAuthStr) == 0 {
HttpNoData(w, http.StatusUnauthorized, cmd.ErrorServiceUnavailable, " ")
return
}
repository := ps.ByName("repo")
dataitem := ps.ByName("item")
r.ParseForm()
ensure, _ := strconv.Atoi(r.Form.Get("ensure"))
reqBody, _ := ioutil.ReadAll(r.Body)
exist, msg, err := judgeRepoOrItemExist(repository, dataitem)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
return
}
if exist == false {
HttpNoData(w, http.StatusOK, cmd.RepoOrItemNotExist, msg)
return
}
if ensure == 0 {
path := "/api/subscriptions/push/" + repository + "/" + dataitem + "?phase=1"
retResp := ds.Response{}
Pages := ds.ResultPages{}
retResp.Data = &Pages
resp, err := commToServerGetRsp("get", path, reqBody)
defer resp.Body.Close()
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
return
}
respbody, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode == http.StatusOK {
unmarshalerr := json.Unmarshal(respbody, &retResp)
if unmarshalerr != nil {
log.Error(unmarshalerr)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
return
}
log.Info(string(respbody))
if Pages.Total > 0 {
HttpNoData(w, http.StatusOK, cmd.ExitsConsumingPlan, "Exist consuming subscription plan")
return
} else {
HttpNoData(w, http.StatusOK, cmd.NoConsumingPlan, "No consuming subscription plan")
return
}
} else {
HttpNoData(w, resp.StatusCode, cmd.ErrorUnknowError, "")
log.Error(string(respbody))
return
}
} else if ensure == 1 {
err = delTagsForDelItem(repository, dataitem)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tags")
return
}
err := delItem(repository, dataitem)
if err != nil {
log.Error(err)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete item")
return
}
resp, err := commToServerGetRsp("delete", r.URL.Path, reqBody)
if err != nil {
log.Error(err)
HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer err")
return
}
defer resp.Body.Close()
respbody, _ := ioutil.ReadAll(resp.Body)
retResp := ds.Response{}
unmarshalerr := json.Unmarshal(respbody, &retResp)
if unmarshalerr != nil {
log.Error(unmarshalerr)
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
return
}
if resp.StatusCode == http.StatusOK {
HttpNoData(w, http.StatusOK, cmd.ResultOK, retResp.Msg)
log.Info("Msg :", retResp.Msg, "HttpCode :", resp.StatusCode)
} else {
HttpNoData(w, resp.StatusCode, retResp.Code, retResp.Msg)
log.Error("Error :", retResp.Msg, "HttpCode :", resp.StatusCode)
rollbackDelItem(repository, dataitem)
rollbackDelTags(repository, dataitem)
}
}
}
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
logq.LogPutqueue(l)
r.ParseForm()
sRepoName := ps.ByName("repo")
sDataItem := ps.ByName("dataitem")
sTag := ps.ByName("tag")
log.Info(sRepoName, sDataItem, sTag)
jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
var irpdmid, idpid int
var stagdetail, itemdesc, dpconn, dpname, dptype string
msg := &ds.MsgResp{}
msg.Msg = "OK."
irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
if len(itemdesc) == 0 {
itemdesc = sRepoName + "_" + sDataItem
}
log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)
stagdetail = GetTagDetail(irpdmid, sTag)
log.Debug("tagdetail", stagdetail)
if len(stagdetail) == 0 {
l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
logq.LogPutqueue(l)
http.Error(rw, sTag+" not found", http.StatusNotFound)
return
}
dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)
datapool, err := dpdriver.New(dptype)
if err != nil {
WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
return
}
filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
log.Println("filename:", filepathname)
if exists := isFileExists(filepathname); !exists {
l := log.Error(filepathname, "not found")
logq.LogPutqueue(l)
putToJobQueue(jobtag, filepathname, "N/A", -1)
msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
resp, _ := json.Marshal(msg)
respStr := string(resp)
rw.WriteHeader(http.StatusNotFound)
fmt.Fprintln(rw, respStr)
return
}
tokenValid := false
retmsg := ""
token := r.Form.Get("token")
username := r.Form.Get("username")
log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
if len(token) > 0 && len(username) > 0 {
log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
logq.LogPutqueue(l)
}
log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
//rw.Header().Set("Source-FileName", stagdetail)
bmd5, err := ComputeMd5(filepathname)
strmd5 := fmt.Sprintf("%x", bmd5)
if err != nil {
log.Error(filepathname, err, bmd5, strmd5)
} else {
rw.Header().Set("X-Source-MD5", strmd5)
}
rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))
l = log.Info("transfering", filepathname, bmd5, strmd5)
logq.LogPutqueue(l)
jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
http.ServeFile(rw, r, filepathname)
updateJobQueue(jobid, "transfered", 0)
return
}
/*download routine, supports resuming broken downloads.*/
func download(uri string, p ds.DsPull, w http.ResponseWriter, c chan int) (int64, error) {
log.Printf("we are going to download %s, save to dp=%s,name=%s\n", uri, p.Datapool, p.DestName)
var out *os.File
var err error
var destfilename, tmpdestfilename, tmpdir, dpconn, dptype string
dpconn, dptype = GetDataPoolDpconnAndDptype(p.Datapool)
if len(dpconn) == 0 {
err = fmt.Errorf("dpconn is null! datapool:%s ", p.Datapool)
return ErrLogAndResp(c, w, http.StatusBadRequest, cmd.ErrorNoRecord, err)
}
//New a datapool object
datapool, err := dpdriver.New(dptype)
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
}
destfilename, tmpdir, tmpdestfilename = datapool.GetDestFileName(dpconn, p.ItemDesc, p.DestName)
os.MkdirAll(tmpdir, 0777)
log.Info("open tmp destfile name:", tmpdestfilename)
out, err = os.OpenFile(tmpdestfilename, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorOpenFile, err)
}
stat, err := out.Stat()
if err != nil {
return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorStatFile, err)
}
out.Seek(stat.Size(), 0)
req, err := http.NewRequest("GET", uri, nil)
req.Header.Set("User-Agent", "go-downloader")
/* Set download starting position with 'Range' in HTTP header*/
req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
log.Printf("%v bytes had already been downloaded.\n", stat.Size())
log.Debug(EnvDebug("http_proxy", false))
resp, err := http.DefaultClient.Do(req)
/*Save response body to file only when HTTP 2xx received. TODO*/
if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
log.Error("http error", err)
if resp != nil {
body, _ := ioutil.ReadAll(resp.Body)
l := log.Error("http status code:", resp.StatusCode, "response Body:", string(body), err)
logq.LogPutqueue(l)
struMsg := &ds.MsgResp{}
json.Unmarshal(body, struMsg)
msg := struMsg.Msg
if resp.StatusCode == 416 {
msg = tmpdestfilename + " has already been downloaded."
}
r, _ := buildResp(resp.StatusCode, msg, nil)
w.WriteHeader(resp.StatusCode)
w.Write(r)
} else {
HttpNoData(w, http.StatusInternalServerError, cmd.ErrorOtherError, err.Error())
}
filesize := stat.Size()
out.Close()
if filesize == 0 {
os.Remove(tmpdestfilename)
}
c <- -1
return 0, err
}
defer resp.Body.Close()
HttpNoData(w, http.StatusOK, cmd.ResultOK, strret)
//write channel
c <- 1
jobtag := p.Repository + "/" + p.Dataitem + ":" + p.Tag
srcsize, err := strconv.ParseInt(resp.Header.Get("X-Source-FileSize"), DECIMAL_BASE, INT_SIZE_64)
md5str := resp.Header.Get("X-Source-MD5")
status := "downloading"
log.Info("pull tag:", jobtag, tmpdestfilename, status, srcsize)
jobid := putToJobQueue(jobtag, tmpdestfilename, status, srcsize)
n, err := io.Copy(out, resp.Body)
if err != nil {
out.Close()
bl := log.Error(err)
logq.LogPutqueue(bl)
dlsize, e := GetFileSize(tmpdestfilename)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
status = "failed"
updateJobQueue(jobid, status, dlsize)
return 0, err
}
out.Close()
status = "downloaded"
if len(md5str) > 0 {
bmd5, err := ComputeMd5(tmpdestfilename)
bmd5str := fmt.Sprintf("%x", bmd5)
log.Debug("md5", md5str, tmpdestfilename, bmd5str)
if err != nil {
log.Error(tmpdestfilename, err, bmd5)
} else if md5str != bmd5str {
l := log.Errorf("check md5 code error! src md5:%v, local md5:%v", md5str, bmd5str)
logq.LogPutqueue(l)
status = "md5 error"
updateJobQueue(jobid, status, 0)
return n, nil
}
}
log.Printf("%d bytes downloaded.", n)
if err := MoveFromTmp(tmpdestfilename, destfilename); err != nil {
status = "MoveFromTmp error"
}
dlsize, e := GetFileSize(destfilename)
if e != nil {
l := log.Error(e)
logq.LogPutqueue(l)
}
status = datapool.StoreFile(status, destfilename, dpconn, p.Datapool, p.ItemDesc, p.DestName)
updateJobQueue(jobid, status, dlsize)
tagComment := GetTagComment(p.Repository, p.Dataitem, p.Tag)
InsertTagToDb(true, p, tagComment)
return n, nil
}
