Beispiel #1
0
func startP2PServer() {
	p2pListener, err := net.Listen("tcp", ":35800")
	if err != nil {
		log.Fatal(err)
	}

	p2psl, err = tcpNew(p2pListener)
	if err != nil {
		log.Fatal(err)
	}

	P2pRouter := httprouter.New()
	P2pRouter.GET("/", sayhello)
	P2pRouter.GET("/pull/:repo/:dataitem/:tag", p2p_pull)
	P2pRouter.GET("/health", p2pHealthyCheckHandler)

	p2pserver := http.Server{Handler: P2pRouter}

	wg.Add(1)
	defer wg.Done()

	log.Info("p2p server start")
	p2pserver.Serve(p2psl)
	log.Info("p2p server stop")

}
Beispiel #2
0
func commToServer(method, path string, buffer []byte, w http.ResponseWriter) (body []byte, err error) {
	//Trace()
	s := log.Info("daemon: connecting to", DefaultServer+path)
	logq.LogPutqueue(s)
	req, err := http.NewRequest(strings.ToUpper(method), DefaultServer+path, bytes.NewBuffer(buffer))
	if len(loginAuthStr) > 0 {
		req.Header.Set("Authorization", loginAuthStr)
	}

	//req.Header.Set("User", "admin")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Error(err)
		d := ds.Result{Code: cmd.ErrorServiceUnavailable, Msg: err.Error()}
		body, e := json.Marshal(d)
		if e != nil {
			log.Error(e)
			return body, e
		}
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write(body)
		return body, err
	}
	defer resp.Body.Close()

	w.WriteHeader(resp.StatusCode)
	body, err = ioutil.ReadAll(resp.Body)
	w.Write(body)
	log.Info(resp.StatusCode, string(body))
	return
}
Beispiel #3
0
func delTagsForDelItem(reponame, itemname string) error {
	log.Println("Begin to remove tags for remove item from db")
	sqlrpdmid := fmt.Sprintf(`SELECT RPDMID FROM DH_DP_RPDM_MAP WHERE REPOSITORY='%s' AND DATAITEM='%s' AND STATUS='A';`, reponame, itemname)

	row, err := g_ds.QueryRow(sqlrpdmid)
	if err != nil {
		l := log.Error("select rpdmid from DH_DP_RPDM_MAP error:", err)
		logq.LogPutqueue(l)
		return err
	}
	var rpdmId int
	row.Scan(&rpdmId)
	if rpdmId == 0 {
		log.Debug(reponame, itemname, "not exist.")
		return nil
	}
	sqldeltag := fmt.Sprintf(`UPDATE DH_RPDM_TAG_MAP SET STATUS='N' WHERE RPDMID=%d`, rpdmId)
	_, err = g_ds.Update(sqldeltag)
	log.Info("sqldeltag", sqldeltag)
	if err != nil {
		l := log.Error("delete tag error:", err)
		logq.LogPutqueue(l)
		return err
	}

	return nil
}
Beispiel #4
0
func (s3 *s3driver) CheckDataAndGetSize(dpconn, itemlocation, fileName string) (exist bool, size int64, err error) {
	bucket := getAwsInfoFromDpconn(dpconn)

	destFullPathFileName := bucket + "/" + itemlocation + "/" + fileName
	log.Info(destFullPathFileName)

	AWS_REGION = Env("AWS_REGION", false)

	svc := s3aws.New(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
	result, err := svc.ListObjects(&s3aws.ListObjectsInput{Bucket: aws.String(bucket),
		Prefix: aws.String(itemlocation + "/" + fileName)})
	if err != nil {
		log.Error("Failed to list objects", err)
		return exist, size, err
	}

	exist = false
	for _, v := range result.Contents {
		log.Infof("Tag:%s, key:%s, size:%v\n", aws.StringValue(v.ETag), aws.StringValue(v.Key), aws.Int64Value(v.Size))
		if aws.StringValue(v.Key) == fileName {
			size = aws.Int64Value(v.Size)
			exist = true
		}
	}

	return
}
Beispiel #5
0
func (hdfs *hdfsdriver) StoreFile(status, filename, dpconn, dp, itemlocation, destfile string) string {

	log.Infof("Begin to upload %v to %v\n", filename, dp)

	client, err := getClient(dpconn)
	if err != nil {
		log.Error("Failed to get a client", err)
		status = "put to hdfs err"
		return status
	}
	defer client.Close()

	err = client.MkdirAll("/"+itemlocation, 1777)
	if err != nil {
		log.Error("Failed to mkdirall in hdfs", err)
		status = "put to hdfs err"
		return status
	}

	hdfsfile := "/" + itemlocation + "/" + destfile
	err = client.CopyToRemote(filename, hdfsfile)
	if err != nil {
		log.Error("Failed to CopyToRemote", err)
		status = "put to hdfs err"
		return status
	}

	status = "put to hdfs ok"
	log.Info("Successfully uploaded to", itemlocation, "in hdfs")
	return status
}
Beispiel #6
0
func (hdfs *hdfsdriver) CheckDataAndGetSize(dpconn, itemlocation, fileName string) (exist bool, size int64, err error) {

	destFullPathFileName := "/" + itemlocation + "/" + fileName
	log.Info(destFullPathFileName)

	exist = false

	client, err := getClient(dpconn)
	if err != nil {
		log.Error("Failed to get a client", err)
		return
	}
	defer client.Close()

	fileinfo, _ := client.Stat(destFullPathFileName)

	if fileinfo != nil {
		exist = true
		cs, _ := client.GetContentSummary(destFullPathFileName)
		size = cs.Size()
	} else {
		err = errors.New("文件不存在")
		return
	}

	return
}
Beispiel #7
0
func UpdateStatMsgTagadded(ID, Status int) (err error) {

	log.Info("update MSG_TAGADDED status")
	sql := fmt.Sprintf(`UPDATE MSG_TAGADDED SET STATUS=%d 
		WHERE ID=%d;`, Status, ID)
	_, err = g_ds.Update(sql)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return
	}
	return
}
Beispiel #8
0
func getDaemonid() (id string) {
	log.Println("TODO get daemonid from db.")
	s := `SELECT DAEMONID FROM DH_DAEMON;`
	row, e := g_ds.QueryRow(s)
	if e != nil {
		l := log.Error(s, "error.", e)
		logq.LogPutqueue(l)
		return
	}
	row.Scan(&id)
	log.Info("daemon id is", id)
	return id
}
Beispiel #9
0
func testserver() {
	var tsl = new(StoppabletcpListener)

	Listener, err := net.Listen("tcp", ":35888")
	if err != nil {
		log.Fatal(err)
	}

	tsl, err = tcpNew(Listener)
	if err != nil {
		log.Fatal(err)
	}

	tRouter := httprouter.New()
	tRouter.GET("/", sayhello)

	http.Handle("/", tRouter)

	server := http.Server{Handler: tRouter}

	log.Info("p2p server start")
	server.Serve(tsl)
	log.Info("p2p server stop")
}
Beispiel #10
0
func epPostHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	reqBody, _ := ioutil.ReadAll(r.Body)
	ep := cmd.FormatEp{}
	if err := json.Unmarshal(reqBody, &ep); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	EntryPoint = ep.Ep
	saveEntryPoint(EntryPoint)

	msg := ds.MsgResp{Msg: "OK. your entrypoint is: " + EntryPoint + "\n" + "The entrypoint is setted successfully.  You can check whether it is available in one minute."}
	log.Info(msg.Msg)
	resp, _ := json.Marshal(&msg)
	fmt.Fprintln(w, string(resp))
	return
}
Beispiel #11
0
func commToServerGetRsp(method, path string, buffer []byte) (resp *http.Response, err error) {

	s := log.Info("daemon: connecting to", DefaultServer+path)
	logq.LogPutqueue(s)
	req, err := http.NewRequest(strings.ToUpper(method), DefaultServer+path, bytes.NewBuffer(buffer))
	if len(loginAuthStr) > 0 {
		req.Header.Set("Authorization", loginAuthStr)
	}

	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		log.Error(err)
		return resp, err
	}

	return resp, nil
}
Beispiel #12
0
func PullTagAutomatic() {
	for {
		time.Sleep(30 * time.Second)

		//log.Debug("AutomaticPullList.Len()", AutomaticPullList.Len())
		var Tags map[int]string
		for e := AutomaticPullList.Front(); e != nil; e = e.Next() {
			v := e.Value.(ds.DsPull)
			log.Info("PullTagAutomatic begin", v.Repository, v.Dataitem)
			Tags = GetTagFromMsgTagadded(v.Repository, v.Dataitem, NOTREAD)

			log.Println("Tags ", Tags)
			go PullItemAutomatic(Tags, v)

		}
	}
}
Beispiel #13
0
func (s3 *s3driver) StoreFile(status, filename, dpconn, dp, itemlocation, destfile string) string {
	bucket := getAwsInfoFromDpconn(dpconn)

	//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
	//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
	AWS_REGION = Env("AWS_REGION", false)

	file, err := os.Open(filename)
	if err != nil {
		l := log.Error("Failed to open file", err)
		logq.LogPutqueue(l)
		status = "put to s3 err"
		return status
	}

	log.Infof("Begin to upload %v to %v\n", filename, dp)

	// Not required, but you could zip the file before uploading it
	// using io.Pipe read/writer to stream gzip'd file contents.
	reader, writer := io.Pipe()
	go func() {
		gw := gzip.NewWriter(writer)
		io.Copy(gw, file)

		file.Close()
		gw.Close()
		writer.Close()

		//updateJobQueueStatus(jobid, "puttos3ok")
	}()
	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
	//uploader := s3manager.NewUploader(session.New(aws.NewConfig()))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   reader,
		Bucket: aws.String(bucket),
		Key:    aws.String( /*dp + "/" + */ itemlocation + "/" + destfile + ".gz"),
	})
	if err != nil {
		log.Error("Failed to upload", err)
		status = "put to s3 err"
		return status
	}
	status = "put to s3 ok"
	log.Info("Successfully uploaded to", result.Location)
	return status
}
Beispiel #14
0
func CheckHealthClock() {
	log.Debug("--------->BEGIN")

	checkHealth(&Errortagsmap)

	timer := time.NewTicker(10 * time.Minute)
	for {
		select {
		case <-timer.C:
			now := time.Now()
			if now.Hour()%6 == 0 {
				log.Info("Time:", now)
				checkHealth(&Errortagsmap)
			}
		}
	}
	log.Debug("---------->END")
}
Beispiel #15
0
func AutomaticPullRmqueue(p ds.DsPull) (exist bool) {
	exist = false
	pullmu.Lock()
	defer pullmu.Unlock()

	var next *list.Element
	for e := AutomaticPullList.Front(); e != nil; e = next {
		v := e.Value.(ds.DsPull)
		if v.Repository == p.Repository && v.Dataitem == p.Dataitem {
			exist = true
			AutomaticPullList.Remove(e)
			log.Info(v, "removed from the queue.")
			break
		} else {
			next = e.Next()
		}
	}
	return
}
Beispiel #16
0
func (s3 *s3driver) GetFileTobeSend(dpconn, dpname, itemlocation, tagdetail string) (filepathname string) {

	bucket := getAwsInfoFromDpconn(dpconn)

	e := os.MkdirAll(gDpPath+"/"+bucket+"/"+itemlocation, 0777)
	if e != nil {
		log.Error(e)
		return
	}

	filepathname = gDpPath + "/" + bucket + "/" + itemlocation + "/" + tagdetail

	if true == isFileExists(filepathname) {
		return
	}

	//AWS_SECRET_ACCESS_KEY = Env("AWS_SECRET_ACCESS_KEY", false)
	//AWS_ACCESS_KEY_ID = Env("AWS_ACCESS_KEY_ID", false)
	AWS_REGION = Env("AWS_REGION", false)
	file, err := os.Create(filepathname)
	if err != nil {
		log.Error("Failed to create file", err)
		return ""
	}
	defer file.Close()

	downloader := s3manager.NewDownloader(session.New(&aws.Config{Region: aws.String(AWS_REGION)}))
	numBytes, err := downloader.Download(file,
		&s3aws.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String( /*dpname + "/" + */ itemlocation + "/" + tagdetail),
		})
	if err != nil {
		log.Info("Failed to download file.", err)
		os.Remove(filepathname)
		return
	}

	log.Println("Downloaded file", file.Name(), numBytes, "bytes")

	return
}
Beispiel #17
0
func checkHealth(errorTagsMap *map[string]string) {

	localfiles := make([]string, 0)
	alllocalfiles := make([]string, 0)
	localfilepath := GetLocalfilePath()

	for _, path := range localfilepath {
		localfiles = ScanLocalFile(path)
		for _, localfile := range localfiles {
			alllocalfiles = append(alllocalfiles, localfile)
		}
	}

	log.Info(alllocalfiles)
	var tagDetails map[string]string
	tagDetails = make(map[string]string)

	err := GetAllTagDetails(&tagDetails)
	if err != nil {
		log.Error(err)
	}

	var i int
	for file, tag := range tagDetails {
		for i = 0; i < len(alllocalfiles); i++ {
			//log.Info("--------->tag:", tag)
			//log.Info("--------->tagfile:",file)
			//log.Info("--------->localfile:",localfiles[i])
			if file == alllocalfiles[i] {
				break
			}
		}
		if i >= len(alllocalfiles) {
			(*errorTagsMap)[file] = tag
		}
	}

	//for errortagfile, errortag := range *errorTagsMap {
	//	log.Info("------->errortag:", errortag, "-------->", errortagfile)
	//}
}
Beispiel #18
0
func judgeRepoOrItemExist(repository, dataitem string) (exist bool, msg string, err error) {

	path := "/api/repositories/" + repository + "/" + dataitem

	exist = false

	resp, err := commToServerGetRsp("get", path, nil)
	if err != nil {
		log.Error(err)
		//HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
		err = errors.New("unkown error")
		return
	}
	result := ds.Response{}
	respbody, _ := ioutil.ReadAll(resp.Body)
	unmarshalerr := json.Unmarshal(respbody, &result)
	if unmarshalerr != nil {
		log.Error(unmarshalerr)
		//HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
		return
	}
	log.Info(string(respbody))

	if resp.StatusCode == http.StatusBadRequest && result.Code == cmd.ServerErrResultCode1009 {
		//HttpNoData(w, http.StatusBadRequest, cmd.RepoOrItemNotExist, result.Msg)
		msg = result.Msg
		return
	} else if resp.StatusCode == http.StatusOK && result.Code == cmd.ServerErrResultCodeOk {
		exist = true
		msg = result.Msg
		return
	}

	return
}
Beispiel #19
0
func judgeTagExist(repository, dataitem, tag string) (exist bool, msg string, err error) {

	path := "/api/repositories/" + repository + "/" + dataitem + "/" + tag

	exist = false

	resp, err := commToServerGetRsp("get", path, nil)
	defer resp.Body.Close()
	if err != nil {
		log.Error(err)
		return
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
		err = errors.New("unkown error")
		return
	}

	result := ds.Response{}
	respbody, _ := ioutil.ReadAll(resp.Body)
	unmarshalerr := json.Unmarshal(respbody, &result)
	if unmarshalerr != nil {
		log.Error(unmarshalerr)
		return
	}
	log.Info(string(respbody))

	if resp.StatusCode == http.StatusBadRequest && result.Code == cmd.ServerErrResultCode1009 {
		msg = result.Msg
		return
	} else if resp.StatusCode == http.StatusOK && result.Code == cmd.ServerErrResultCodeOk {
		exist = true
		msg = result.Msg
		return
	}

	return
}
Beispiel #20
0
func repoDelTagHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	if len(loginAuthStr) == 0 {
		HttpNoData(w, http.StatusUnauthorized, cmd.ErrorServiceUnavailable, " ")
		return
	}

	repository := ps.ByName("repo")
	dataitem := ps.ByName("item")
	tag := ps.ByName("tag")

	reqBody, _ := ioutil.ReadAll(r.Body)

	if strings.Contains(tag, "*") {
		tagsname, err := getBatchDelTagsName(repository, dataitem, tag)
		if err != nil {
			log.Error(err)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag.")
			return
		}
		if len(tagsname) == 0 {
			log.Println("没有匹配的tag")
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "No match tag.")
			return
		}
		successflag := true
		for _, tagname := range tagsname {
			if successflag {
				_, err := delTag(repository, dataitem, tagname)
				if err != nil {
					log.Error(err)
					HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag")
					return
				}

				path := "/api/repositories/" + repository + "/" + dataitem + "/" + tagname
				resp, err := commToServerGetRsp("delete", path, reqBody)
				if err != nil {
					log.Error(err)
					HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer error")
					return
				}
				defer resp.Body.Close()

				result := ds.Response{}

				respbody, err := ioutil.ReadAll(resp.Body)

				unmarshalerr := json.Unmarshal(respbody, &result)
				if unmarshalerr != nil {
					log.Error(unmarshalerr)
					HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
					return
				}
				if resp.StatusCode == http.StatusOK && result.Code == 0 {
					continue
				} else if resp.StatusCode == http.StatusOK && result.Code != 0 {
					HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
					log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
					successflag = false
					break
				} else {
					HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
					log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
					successflag = false
					break
				}
			}

		}
		if successflag {
			log.Info("批量删除tag成功")
			HttpNoData(w, http.StatusOK, cmd.ResultOK, "ok.")
		}
	} else {
		tagid, err := delTag(repository, dataitem, tag)
		if err != nil {
			log.Error(err)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tag")
			return
		}

		reqBody, _ := ioutil.ReadAll(r.Body)
		path := "/api/repositories/" + repository + "/" + dataitem + "/" + tag
		resp, err := commToServerGetRsp("delete", path, reqBody)
		if err != nil {
			log.Error(err)
			HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer error")
			return
		}
		defer resp.Body.Close()

		result := ds.Response{}

		respbody, err := ioutil.ReadAll(resp.Body)

		unmarshalerr := json.Unmarshal(respbody, &result)
		if unmarshalerr != nil {
			log.Error(unmarshalerr)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
			return
		}
		if resp.StatusCode == http.StatusOK && result.Code == 0 {
			HttpNoData(w, http.StatusOK, cmd.ResultOK, result.Msg)
			log.Info("Msg :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
		} else if resp.StatusCode == http.StatusOK && result.Code != 0 {
			HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
			log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
			rollbackDelTag(tagid)
		} else {
			HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
			log.Info("Error :", result.Msg, "ResultCode:", result.Code, "HttpCode :", resp.StatusCode)
		}
	}
}
Beispiel #21
0
func GetMessages() {
	log.Info("start GetMessages from messages server")
	var sleepInterval int
	var srtInterval string
	var e error
	url := DefaultServerAPI + "/notifications?forclient=1&type=item_event&status=0"
	for AutoPull == true {

		if srtInterval = os.Getenv("DATAHUB_MSG_INTERVAL"); len(srtInterval) > 0 {
			sleepInterval, e = strconv.Atoi(srtInterval)
			if e != nil {
				l := log.Error(e)
				logq.LogPutqueue(l)
			}
		} else {
			sleepInterval = 600
		}

		time.Sleep(time.Duration(sleepInterval) * time.Second)
		log.Debug("connecting to", url)
		req, err := http.NewRequest("GET", url, nil)

		if len(loginAuthStr) > 0 {
			req.Header.Set("Authorization", loginAuthStr)
		}

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			l := log.Error(err)
			logq.LogPutqueue(l)

			continue
		}
		defer resp.Body.Close()

		body, _ := ioutil.ReadAll(resp.Body)

		if resp.StatusCode == http.StatusOK {
			log.Debugf("HeartBeat http statuscode:%v,  http body:%s", resp.StatusCode, body)

			result := ds.Result{}
			Pages := ds.ResultPages{}
			MessagesSlice := []Messages{}
			Pages.Results = &MessagesSlice
			result.Data = &Pages

			if err := json.Unmarshal(body, &result); err == nil {
				if result.Code == 0 {
					log.Debug(result)
					for _, v := range MessagesSlice {
						if v.Type == "item_event" && v.Data.Event == TAGADDED {
							InsertToTagadded(v.Data.EventTime, v.Data.Repname, v.Data.Itemname, v.Data.Tag, NOTREAD)
						}
					}
				} else {
					l := log.Error("Get Messages errror:", result.Code)
					logq.LogPutqueue(l)
				}
			} else {
				log.Error(err)
			}

		} else if resp.StatusCode == http.StatusUnauthorized {
			log.Debug("not login", http.StatusUnauthorized)
			urllogin := DefaultServerAPI + "/"
			reql, err := http.NewRequest("GET", urllogin, nil)
			if len(loginBasicAuthStr) > 0 {
				reql.Header.Set("Authorization", loginBasicAuthStr)
				log.Info("user name:", gstrUsername)
			} else {
				log.Warn("not login")
				continue
			}

			respl, err := http.DefaultClient.Do(reql)
			if err != nil {
				log.Error(err)
				continue
			}
			defer respl.Body.Close()

			result := &ds.Result{}
			log.Println("login return", respl.StatusCode)
			if respl.StatusCode == 200 {
				body, _ := ioutil.ReadAll(respl.Body)
				log.Println(string(body))

				result.Data = &tk{}
				if err = json.Unmarshal(body, result); err != nil {
					log.Error(err)
					log.Println(respl.StatusCode, string(body))
					continue
				} else {
					loginAuthStr = "Token " + result.Data.(*tk).Token
					loginLogged = true
					log.Println(loginAuthStr)
				}
			}
		}
	}
}
Beispiel #22
0
func pubItemHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	log.Println(r.URL.Path, "(pub dataitem)")
	repo := ps.ByName("repo")
	item := ps.ByName("item")

	if CheckLength(w, repo, MaxRepoLength) == false {
		return
	}

	if CheckLength(w, item, MaxItemLength) == false {
		return
	}

	if len(loginAuthStr) == 0 {
		HttpNoData(w, http.StatusUnauthorized, cmd.ErrorUnAuthorization, "Unlogin")
		return
	}

	pub := ds.PubPara{}
	pub.ItemDesc = strings.Trim(pub.ItemDesc, "/")
	if strings.Contains(pub.ItemDesc, "/") == true {
		log.Println("The path of item can't contain '/'.", pub.ItemDesc)
		w.Write([]byte(`{"msg":"The path of item can't contain '/'."}`))
		return
	}
	if CheckLength(w, pub.Comment, MaxCommentLength) == false {
		return
	}

	reqBody, _ := ioutil.ReadAll(r.Body)
	if err := json.Unmarshal(reqBody, &pub); err != nil {
		HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal, "pub dataitem error while unmarshal reqBody")
		return
	}

	if CheckDataPoolExist(pub.Datapool) == false {
		HttpNoData(w, http.StatusBadRequest, cmd.ErrorUnmarshal,
			fmt.Sprintf("Datapool %s not found", pub.Datapool))
		return
	}

	//ToDO  check item dir exist

	priceplans := []PricePlan{}

	meta, sample, priceplans := GetMetaAndSampleAndPricePlan(pub.Datapool, pub.ItemDesc)
	icpub := ic{AccessType: pub.Accesstype,
		Comment:     pub.Comment,
		Meta:        meta,
		Sample:      sample,
		Ch_itemname: pub.Ch_itemname}

	//{"itemaccesstype":"private","comment":"","meta":"  ","label":{"sys":{"supply_style":"batch"}}}
	isys := Sys{Supplystyle: pub.SupplyStyle}
	icpub.Slabel = Label{Ssys: isys}
	if len(priceplans) > 0 {
		log.Info(priceplans)
		icpub.PricePlans = priceplans
	}

	body, err := json.Marshal(icpub)
	if err != nil {
		s := "pub dataitem error while marshal icpub struct"
		log.Println(s)
		HttpNoData(w, http.StatusBadRequest, cmd.ErrorMarshal, s)
		return
	}
	log.Println(string(body))

	log.Println("daemon: connecting to", DefaultServer+r.URL.Path)
	req, err := http.NewRequest("POST", DefaultServer+r.URL.Path, bytes.NewBuffer(body))

	req.Header.Set("Authorization", loginAuthStr)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		s := "Pub dataitem service unavailable"
		HttpNoData(w, http.StatusServiceUnavailable, cmd.ErrorServiceUnavailable, s)
		return
	}
	defer resp.Body.Close()
	//Get server result
	rbody, _ := ioutil.ReadAll(resp.Body)
	log.Println(resp.StatusCode, string(rbody))

	if resp.StatusCode == http.StatusOK {
		err := MkdirForDataItem(repo, item, pub.Datapool, pub.ItemDesc)
		if err != nil {
			RollBackItem(repo, item)
			HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
				fmt.Sprintf("Mkdir error! %s", err.Error()))
		} else {
			createTime := time.Now().String()
			err = InsertItemToDb(repo, item, pub.Datapool, pub.ItemDesc, createTime)
			if err != nil {
				RollBackItem(repo, item)
				HttpNoData(w, http.StatusBadRequest, cmd.ErrorInsertItem,
					"Insert dataitem to datapool error, please check it immediately!")
			} else {
				HttpNoData(w, http.StatusOK, cmd.ResultOK, "OK")

				g_DaemonRole = PUBLISHER
			}
		}
	} else {

		result := ds.Result{}
		err = json.Unmarshal(rbody, &result)
		if err != nil {
			s := "Pub dataitem error while unmarshal server response"
			log.Println(s, err)
			HttpNoData(w, resp.StatusCode, cmd.ErrorUnmarshal, s)
			return
		}
		log.Println(resp.StatusCode, result.Msg)
		HttpNoData(w, resp.StatusCode, result.Code, result.Msg)
	}

	return

}
Beispiel #23
0
/*UpdateSql04To05()  Temporarily not use*/
func UpdateSql04To05() (err error) {
	//UPDATE DH_DP
	TrimRightDpconn := `update DH_DP set DPCONN =substr(DPCONN,0,length(DPCONN)) where DPCONN like '%/';`
	_, err = g_ds.Update(TrimRightDpconn)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	UpDhDp := `UPDATE DH_DP SET DPCONN=DPCONN||"/"||DPNAME;`
	_, err = g_ds.Update(UpDhDp)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}

	//UPDATE DH_DP_RPDM_MAP
	RenameDpRpdmMap := "ALTER TABLE DH_DP_RPDM_MAP RENAME TO OLD_DH_DP_RPDM_MAP;"
	_, err = g_ds.Exec(RenameDpRpdmMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	_, err = g_ds.Create(ds.Create_dh_dp_repo_ditem_map)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	InsertDpRpdmMap := `INSERT INTO DH_DP_RPDM_MAP(RPDMID, REPOSITORY, DATAITEM, DPID, ITEMDESC
						, PUBLISH, CREATE_TIME, STATUS) 
						SELECT RPDMID, REPOSITORY, DATAITEM, DPID, REPOSITORY||"/"||DATAITEM, 
						PUBLISH, CREATE_TIME, 'A' FROM OLD_DH_DP_RPDM_MAP;`
	DropOldDpRpdmMap := `DROP TABLE OLD_DH_DP_RPDM_MAP;`
	_, err = g_ds.Insert(InsertDpRpdmMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	_, err = g_ds.Drop(DropOldDpRpdmMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}

	//UPDATE DH_RPDM_TAG_MAP
	RenameTagMap := "ALTER TABLE DH_RPDM_TAG_MAP RENAME TO OLD_DH_RPDM_TAG_MAP;"
	_, err = g_ds.Exec(RenameTagMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	_, err = g_ds.Create(ds.Create_dh_repo_ditem_tag_map)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	InsertTagMap := `INSERT INTO DH_RPDM_TAG_MAP(TAGID, TAGNAME, RPDMID, DETAIL, CREATE_TIME, STATUS) 
					SELECT NULL, TAGNAME, RPDMID, DETAIL, CREATE_TIME, 'A' FROM OLD_DH_RPDM_TAG_MAP;`
	DropOldTagMap := `DROP TABLE OLD_DH_RPDM_TAG_MAP;`
	_, err = g_ds.Insert(InsertTagMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	_, err = g_ds.Drop(DropOldTagMap)
	if err != nil {
		l := log.Error(err)
		logq.LogPutqueue(l)
		return err
	}
	log.Info("update db successfully!")
	return
}
Beispiel #24
0
func RunDaemon() {
	//fmt.Println("Run daemon..")
	// Daemonizing echo server application.

	switch isDaemon, err := daemonigo.Daemonize(); {
	case !isDaemon:
		return
	case err != nil:
		log.Fatal("main(): could not start daemon, reason -> %s", err.Error())
	}
	//fmt.Printf("server := http.Server{}\n")

	if false == isDirExists(g_strDpPath) {
		err := os.MkdirAll(g_strDpPath, 0755)
		if err != nil {
			log.Printf("mkdir %s error! %v ", g_strDpPath, err)
		}

	}

	DaemonAuthrization = utils.Getguid()
	log.Println("DaemonAuthrization", DaemonAuthrization)

	dbinit()

	if len(DaemonID) == 40 {
		log.Println("daemonid", DaemonID)
		saveDaemonID(DaemonID)
	} else {
		log.Println("get daemonid from db")
		DaemonID = getDaemonid()
	}

	LoadJobFromDB()

	os.Chdir(g_strDpPath)
	originalListener, err := net.Listen("tcp", DaemonCliServer)
	if err != nil {
		log.Fatal(err)
	} //else {
	// 	if err = os.Chmod(cmd.UnixSock, os.ModePerm); err != nil {
	// 		l := log.Error(err)
	// 		logq.LogPutqueue(l)
	// 	}
	// }

	sl, err = tcpNew(originalListener)
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	router.GET("/", serverFileHandler)
	router.POST("/api/datapools", dpPostOneHandler)
	router.GET("/api/datapools", dpGetAllHandler)
	router.GET("/api/datapools/:dpname", dpGetOneHandler)
	router.DELETE("/api/datapools/:dpname", dpDeleteOneHandler)

	router.GET("/api/ep", epGetHandler)
	router.POST("/api/ep", epPostHandler)
	router.DELETE("/api/ep", epDeleteHandler)

	router.GET("/api/repositories/:repo/:item/:tag", repoTagHandler)
	router.GET("/api/repositories/:repo/:item", repoItemHandler)
	router.GET("/api/repositories/:repo", repoRepoNameHandler)
	router.GET("/api/repositories", repoHandler)
	router.GET("/api/repositories/:repo/:item/:tag/judge", judgeTagExistHandler)
	router.DELETE("/api/repositories/:repo/:item", repoDelOneItemHandler)
	router.DELETE("/api/repositories/:repo/:item/:tag", repoDelTagHandler)

	router.GET("/api/subscriptions/dataitems", subsHandler)
	router.GET("/api/subscriptions/pull/:repo/:item", subsHandler)

	router.POST("/api/repositories/:repo/:item", pubItemHandler)
	router.POST("/api/repositories/:repo/:item/:tag", newPubTagHandler)

	router.POST("/api/subscriptions/:repo/:item/pull", pullHandler)

	router.GET("/api/job", jobHandler)
	router.GET("/api/job/:id", jobDetailHandler)
	router.DELETE("/api/job/:id", jobRmHandler)
	router.DELETE("/api/job", jobRmAllHandler)

	router.GET("/api/daemon/:repo/:item/:tag", tagStatusHandler)
	router.GET("/api/daemon/:repo/:item", tagOfItemStatusHandler)

	router.GET("/api/heartbeat/status/:user", userStatusHandler)

	http.Handle("/", router)
	http.HandleFunc("/api/stop", stopHttp)
	http.HandleFunc("/api/users/auth", loginHandler)
	http.HandleFunc("/api/users/logout", logoutHandler)

	router.GET("/api/users/whoami", whoamiHandler)
	router.GET("/api/pulled/:repo/:item", itemPulledHandler)

	router.GET("/api/datapool/published/:dpname", publishedOfDatapoolHandler)
	router.GET("/api/datapool/pulled/:dpname", pulledOfDatapoolHandler)
	router.GET("/api/datapool/published/:dpname/:repo", publishedOfRepoHandler)
	router.GET("/api/datapool/pulled/:dpname/:repo", pulledOfRepoHandler)
	router.POST("/api/datapool/check", checkDpConnectHandler)
	router.GET("/api/datapool/other/:dpname", dpGetOtherDataHandler)
	router.GET("/api/datapool/pulled/:dpname/:repo/:item", pulledTagOfItemHandler)
	router.GET("/api/datapool/published/:dpname/:repo/:item", publishedTagOfItemHandler)
	//router.POST("/api/datapool/newpublishtag", newPublishTagHandler)

	router.NotFound = &mux{}

	server := http.Server{}

	go func() {

		stop := make(chan os.Signal)
		signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

		select {
		case signal := <-stop:
			log.Printf("Got signal:%v", signal)
		}

		sl.Stop()
		if len(DaemonID) > 0 {
			p2psl.Stop()
		}

	}()

	if len(DaemonID) > 0 {
		go startP2PServer()
		go HeartBeat()
		go CheckHealthClock()
		//go datapoolMonitor()  //Temporarily not use
		go GetMessages()
		go PullTagAutomatic()
	} else {
		l := log.Error("no daemonid specificed.")
		logq.LogPutqueue(l)
		fmt.Println("You don't have a daemonid specificed.")
	}

	/*
		wg.Add(1)
		defer wg.Done()
	*/
	log.Info("starting daemon listener...")
	server.Serve(sl)
	log.Info("Stopping daemon listener...")

	if len(DaemonID) > 0 {
		wg.Wait()
	}

	daemonigo.UnlockPidFile()
	g_ds.Db.Close()

	log.Info("daemon exit....")
	log.CloseLogFile()

}
Beispiel #25
0
func repoDelOneItemHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	if len(loginAuthStr) == 0 {
		HttpNoData(w, http.StatusUnauthorized, cmd.ErrorServiceUnavailable, " ")
		return
	}
	repository := ps.ByName("repo")
	dataitem := ps.ByName("item")

	r.ParseForm()
	ensure, _ := strconv.Atoi(r.Form.Get("ensure"))

	reqBody, _ := ioutil.ReadAll(r.Body)

	exist, msg, err := judgeRepoOrItemExist(repository, dataitem)
	if err != nil {
		log.Error(err)
		HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
		return
	}
	if exist == false {
		HttpNoData(w, http.StatusOK, cmd.RepoOrItemNotExist, msg)
		return
	}

	if ensure == 0 {
		path := "/api/subscriptions/push/" + repository + "/" + dataitem + "?phase=1"

		retResp := ds.Response{}
		Pages := ds.ResultPages{}
		retResp.Data = &Pages

		resp, err := commToServerGetRsp("get", path, reqBody)
		defer resp.Body.Close()
		if err != nil {
			log.Error(err)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorServiceUnavailable, err.Error())
			return
		}
		respbody, _ := ioutil.ReadAll(resp.Body)
		if resp.StatusCode == http.StatusOK {
			unmarshalerr := json.Unmarshal(respbody, &retResp)
			if unmarshalerr != nil {
				log.Error(unmarshalerr)
				HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
				return
			}
			log.Info(string(respbody))
			if Pages.Total > 0 {

				HttpNoData(w, http.StatusOK, cmd.ExitsConsumingPlan, "Exist consuming subscription plan")
				return
			} else {

				HttpNoData(w, http.StatusOK, cmd.NoConsumingPlan, "No consuming subscription plan")
				return
			}
		} else {
			HttpNoData(w, resp.StatusCode, cmd.ErrorUnknowError, "")
			log.Error(string(respbody))
			return
		}

	} else if ensure == 1 {
		err = delTagsForDelItem(repository, dataitem)
		if err != nil {
			log.Error(err)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete tags")
			return
		}
		err := delItem(repository, dataitem)
		if err != nil {
			log.Error(err)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorSqlExec, "error while delete item")
			return
		}

		resp, err := commToServerGetRsp("delete", r.URL.Path, reqBody)
		if err != nil {
			log.Error(err)
			HttpNoData(w, resp.StatusCode, cmd.ErrorServiceUnavailable, "commToServer err")
			return
		}
		defer resp.Body.Close()

		respbody, _ := ioutil.ReadAll(resp.Body)
		retResp := ds.Response{}
		unmarshalerr := json.Unmarshal(respbody, &retResp)
		if unmarshalerr != nil {
			log.Error(unmarshalerr)
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorUnmarshal, "error while unmarshal respBody")
			return
		}
		if resp.StatusCode == http.StatusOK {
			HttpNoData(w, http.StatusOK, cmd.ResultOK, retResp.Msg)
			log.Info("Msg :", retResp.Msg, "HttpCode :", resp.StatusCode)
		} else {
			HttpNoData(w, resp.StatusCode, retResp.Code, retResp.Msg)
			log.Error("Error :", retResp.Msg, "HttpCode :", resp.StatusCode)
			rollbackDelItem(repository, dataitem)
			rollbackDelTags(repository, dataitem)
		}
	}

}
Beispiel #26
0
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
	logq.LogPutqueue(l)

	r.ParseForm()
	sRepoName := ps.ByName("repo")
	sDataItem := ps.ByName("dataitem")
	sTag := ps.ByName("tag")

	log.Info(sRepoName, sDataItem, sTag)
	jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
	var irpdmid, idpid int
	var stagdetail, itemdesc, dpconn, dpname, dptype string
	msg := &ds.MsgResp{}
	msg.Msg = "OK."

	irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
	if len(itemdesc) == 0 {
		itemdesc = sRepoName + "_" + sDataItem
	}
	log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)

	stagdetail = GetTagDetail(irpdmid, sTag)
	log.Debug("tagdetail", stagdetail)
	if len(stagdetail) == 0 {
		l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
		logq.LogPutqueue(l)
		http.Error(rw, sTag+" not found", http.StatusNotFound)
		return
	}

	dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
	log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)

	datapool, err := dpdriver.New(dptype)
	if err != nil {
		WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
		return
	}

	filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
	//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
	log.Println("filename:", filepathname)
	if exists := isFileExists(filepathname); !exists {
		l := log.Error(filepathname, "not found")
		logq.LogPutqueue(l)
		putToJobQueue(jobtag, filepathname, "N/A", -1)
		msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
		resp, _ := json.Marshal(msg)
		respStr := string(resp)
		rw.WriteHeader(http.StatusNotFound)
		fmt.Fprintln(rw, respStr)
		return
	}

	tokenValid := false
	retmsg := ""
	token := r.Form.Get("token")
	username := r.Form.Get("username")

	log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
	if len(token) > 0 && len(username) > 0 {
		log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
			"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
		logq.LogPutqueue(l)
	}
	log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
	//rw.Header().Set("Source-FileName", stagdetail)
	bmd5, err := ComputeMd5(filepathname)
	strmd5 := fmt.Sprintf("%x", bmd5)
	if err != nil {
		log.Error(filepathname, err, bmd5, strmd5)
	} else {
		rw.Header().Set("X-Source-MD5", strmd5)
	}
	rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))

	l = log.Info("transfering", filepathname, bmd5, strmd5)
	logq.LogPutqueue(l)

	jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
	http.ServeFile(rw, r, filepathname)
	updateJobQueue(jobid, "transfered", 0)

	return
}
Beispiel #27
0
/*download routine, supports resuming broken downloads.*/
func download(uri string, p ds.DsPull, w http.ResponseWriter, c chan int) (int64, error) {
	log.Printf("we are going to download %s, save to dp=%s,name=%s\n", uri, p.Datapool, p.DestName)

	var out *os.File
	var err error
	var destfilename, tmpdestfilename, tmpdir, dpconn, dptype string

	dpconn, dptype = GetDataPoolDpconnAndDptype(p.Datapool)
	if len(dpconn) == 0 {
		err = fmt.Errorf("dpconn is null! datapool:%s ", p.Datapool)
		return ErrLogAndResp(c, w, http.StatusBadRequest, cmd.ErrorNoRecord, err)
	}

	//New a datapool object
	datapool, err := dpdriver.New(dptype)
	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
	}
	destfilename, tmpdir, tmpdestfilename = datapool.GetDestFileName(dpconn, p.ItemDesc, p.DestName)

	os.MkdirAll(tmpdir, 0777)

	log.Info("open tmp destfile name:", tmpdestfilename)
	out, err = os.OpenFile(tmpdestfilename, os.O_RDWR|os.O_CREATE, 0644)

	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorOpenFile, err)
	}

	stat, err := out.Stat()
	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorStatFile, err)
	}
	out.Seek(stat.Size(), 0)

	req, err := http.NewRequest("GET", uri, nil)
	req.Header.Set("User-Agent", "go-downloader")
	/* Set download starting position with 'Range' in HTTP header*/
	req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
	log.Printf("%v bytes had already been downloaded.\n", stat.Size())

	log.Debug(EnvDebug("http_proxy", false))

	resp, err := http.DefaultClient.Do(req)

	/*Save response body to file only when HTTP 2xx received. TODO*/
	if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
		log.Error("http error", err)
		if resp != nil {
			body, _ := ioutil.ReadAll(resp.Body)
			l := log.Error("http status code:", resp.StatusCode, "response Body:", string(body), err)
			logq.LogPutqueue(l)
			struMsg := &ds.MsgResp{}
			json.Unmarshal(body, struMsg)
			msg := struMsg.Msg
			if resp.StatusCode == 416 {
				msg = tmpdestfilename + " has already been downloaded."
			}
			r, _ := buildResp(resp.StatusCode, msg, nil)

			w.WriteHeader(resp.StatusCode)
			w.Write(r)
		} else {
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorOtherError, err.Error())
		}
		filesize := stat.Size()
		out.Close()
		if filesize == 0 {
			os.Remove(tmpdestfilename)
		}
		c <- -1
		return 0, err
	}
	defer resp.Body.Close()

	HttpNoData(w, http.StatusOK, cmd.ResultOK, strret)

	//write channel
	c <- 1

	jobtag := p.Repository + "/" + p.Dataitem + ":" + p.Tag

	srcsize, err := strconv.ParseInt(resp.Header.Get("X-Source-FileSize"), DECIMAL_BASE, INT_SIZE_64)
	md5str := resp.Header.Get("X-Source-MD5")
	status := "downloading"
	log.Info("pull tag:", jobtag, tmpdestfilename, status, srcsize)
	jobid := putToJobQueue(jobtag, tmpdestfilename, status, srcsize)

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		out.Close()
		bl := log.Error(err)
		logq.LogPutqueue(bl)
		dlsize, e := GetFileSize(tmpdestfilename)
		if e != nil {
			l := log.Error(e)
			logq.LogPutqueue(l)
		}
		status = "failed"
		updateJobQueue(jobid, status, dlsize)
		return 0, err
	}
	out.Close()

	status = "downloaded"

	if len(md5str) > 0 {
		bmd5, err := ComputeMd5(tmpdestfilename)
		bmd5str := fmt.Sprintf("%x", bmd5)
		log.Debug("md5", md5str, tmpdestfilename, bmd5str)
		if err != nil {
			log.Error(tmpdestfilename, err, bmd5)
		} else if md5str != bmd5str {
			l := log.Errorf("check md5 code error! src md5:%v,  local md5:%v", md5str, bmd5str)
			logq.LogPutqueue(l)
			status = "md5 error"
			updateJobQueue(jobid, status, 0)
			return n, nil
		}
	}
	log.Printf("%d bytes downloaded.", n)

	if err := MoveFromTmp(tmpdestfilename, destfilename); err != nil {
		status = "MoveFromTmp error"
	}

	dlsize, e := GetFileSize(destfilename)
	if e != nil {
		l := log.Error(e)
		logq.LogPutqueue(l)
	}

	status = datapool.StoreFile(status, destfilename, dpconn, p.Datapool, p.ItemDesc, p.DestName)
	updateJobQueue(jobid, status, dlsize)

	tagComment := GetTagComment(p.Repository, p.Dataitem, p.Tag)

	InsertTagToDb(true, p, tagComment)
	return n, nil
}