コード例 #1
0
func pullHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	log.Println(r.URL.Path + "(pull)\n")
	result, _ := ioutil.ReadAll(r.Body)
	reqJson := ds.DsPull{}
	var strret string

	if err := json.Unmarshal(result, &reqJson); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}

	reqJson.Repository = ps.ByName("repo")
	reqJson.Dataitem = ps.ByName("item")
	if exist := CheckDataPoolExist(reqJson.Datapool); exist == false {
		strret = reqJson.Datapool + " not found. " + reqJson.Tag + " will be pull into " + g_strDpPath
	} else {
		strret = reqJson.Repository + "/" + reqJson.Dataitem + "/" + reqJson.Tag + " will be pull into " + reqJson.Datapool
	}
	fmt.Println(strret)
	msgret := ds.MsgResp{Msg: strret}
	resp, _ := json.Marshal(msgret)
	w.Write(resp)

	//url := "/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag

	//token, entrypoint, err := getAccessToken(url, w)
	/*if err != nil {
		return
	} else {
		url = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag +
			"?token=" + token + "?username="******"/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag

		token, entrypoint, err := getAccessToken(url, w)
		if err != nil {
			return
		} else {
			url = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag +
				"?token=" + token + "?username="******"/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag
	entrypoint := ""
	go dl(url, entrypoint, reqJson)
	return

}
コード例 #2
0
ファイル: pull_handler.go プロジェクト: asiainfoLDP/datahub
func PullItemAutomatic(Tags map[int]string, v ds.DsPull) {
	var d ds.DsPull = v
	for id, tag := range Tags {
		var chn = make(chan int)
		d.Tag = tag
		d.DestName = d.Tag

		go PullOneTagAutomatic(d, chn)
		<-chn

		UpdateStatMsgTagadded(id, ALREADYREAD)
	}
}
コード例 #3
0
ファイル: pull.go プロジェクト: yepengxj/datahub-notebook
func Pull(login bool, args []string) (err error) {

	if len(args) != 2 {
		fmt.Println("invalid argument..")
		pullUsage()
		return
	}
	u, err := url.Parse(args[0])
	if err != nil {
		panic(err)
	}
	source := u.Path
	if u.Path[0] == '/' {
		source = u.Path[1:]
	}

	var repo, item string
	ds := ds.DsPull{}
	if url := strings.Split(source, "/"); len(url) != 2 {
		fmt.Println("invalid argument..")
		pullUsage()
		return
	} else {
		target := strings.Split(url[1], ":")
		if len(target) == 1 {
			target = append(target, "latest")
		} else if len(target[1]) == 0 {
			target[1] = "latest"
		}
		//uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1])
		repo = url[0]
		item = target[0]
		ds.Tag = target[1]
		ds.DestName = ds.Tag
		ds.Datapool = args[1]
	}

	//fmt.Println("uri:", uri)

	jsonData, err := json.Marshal(ds)
	if err != nil {
		return
	}

	resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData)
	if err != nil {
		fmt.Println(err)
		os.Exit(2)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		//body, _ := ioutil.ReadAll(resp.Body)
		//fmt.Println(body)
		body, _ := ioutil.ReadAll(resp.Body)
		ShowMsgResp(body, true)
		//fmt.Printf("%s/%s:%s will be download to %s\n.", repo, item, ds.Tag, ds.Datapool)

	} else if resp.StatusCode == 401 {
		if err := Login(false, nil); err == nil {
			Pull(login, args)
		} else {
			fmt.Println(err)
			return err
		}
	} else {
		fmt.Println(resp.StatusCode)
	}
	//body, _ := ioutil.ReadAll(resp.Body)
	//fmt.Println(body)

	return nil // dl(uri)
	//return nil
}
コード例 #4
0
ファイル: pull.go プロジェクト: asiainfoLDP/datahub
func Pull(login bool, args []string) (err error) {
	var repo, item string
	dstruc := ds.DsPull{}
	f := mflag.NewFlagSet("pull", mflag.ContinueOnError)
	f.StringVar(&dstruc.DestName, []string{"-destname", "d"}, "", "Indicates the name that tag will be stored as ")
	pbAutomatic := f.Bool([]string{"-automatic", "a"}, false, "Pull the new tags of a dataitem automatically")
	pbCancelAutomatic := f.Bool([]string{"-cancel", "c"}, false, "Cancel the automatical pulling of a dataitem")

	if len(args) < 2 || (len(args) >= 2 && (args[0][0] == '-' || args[1][0] == '-')) {
		//fmt.Println(ErrMsgArgument)
		pullUsage()
		return
	}
	f.Usage = pullUsage
	if err = f.Parse(args[2:]); err != nil {
		fmt.Println(err)
		return err
	}
	u, err := url.Parse(args[0])
	if err != nil {
		fmt.Println(err)
		return
	}
	dstruc.Automatic = *pbAutomatic
	dstruc.CancelAutomatic = *pbCancelAutomatic

	source := strings.Trim(u.Path, "/")

	if url := strings.Split(source, "/"); len(url) != 2 {
		fmt.Println(ErrMsgArgument)
		pullUsage()
		return
	} else {
		target := strings.Split(url[1], ":")
		if len(target) == 1 {
			target = append(target, "latest")
		} else if len(target[1]) == 0 {
			target[1] = "latest"
		}
		//uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1])
		repo = url[0]
		item = target[0]
		dstruc.Tag = target[1]
		if len(dstruc.DestName) == 0 {
			dstruc.DestName = dstruc.Tag
		}
	}

	//get datapool and itemdesc
	if store := strings.Split(strings.Trim(args[1], "/"), "://"); len(store) == 1 {
		dstruc.Datapool = store[0]
		dstruc.ItemDesc = repo + "_" + item
	} else if len(store) == 2 {
		dstruc.Datapool = store[0]
		dstruc.ItemDesc = strings.Trim(store[1], "/")
		if len(dstruc.Datapool) == 0 {
			fmt.Println("Datahub : DATAPOOL://LOCATION are required!")
			pullUsage()
			return
		}
		if len(dstruc.ItemDesc) == 0 {
			dstruc.ItemDesc = repo + "_" + item
		}
	} else {
		fmt.Println("Error : DATAPOOL://LOCATION format error!")
		pullUsage()
		return
	}

	jsonData, err := json.Marshal(dstruc)
	if err != nil {
		fmt.Println("Error")
		return
	}

	resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData)
	if err != nil {
		fmt.Println(err)
		os.Exit(2)
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		//body, _ := ioutil.ReadAll(resp.Body)
		//ShowMsgResp(body, true)
		showResponse(resp)

	} else if resp.StatusCode == http.StatusUnauthorized {
		if err := Login(false, nil); err == nil {
			return Pull(login, args)
		} else {
			fmt.Println(err)
			return err
		}
	} else {
		result := ds.Response{}
		respbody, _ := ioutil.ReadAll(resp.Body)
		//fmt.Println(string(respbody))
		unmarshalerr := json.Unmarshal(respbody, &result)
		if unmarshalerr != nil {
			fmt.Println("Error : Pull error.", unmarshalerr)
			return unmarshalerr
		}
		if result.Code == ServerErrResultCode5009 {
			fmt.Println("DataHub : Failed to get subscription")
		} else if result.Code == ServerErrResultCode5012 {
			fmt.Println("DataHub : Permission denied,you have not subscribed current dataitem yet.")
		} else if result.Code == ServerErrResultCode5023 {
			fmt.Println("DataHub : Currently the data is unavaliable.")
		} else {
			//showError(resp)
			fmt.Println("Error : ", result.Msg)
		}
	}
	//showError(resp)

	return nil
}
コード例 #5
0
/*download routine, supports resuming broken downloads.*/
func download(url string, p ds.DsPull) (int64, error) {
	log.Printf("we are going to download %s, save to dp=%s,name=%s\n", url, p.Datapool, p.DestName)

	var out *os.File
	var err error
	var destfilename string
	dpexist := CheckDataPoolExist(p.Datapool)
	if dpexist == false {
		destfilename = g_strDpPath + p.DestName
	} else {
		dpconn := GetDataPoolDpconn(p.Datapool)
		if len(dpconn) == 0 {
			destfilename = g_strDpPath + p.DestName
		} else {
			os.MkdirAll(dpconn+"/"+p.Datapool+"/"+p.Repository+"/"+p.Dataitem, 0755)
			destfilename = dpconn + "/" + p.Datapool + "/" + p.Repository + "/" + p.Dataitem + "/" + p.DestName
		}
	}
	fmt.Println(destfilename)
	out, err = os.OpenFile(destfilename, os.O_RDWR|os.O_CREATE, 0644)

	if err != nil {
		return 0, err
	}

	stat, err := out.Stat()
	if err != nil {
		out.Close()
		return 0, err
	}
	out.Seek(stat.Size(), 0)
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "go-downloader")
	/* Set download starting position with 'Range' in HTTP header*/
	req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
	log.Printf("%v bytes had already been downloaded.\n", stat.Size())

	resp, err := http.DefaultClient.Do(req)

	/*Save response body to file only when HTTP 2xx received. TODO*/
	if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
		if resp != nil {
			log.Println("http status code:", resp.StatusCode, err)
			body, _ := ioutil.ReadAll(resp.Body)
			log.Println("response Body:", string(body))
		}
		filesize := stat.Size()
		out.Close()
		if filesize == 0 {
			os.Remove(destfilename)
		}
		return 0, err
	}
	defer resp.Body.Close()
	fname := resp.Header.Get("Source-Filename")
	if len(fname) > 0 {
		p.DestName = fname
	}

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		out.Close()
		return 0, err
	}
	out.Close()
	log.Printf("%d bytes downloaded.", n)
	InsertTagToDb(dpexist, p)
	return n, nil
}
コード例 #6
0
ファイル: pull_handler.go プロジェクト: asiainfoLDP/datahub
func pullHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	MetaFile := "meta.txt"
	SampleFile := "sample.txt"
	log.Println(r.URL.Path + "(pull)")
	result, _ := ioutil.ReadAll(r.Body)
	p := ds.DsPull{}
	p.ItemDesc = strings.Trim(p.ItemDesc, "/")
	if strings.Contains(p.ItemDesc, "/") == true {
		log.Println("The path of item can't contain '/'.", p.ItemDesc)
		w.Write([]byte(`{"msg":"The path of item can't contain '/'."}`))
		return
	}

	if err := json.Unmarshal(result, &p); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}

	p.Repository = ps.ByName("repo")
	p.Dataitem = ps.ByName("item")

	dpexist := CheckDataPoolExist(p.Datapool)
	if dpexist == false {
		e := fmt.Sprintf("Datapool '%s' does not exist.", p.Datapool)
		l := log.Error("Code:", cmd.ErrorDatapoolNotExits, e)
		logq.LogPutqueue(l)
		msgret := ds.Result{Code: cmd.ErrorDatapoolNotExits, Msg: e}
		resp, _ := json.Marshal(msgret)
		w.WriteHeader(http.StatusBadRequest)
		w.Write(resp)
		return
	}

	alItemdesc, err := GetItemDesc(p.Repository, p.Dataitem)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
	if len(alItemdesc) != 0 && p.ItemDesc != alItemdesc {
		p.ItemDesc = alItemdesc
		//TODO add log tishi
	} else if len(p.ItemDesc) == 0 && len(alItemdesc) == 0 {
		p.ItemDesc = p.Repository + "_" + p.Dataitem
	}

	if dpconn := GetDataPoolDpconn(p.Datapool); len(dpconn) == 0 {
		strret = p.Datapool + " not found. " + p.Tag + " will be pulled into " + g_strDpPath + "/" + p.ItemDesc
	} else {
		if len(p.DestName) == 0 {
			strret = p.Repository + "/" + p.Dataitem + ":" + p.Tag + " will be pulled soon and can be found as " +
				strings.Split(dpconn, "##")[0] + "/" + p.ItemDesc + "/" + p.Tag
		} else {
			strret = p.Repository + "/" + p.Dataitem + ":" + p.Tag + " will be pulled soon and can be found as " +
				strings.Split(dpconn, "##")[0] + "/" + p.ItemDesc + "/" + p.DestName
		}
	}

	//add to automatic pull list
	if p.Automatic == true {
		if true == CheckExistInQueue(p) {
			strret = p.Repository + "/" + p.Dataitem + " is being pulled automatically."
		} else {
			AutomaticPullPutqueue(p)
			strret = p.Repository + "/" + p.Dataitem + " will be pulled automatically."
		}

		msgret := ds.MsgResp{Msg: strret}
		resp, _ := json.Marshal(msgret)
		w.WriteHeader(http.StatusOK)
		w.Write(resp)
		return
	}
	if p.CancelAutomatic == true {

		if true == AutomaticPullRmqueue(p) {
			strret = "Cancel the automatical pulling of " + p.Repository + "/" + p.Dataitem + " successfully."
		} else {
			strret = "you have already cancelled the automatical pulling of " + p.Repository + "/" + p.Dataitem
		}
		msgret := ds.MsgResp{Msg: strret}
		resp, _ := json.Marshal(msgret)
		w.WriteHeader(http.StatusOK)
		w.Write(resp)
		return
	}

	uri := "/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + p.Tag

	token, entrypoint, err := getAccessToken(uri, w)
	if err != nil {
		log.Println(err)
		strret = err.Error()
		return
	} else {
		uri = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + p.Tag +
			"?token=" + token + "&username="******"Metafile and Samplefile is already exist")
		} else {
			dpconn := GetDataPoolDpconn(p.Datapool)
			os.MkdirAll(dpconn+"/"+p.ItemDesc+"/", 0777)

			m, err := os.OpenFile(dpconn+"/"+p.ItemDesc+"/"+MetaFile, os.O_RDWR|os.O_CREATE, 0644)

			if err != nil {
				return
			}
			defer m.Close()
			s, err := os.OpenFile(dpconn+"/"+p.ItemDesc+"/"+SampleFile, os.O_RDWR|os.O_CREATE, 0644)

			if err != nil {
				return
			}
			defer s.Close()
			path := "/api/repositories/" + ps.ByName("repo") + "/" + ps.ByName("item")

			resp, err := commToServerGetRsp("get", path, nil)
			if err != nil {
				log.Error(err)
				return
			}

			body, err := ioutil.ReadAll(resp.Body)

			ms := ds.ItemMs{}
			retResp := ds.Response{Data: &ms}
			json.Unmarshal(body, &retResp)

			m.WriteString(ms.Meta)
			s.WriteString(ms.Sample)

			defer resp.Body.Close()

		}
	}
	log.Println(strret)

	return
}