Example #1
0
//fetch file by shock url,  TODO remove
func fetchFile_old(filename string, url string, token string) (size int64, err error) {
	fmt.Printf("fetching file name=%s, url=%s\n", filename, url)
	localfile, err := os.Create(filename)
	if err != nil {
		return 0, err
	}
	defer localfile.Close()

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	//download file from Shock
	res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
	if err != nil {
		return 0, err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 { //err in fetching data
		resbody, _ := ioutil.ReadAll(res.Body)
		msg := fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody)
		return 0, errors.New(msg)
	}

	size, err = io.Copy(localfile, res.Body)
	if err != nil {
		return 0, err
	}
	return
}
Example #2
0
func FetchPrivateEnvByWorkId(workid string) (envs map[string]string, err error) {
	targeturl := fmt.Sprintf("%s/work/%s?privateenv&client=%s", conf.SERVER_URL, workid, core.Self.Id)
	var headers httpclient.Header
	if conf.CLIENT_GROUP_TOKEN != "" {
		headers = httpclient.Header{
			"Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN},
		}
	}
	res, err := httpclient.Get(targeturl, headers, nil, nil)
	if err != nil {
		return envs, err
	}
	defer res.Body.Close()
	var jsonstream string
	if res.Header != nil {
		if _, ok := res.Header["Privateenv"]; ok {
			jsonstream = res.Header["Privateenv"][0]
		}
	}
	tmp_map := new(map[string]string)

	if err := json.Unmarshal([]byte(jsonstream), tmp_map); err != nil {
		return nil, err
	}

	envs = make(map[string]string)

	for key, val := range *tmp_map {
		envs[key] = val
	}
	return
}
Example #3
0
func (n *Node) Get() (err error) {
	if n.Id == "" {
		return errors.New("missing node Id")
	}
	url := conf.Server.Url + "/node/" + n.Id
	res, err := client.Get(url, client.Header{}, nil)
	if err == nil {
		if res.StatusCode == 200 {
			r := WNode{Data: n}
			body, _ := ioutil.ReadAll(res.Body)
			if err = json.Unmarshal(body, &r); err == nil {
				return err
			}
		} else {
			r := Wrapper{}
			body, _ := ioutil.ReadAll(res.Body)
			if err = json.Unmarshal(body, &r); err == nil {
				return errors.New(res.Status + ": " + (*r.Error)[0])
			} else {
				return errors.New("request error: " + res.Status)
			}
		}
	}
	return
}
Example #4
0
func (n *Node) Download(opts Opts) (download io.Reader, err error) {
	if n.Id == "" {
		return nil, errors.New("missing node Id")
	}
	url := conf.Server.Url + "/node/" + n.Id + "?download"
	if opts.HasKey("index") {
		url += "&index=" + opts.Value("index")
		if opts.HasKey("parts") {
			url += "&part=" + opts.Value("parts")
		} else {
			return nil, errors.New("missing index parameter: parts")
		}
		if opts.HasKey("index_options") {
			// index options should be in form key=value
			url += "&" + opts.Value("index_options")
		}
	}

	res, err := client.Get(url, client.Header{}, nil)
	if err == nil {
		if res.StatusCode == 200 {
			return res.Body, nil
		} else {
			r := Wrapper{}
			body, _ := ioutil.ReadAll(res.Body)
			if err = json.Unmarshal(body, &r); err == nil {
				return nil, errors.New(res.Status + ": " + (*r.Error)[0])
			} else {
				return nil, errors.New("request error: " + res.Status)
			}
		}
	}
	return nil, err
}
Example #5
0
File: shock.go Project: MG-RAST/AWE
func ShockGet(host string, nodeid string, token string) (node *ShockNode, err error) {
	if host == "" || nodeid == "" {
		return nil, errors.New("empty shock host or node id")
	}
	//logger.Debug(1, fmt.Sprintf("ShockGet: %s %s %s", host, nodeid, token))
	var res *http.Response
	shockurl := fmt.Sprintf("%s/node/%s", host, nodeid)

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	c := make(chan int, 1)
	go func() {
		res, err = httpclient.Get(shockurl, httpclient.Header{}, nil, user)
		c <- 1 //we are ending
	}()
	select {
	case <-c:
	//go ahead
	case <-time.After(SHOCK_TIMEOUT):
		return nil, errors.New("timeout when getting node from shock, url=" + shockurl)
	}
	if err != nil {
		return
	}
	defer res.Body.Close()

	jsonstream, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	response := new(ShockResponse)
	if err := json.Unmarshal(jsonstream, response); err != nil {
		return nil, err
	}
	if len(response.Errs) > 0 {
		return nil, errors.New(strings.Join(response.Errs, ","))
	}
	node = &response.Data
	if node == nil {
		err = errors.New("empty node got from Shock")
	}
	return
}
Example #6
0
func OAuthToken(username, password string) (t *Token, err error) {
	client.SetBasicAuth(username, password)
	res, err := client.Get(conf.Auth.TokenUrl, client.Header{}, nil)
	if err == nil {
		if res.StatusCode == http.StatusCreated {
			if body, err := ioutil.ReadAll(res.Body); err == nil {
				if err = json.Unmarshal(body, &t); err != nil {
					return nil, err
				}
			}
		} else {
			return nil, errors.New("Authentication failed: Unexpected response status: " + res.Status)
		}
	} else {
		return nil, err
	}
	return
}
Example #7
0
func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
	u, _ := url.Parse(urlStr)
	if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
		return "", nil, errors.New("Not a valid url: " + urlStr)
	}
	pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
	fileName := pathParts[len(pathParts)-1]
	cleanPath := strings.Join(pathParts, "/")

	if (u.Scheme == "http") || (u.Scheme == "https") {
		res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
		if err != nil {
			return "", nil, errors.New("httpclient returned: " + err.Error())
		}
		if res.StatusCode != 200 { //err in fetching data
			resbody, _ := ioutil.ReadAll(res.Body)
			return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
		}
		return fileName, res.Body, err
	} else if u.Scheme == "ftp" {
		// set port if missing
		ftpHost := u.Host
		hostParts := strings.Split(u.Host, ":")
		if len(hostParts) == 1 {
			ftpHost = u.Host + ":21"
		}
		c, _, _, err := tinyftp.Dial("tcp", ftpHost)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		defer c.Close()
		if _, _, err = c.Login("", ""); err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		dconn, _, _, err := c.RetrieveFrom(cleanPath)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		return fileName, dconn, err
	}
	return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}
Example #8
0
func FetchDataTokenByWorkId(workid string) (token string, err error) {
	targeturl := fmt.Sprintf("%s/work/%s?datatoken&client=%s", conf.SERVER_URL, workid, core.Self.Id)
	var headers httpclient.Header
	if conf.CLIENT_GROUP_TOKEN != "" {
		headers = httpclient.Header{
			"Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN},
		}
	}
	res, err := httpclient.Get(targeturl, headers, nil, nil)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	if res.Header != nil {
		if _, ok := res.Header["Datatoken"]; ok {
			token = res.Header["Datatoken"][0]
		}
	}
	return
}
Example #9
0
File: shock.go Project: MG-RAST/AWE
func FetchShockStream(url string, token string) (r io.ReadCloser, err error) {

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	//download file from Shock
	res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
	if err != nil {
		//return nil, err
		return nil, errors.New("httpclient.Get returned: " + err.Error())
	}

	if res.StatusCode != 200 { //err in fetching data
		resbody, _ := ioutil.ReadAll(res.Body)
		return nil, errors.New(fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody))
	}

	return res.Body, err
}