//fetch file by shock url, TODO remove
func fetchFile_old(filename string, url string, token string) (size int64, err error) {
fmt.Printf("fetching file name=%s, url=%s\n", filename, url)
localfile, err := os.Create(filename)
if err != nil {
return 0, err
}
defer localfile.Close()
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
//download file from Shock
res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
if err != nil {
return 0, err
}
defer res.Body.Close()
if res.StatusCode != 200 { //err in fetching data
resbody, _ := ioutil.ReadAll(res.Body)
msg := fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody)
return 0, errors.New(msg)
}
size, err = io.Copy(localfile, res.Body)
if err != nil {
return 0, err
}
return
}
func FetchPrivateEnvByWorkId(workid string) (envs map[string]string, err error) {
targeturl := fmt.Sprintf("%s/work/%s?privateenv&client=%s", conf.SERVER_URL, workid, core.Self.Id)
var headers httpclient.Header
if conf.CLIENT_GROUP_TOKEN != "" {
headers = httpclient.Header{
"Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN},
}
}
res, err := httpclient.Get(targeturl, headers, nil, nil)
if err != nil {
return envs, err
}
defer res.Body.Close()
var jsonstream string
if res.Header != nil {
if _, ok := res.Header["Privateenv"]; ok {
jsonstream = res.Header["Privateenv"][0]
}
}
tmp_map := new(map[string]string)
if err := json.Unmarshal([]byte(jsonstream), tmp_map); err != nil {
return nil, err
}
envs = make(map[string]string)
for key, val := range *tmp_map {
envs[key] = val
}
return
}
func (n *Node) Get() (err error) {
if n.Id == "" {
return errors.New("missing node Id")
}
url := conf.Server.Url + "/node/" + n.Id
res, err := client.Get(url, client.Header{}, nil)
if err == nil {
if res.StatusCode == 200 {
r := WNode{Data: n}
body, _ := ioutil.ReadAll(res.Body)
if err = json.Unmarshal(body, &r); err == nil {
return err
}
} else {
r := Wrapper{}
body, _ := ioutil.ReadAll(res.Body)
if err = json.Unmarshal(body, &r); err == nil {
return errors.New(res.Status + ": " + (*r.Error)[0])
} else {
return errors.New("request error: " + res.Status)
}
}
}
return
}
func (n *Node) Download(opts Opts) (download io.Reader, err error) {
if n.Id == "" {
return nil, errors.New("missing node Id")
}
url := conf.Server.Url + "/node/" + n.Id + "?download"
if opts.HasKey("index") {
url += "&index=" + opts.Value("index")
if opts.HasKey("parts") {
url += "&part=" + opts.Value("parts")
} else {
return nil, errors.New("missing index parameter: parts")
}
if opts.HasKey("index_options") {
// index options should be in form key=value
url += "&" + opts.Value("index_options")
}
}
res, err := client.Get(url, client.Header{}, nil)
if err == nil {
if res.StatusCode == 200 {
return res.Body, nil
} else {
r := Wrapper{}
body, _ := ioutil.ReadAll(res.Body)
if err = json.Unmarshal(body, &r); err == nil {
return nil, errors.New(res.Status + ": " + (*r.Error)[0])
} else {
return nil, errors.New("request error: " + res.Status)
}
}
}
return nil, err
}
func ShockGet(host string, nodeid string, token string) (node *ShockNode, err error) {
if host == "" || nodeid == "" {
return nil, errors.New("empty shock host or node id")
}
//logger.Debug(1, fmt.Sprintf("ShockGet: %s %s %s", host, nodeid, token))
var res *http.Response
shockurl := fmt.Sprintf("%s/node/%s", host, nodeid)
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
c := make(chan int, 1)
go func() {
res, err = httpclient.Get(shockurl, httpclient.Header{}, nil, user)
c <- 1 //we are ending
}()
select {
case <-c:
//go ahead
case <-time.After(SHOCK_TIMEOUT):
return nil, errors.New("timeout when getting node from shock, url=" + shockurl)
}
if err != nil {
return
}
defer res.Body.Close()
jsonstream, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
response := new(ShockResponse)
if err := json.Unmarshal(jsonstream, response); err != nil {
return nil, err
}
if len(response.Errs) > 0 {
return nil, errors.New(strings.Join(response.Errs, ","))
}
node = &response.Data
if node == nil {
err = errors.New("empty node got from Shock")
}
return
}
func OAuthToken(username, password string) (t *Token, err error) {
client.SetBasicAuth(username, password)
res, err := client.Get(conf.Auth.TokenUrl, client.Header{}, nil)
if err == nil {
if res.StatusCode == http.StatusCreated {
if body, err := ioutil.ReadAll(res.Body); err == nil {
if err = json.Unmarshal(body, &t); err != nil {
return nil, err
}
}
} else {
return nil, errors.New("Authentication failed: Unexpected response status: " + res.Status)
}
} else {
return nil, err
}
return
}
func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
u, _ := url.Parse(urlStr)
if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
return "", nil, errors.New("Not a valid url: " + urlStr)
}
pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
fileName := pathParts[len(pathParts)-1]
cleanPath := strings.Join(pathParts, "/")
if (u.Scheme == "http") || (u.Scheme == "https") {
res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
if err != nil {
return "", nil, errors.New("httpclient returned: " + err.Error())
}
if res.StatusCode != 200 { //err in fetching data
resbody, _ := ioutil.ReadAll(res.Body)
return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
}
return fileName, res.Body, err
} else if u.Scheme == "ftp" {
// set port if missing
ftpHost := u.Host
hostParts := strings.Split(u.Host, ":")
if len(hostParts) == 1 {
ftpHost = u.Host + ":21"
}
c, _, _, err := tinyftp.Dial("tcp", ftpHost)
if err != nil {
return "", nil, errors.New("ftpclient returned: " + err.Error())
}
defer c.Close()
if _, _, err = c.Login("", ""); err != nil {
return "", nil, errors.New("ftpclient returned: " + err.Error())
}
dconn, _, _, err := c.RetrieveFrom(cleanPath)
if err != nil {
return "", nil, errors.New("ftpclient returned: " + err.Error())
}
return fileName, dconn, err
}
return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}
func FetchDataTokenByWorkId(workid string) (token string, err error) {
targeturl := fmt.Sprintf("%s/work/%s?datatoken&client=%s", conf.SERVER_URL, workid, core.Self.Id)
var headers httpclient.Header
if conf.CLIENT_GROUP_TOKEN != "" {
headers = httpclient.Header{
"Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN},
}
}
res, err := httpclient.Get(targeturl, headers, nil, nil)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.Header != nil {
if _, ok := res.Header["Datatoken"]; ok {
token = res.Header["Datatoken"][0]
}
}
return
}
func FetchShockStream(url string, token string) (r io.ReadCloser, err error) {
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
//download file from Shock
res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
if err != nil {
//return nil, err
return nil, errors.New("httpclient.Get returned: " + err.Error())
}
if res.StatusCode != 200 { //err in fetching data
resbody, _ := ioutil.ReadAll(res.Body)
return nil, errors.New(fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody))
}
return res.Body, err
}