示例#1
0
//fetch file by shock url
func fetchFile(filename string, url string, token string) (err error) {
	fmt.Printf("fetching file name=%s, url=%s\n", filename, url)
	localfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer localfile.Close()

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	//download file from Shock
	res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
	if err != nil {
		return err
	}

	defer res.Body.Close()

	if res.StatusCode != 200 { //err in fetching data
		resbody, _ := ioutil.ReadAll(res.Body)
		msg := fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody)
		return errors.New(msg)
	}

	_, err = io.Copy(localfile, res.Body)
	if err != nil {
		return err
	}

	return
}
示例#2
0
文件: shock.go 项目: wtangiit/AWE
func ShockGet(host string, nodeid string, token string) (node *ShockNode, err error) {
	if host == "" || nodeid == "" {
		return nil, errors.New("empty shock host or node id")
	}
	logger.Debug(1, fmt.Sprintf("ShockGet: %s %s %s", host, nodeid, token))
	var res *http.Response
	shockurl := fmt.Sprintf("%s/node/%s", host, nodeid)

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	c := make(chan int, 1)
	go func() {
		res, err = httpclient.Get(shockurl, httpclient.Header{}, nil, user)
		c <- 1 //we are ending
	}()
	select {
	case <-c:
	//go ahead
	case <-time.After(conf.SHOCK_TIMEOUT):
		return nil, errors.New("timeout when getting node from shock, url=" + shockurl)
	}
	if err != nil {
		return
	}
	defer res.Body.Close()

	jsonstream, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	response := new(ShockResponse)
	if err := json.Unmarshal(jsonstream, response); err != nil {
		return nil, err
	}
	if len(response.Errs) > 0 {
		return nil, errors.New(strings.Join(response.Errs, ","))
	}
	node = &response.Data
	if node == nil {
		err = errors.New("empty node got from Shock")
	}
	return
}
示例#3
0
文件: shock.go 项目: wtangiit/AWE
func FetchShockStream(url string, token string) (r io.ReadCloser, err error) {

	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}

	//download file from Shock
	res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
	if err != nil {
		return nil, err
	}

	if res.StatusCode != 200 { //err in fetching data
		resbody, _ := ioutil.ReadAll(res.Body)
		return nil, errors.New(fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody))
	}

	return res.Body, err
}
示例#4
0
文件: core.go 项目: narayandesai/AWE
//shock access functions
func createOrUpdate(opts Opts, host string, nodeid string, token string) (node *ShockNode, err error) {
	url := host + "/node"
	method := "POST"
	if nodeid != "" {
		url += "/" + nodeid
		method = "PUT"
	}
	form := httpclient.NewForm()
	if opts.HasKey("attributes") {
		form.AddFile("attributes", opts.Value("attributes"))
	}
	if opts.HasKey("upload_type") {
		switch opts.Value("upload_type") {
		case "full":
			if opts.HasKey("full") {
				form.AddFile("upload", opts.Value("full"))
			} else {
				return nil, errors.New("missing file parameter: upload")
			}
		case "parts":
			if opts.HasKey("parts") {
				form.AddParam("parts", opts.Value("parts"))
			} else {
				return nil, errors.New("missing partial upload parameter: parts")
			}
		case "part":
			if opts.HasKey("part") && opts.HasKey("file") {
				form.AddFile(opts.Value("part"), opts.Value("file"))
			} else {
				return nil, errors.New("missing partial upload parameter: part or file")
			}
		case "remote_path":
			if opts.HasKey("remote_path") {
				form.AddParam("path", opts.Value("remote_path"))
			} else {
				return nil, errors.New("missing remote path parameter: path")
			}
		case "virtual_file":
			if opts.HasKey("virtual_file") {
				form.AddParam("type", "virtual")
				form.AddParam("source", opts.Value("virtual_file"))
			} else {
				return nil, errors.New("missing virtual node parameter: source")
			}
		case "index":
			if opts.HasKey("index_type") {
				url += "?index=" + opts.Value("index_type")
			} else {
				return nil, errors.New("missing index type when creating index")
			}
		}
	}
	err = form.Create()
	if err != nil {
		return
	}
	headers := httpclient.Header{
		"Content-Type":   form.ContentType,
		"Content-Length": strconv.FormatInt(form.Length, 10),
	}
	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}
	if res, err := httpclient.Do(method, url, headers, form.Reader, user); err == nil {
		defer res.Body.Close()
		jsonstream, _ := ioutil.ReadAll(res.Body)
		response := new(ShockResponse)
		if err := json.Unmarshal(jsonstream, response); err != nil {
			return nil, errors.New(fmt.Sprintf("failed to marshal response:\"%s\"", jsonstream))
		}
		if len(response.Errs) > 0 {
			return nil, errors.New(strings.Join(response.Errs, ","))
		}
		node = &response.Data
	} else {
		return nil, err
	}
	return
}
示例#5
0
文件: core.go 项目: wtangiit/AWE
//shock access functions
func createOrUpdate(opts Opts, host string, nodeid string, token string, nodeattr map[string]interface{}) (node *shock.ShockNode, err error) {
	url := host + "/node"
	method := "POST"
	if nodeid != "" {
		url += "/" + nodeid
		method = "PUT"
	}
	form := httpclient.NewForm()
	if opts.HasKey("attributes") {
		form.AddFile("attributes", opts.Value("attributes"))
	}
	//if opts.HasKey("attributes_str") {
	//	form.AddParam("attributes_str", opts.Value("attributes_str"))
	//}

	if len(nodeattr) != 0 {
		nodeattr_json, err := json.Marshal(nodeattr)
		if err != nil {
			return nil, errors.New("error marshalling NodeAttr")
		}
		form.AddParam("attributes_str", string(nodeattr_json[:]))
	}

	if opts.HasKey("upload_type") {
		switch opts.Value("upload_type") {
		case "basic":
			if opts.HasKey("file") {
				form.AddFile("upload", opts.Value("file"))
			}
		case "parts":
			if opts.HasKey("parts") {
				form.AddParam("parts", opts.Value("parts"))
			} else {
				return nil, errors.New("missing partial upload parameter: parts")
			}
			if opts.HasKey("file_name") {
				form.AddParam("file_name", opts.Value("file_name"))
			}
		case "part":
			if opts.HasKey("part") && opts.HasKey("file") {
				form.AddFile(opts.Value("part"), opts.Value("file"))
			} else {
				return nil, errors.New("missing partial upload parameter: part or file")
			}
		case "remote_path":
			if opts.HasKey("remote_path") {
				form.AddParam("path", opts.Value("remote_path"))
			} else {
				return nil, errors.New("missing remote path parameter: path")
			}
		case "virtual_file":
			if opts.HasKey("virtual_file") {
				form.AddParam("type", "virtual")
				form.AddParam("source", opts.Value("virtual_file"))
			} else {
				return nil, errors.New("missing virtual node parameter: source")
			}
		case "index":
			if opts.HasKey("index_type") {
				url += "/index/" + opts.Value("index_type")
			} else {
				return nil, errors.New("missing index type when creating index")
			}
		case "copy":
			if opts.HasKey("parent_node") {
				form.AddParam("copy_data", opts.Value("parent_node"))
			} else {
				return nil, errors.New("missing copy node parameter: parent_node")
			}
			if opts.HasKey("copy_indexes") {
				form.AddParam("copy_indexes", "1")
			}
		case "subset":
			if opts.HasKey("parent_node") && opts.HasKey("parent_index") && opts.HasKey("file") {
				form.AddParam("parent_node", opts.Value("parent_node"))
				form.AddParam("parent_index", opts.Value("parent_index"))
				form.AddFile("subset_indices", opts.Value("file"))
			} else {
				return nil, errors.New("missing subset node parameter: parent_node or parent_index or file")
			}
		}
	}
	err = form.Create()
	if err != nil {
		return
	}
	headers := httpclient.Header{
		"Content-Type":   []string{form.ContentType},
		"Content-Length": []string{strconv.FormatInt(form.Length, 10)},
	}
	var user *httpclient.Auth
	if token != "" {
		user = httpclient.GetUserByTokenAuth(token)
	}
	if res, err := httpclient.Do(method, url, headers, form.Reader, user); err == nil {
		defer res.Body.Close()
		jsonstream, _ := ioutil.ReadAll(res.Body)
		response := new(shock.ShockResponse)
		if err := json.Unmarshal(jsonstream, response); err != nil {
			return nil, errors.New(fmt.Sprintf("failed to marshal response:\"%s\"", jsonstream))
		}
		if len(response.Errs) > 0 {
			return nil, errors.New(strings.Join(response.Errs, ","))
		}
		node = &response.Data
	} else {
		return nil, err
	}
	return
}