//fetch file by shock url
func fetchFile(filename string, url string, token string) (err error) {
fmt.Printf("fetching file name=%s, url=%s\n", filename, url)
localfile, err := os.Create(filename)
if err != nil {
return err
}
defer localfile.Close()
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
//download file from Shock
res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 { //err in fetching data
resbody, _ := ioutil.ReadAll(res.Body)
msg := fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody)
return errors.New(msg)
}
_, err = io.Copy(localfile, res.Body)
if err != nil {
return err
}
return
}
func ShockGet(host string, nodeid string, token string) (node *ShockNode, err error) {
if host == "" || nodeid == "" {
return nil, errors.New("empty shock host or node id")
}
logger.Debug(1, fmt.Sprintf("ShockGet: %s %s %s", host, nodeid, token))
var res *http.Response
shockurl := fmt.Sprintf("%s/node/%s", host, nodeid)
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
c := make(chan int, 1)
go func() {
res, err = httpclient.Get(shockurl, httpclient.Header{}, nil, user)
c <- 1 //we are ending
}()
select {
case <-c:
//go ahead
case <-time.After(conf.SHOCK_TIMEOUT):
return nil, errors.New("timeout when getting node from shock, url=" + shockurl)
}
if err != nil {
return
}
defer res.Body.Close()
jsonstream, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
response := new(ShockResponse)
if err := json.Unmarshal(jsonstream, response); err != nil {
return nil, err
}
if len(response.Errs) > 0 {
return nil, errors.New(strings.Join(response.Errs, ","))
}
node = &response.Data
if node == nil {
err = errors.New("empty node got from Shock")
}
return
}
func FetchShockStream(url string, token string) (r io.ReadCloser, err error) {
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
//download file from Shock
res, err := httpclient.Get(url, httpclient.Header{}, nil, user)
if err != nil {
return nil, err
}
if res.StatusCode != 200 { //err in fetching data
resbody, _ := ioutil.ReadAll(res.Body)
return nil, errors.New(fmt.Sprintf("op=fetchFile, url=%s, res=%s", url, resbody))
}
return res.Body, err
}
//shock access functions
func createOrUpdate(opts Opts, host string, nodeid string, token string) (node *ShockNode, err error) {
url := host + "/node"
method := "POST"
if nodeid != "" {
url += "/" + nodeid
method = "PUT"
}
form := httpclient.NewForm()
if opts.HasKey("attributes") {
form.AddFile("attributes", opts.Value("attributes"))
}
if opts.HasKey("upload_type") {
switch opts.Value("upload_type") {
case "full":
if opts.HasKey("full") {
form.AddFile("upload", opts.Value("full"))
} else {
return nil, errors.New("missing file parameter: upload")
}
case "parts":
if opts.HasKey("parts") {
form.AddParam("parts", opts.Value("parts"))
} else {
return nil, errors.New("missing partial upload parameter: parts")
}
case "part":
if opts.HasKey("part") && opts.HasKey("file") {
form.AddFile(opts.Value("part"), opts.Value("file"))
} else {
return nil, errors.New("missing partial upload parameter: part or file")
}
case "remote_path":
if opts.HasKey("remote_path") {
form.AddParam("path", opts.Value("remote_path"))
} else {
return nil, errors.New("missing remote path parameter: path")
}
case "virtual_file":
if opts.HasKey("virtual_file") {
form.AddParam("type", "virtual")
form.AddParam("source", opts.Value("virtual_file"))
} else {
return nil, errors.New("missing virtual node parameter: source")
}
case "index":
if opts.HasKey("index_type") {
url += "?index=" + opts.Value("index_type")
} else {
return nil, errors.New("missing index type when creating index")
}
}
}
err = form.Create()
if err != nil {
return
}
headers := httpclient.Header{
"Content-Type": form.ContentType,
"Content-Length": strconv.FormatInt(form.Length, 10),
}
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
if res, err := httpclient.Do(method, url, headers, form.Reader, user); err == nil {
defer res.Body.Close()
jsonstream, _ := ioutil.ReadAll(res.Body)
response := new(ShockResponse)
if err := json.Unmarshal(jsonstream, response); err != nil {
return nil, errors.New(fmt.Sprintf("failed to marshal response:\"%s\"", jsonstream))
}
if len(response.Errs) > 0 {
return nil, errors.New(strings.Join(response.Errs, ","))
}
node = &response.Data
} else {
return nil, err
}
return
}
//shock access functions
func createOrUpdate(opts Opts, host string, nodeid string, token string, nodeattr map[string]interface{}) (node *shock.ShockNode, err error) {
url := host + "/node"
method := "POST"
if nodeid != "" {
url += "/" + nodeid
method = "PUT"
}
form := httpclient.NewForm()
if opts.HasKey("attributes") {
form.AddFile("attributes", opts.Value("attributes"))
}
//if opts.HasKey("attributes_str") {
// form.AddParam("attributes_str", opts.Value("attributes_str"))
//}
if len(nodeattr) != 0 {
nodeattr_json, err := json.Marshal(nodeattr)
if err != nil {
return nil, errors.New("error marshalling NodeAttr")
}
form.AddParam("attributes_str", string(nodeattr_json[:]))
}
if opts.HasKey("upload_type") {
switch opts.Value("upload_type") {
case "basic":
if opts.HasKey("file") {
form.AddFile("upload", opts.Value("file"))
}
case "parts":
if opts.HasKey("parts") {
form.AddParam("parts", opts.Value("parts"))
} else {
return nil, errors.New("missing partial upload parameter: parts")
}
if opts.HasKey("file_name") {
form.AddParam("file_name", opts.Value("file_name"))
}
case "part":
if opts.HasKey("part") && opts.HasKey("file") {
form.AddFile(opts.Value("part"), opts.Value("file"))
} else {
return nil, errors.New("missing partial upload parameter: part or file")
}
case "remote_path":
if opts.HasKey("remote_path") {
form.AddParam("path", opts.Value("remote_path"))
} else {
return nil, errors.New("missing remote path parameter: path")
}
case "virtual_file":
if opts.HasKey("virtual_file") {
form.AddParam("type", "virtual")
form.AddParam("source", opts.Value("virtual_file"))
} else {
return nil, errors.New("missing virtual node parameter: source")
}
case "index":
if opts.HasKey("index_type") {
url += "/index/" + opts.Value("index_type")
} else {
return nil, errors.New("missing index type when creating index")
}
case "copy":
if opts.HasKey("parent_node") {
form.AddParam("copy_data", opts.Value("parent_node"))
} else {
return nil, errors.New("missing copy node parameter: parent_node")
}
if opts.HasKey("copy_indexes") {
form.AddParam("copy_indexes", "1")
}
case "subset":
if opts.HasKey("parent_node") && opts.HasKey("parent_index") && opts.HasKey("file") {
form.AddParam("parent_node", opts.Value("parent_node"))
form.AddParam("parent_index", opts.Value("parent_index"))
form.AddFile("subset_indices", opts.Value("file"))
} else {
return nil, errors.New("missing subset node parameter: parent_node or parent_index or file")
}
}
}
err = form.Create()
if err != nil {
return
}
headers := httpclient.Header{
"Content-Type": []string{form.ContentType},
"Content-Length": []string{strconv.FormatInt(form.Length, 10)},
}
var user *httpclient.Auth
if token != "" {
user = httpclient.GetUserByTokenAuth(token)
}
if res, err := httpclient.Do(method, url, headers, form.Reader, user); err == nil {
defer res.Body.Close()
jsonstream, _ := ioutil.ReadAll(res.Body)
response := new(shock.ShockResponse)
if err := json.Unmarshal(jsonstream, response); err != nil {
return nil, errors.New(fmt.Sprintf("failed to marshal response:\"%s\"", jsonstream))
}
if len(response.Errs) > 0 {
return nil, errors.New(strings.Join(response.Errs, ","))
}
node = &response.Data
} else {
return nil, err
}
return
}