func pullHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
log.Println(r.URL.Path + "(pull)\n")
result, _ := ioutil.ReadAll(r.Body)
reqJson := ds.DsPull{}
var strret string
if err := json.Unmarshal(result, &reqJson); err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
reqJson.Repository = ps.ByName("repo")
reqJson.Dataitem = ps.ByName("item")
if exist := CheckDataPoolExist(reqJson.Datapool); exist == false {
strret = reqJson.Datapool + " not found. " + reqJson.Tag + " will be pull into " + g_strDpPath
} else {
strret = reqJson.Repository + "/" + reqJson.Dataitem + "/" + reqJson.Tag + " will be pull into " + reqJson.Datapool
}
fmt.Println(strret)
msgret := ds.MsgResp{Msg: strret}
resp, _ := json.Marshal(msgret)
w.Write(resp)
//url := "/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag
//token, entrypoint, err := getAccessToken(url, w)
/*if err != nil {
return
} else {
url = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag +
"?token=" + token + "?username="******"/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag
token, entrypoint, err := getAccessToken(url, w)
if err != nil {
return
} else {
url = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag +
"?token=" + token + "?username="******"/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + reqJson.Tag
entrypoint := ""
go dl(url, entrypoint, reqJson)
return
}
func PullItemAutomatic(Tags map[int]string, v ds.DsPull) {
var d ds.DsPull = v
for id, tag := range Tags {
var chn = make(chan int)
d.Tag = tag
d.DestName = d.Tag
go PullOneTagAutomatic(d, chn)
<-chn
UpdateStatMsgTagadded(id, ALREADYREAD)
}
}
func Pull(login bool, args []string) (err error) {
if len(args) != 2 {
fmt.Println("invalid argument..")
pullUsage()
return
}
u, err := url.Parse(args[0])
if err != nil {
panic(err)
}
source := u.Path
if u.Path[0] == '/' {
source = u.Path[1:]
}
var repo, item string
ds := ds.DsPull{}
if url := strings.Split(source, "/"); len(url) != 2 {
fmt.Println("invalid argument..")
pullUsage()
return
} else {
target := strings.Split(url[1], ":")
if len(target) == 1 {
target = append(target, "latest")
} else if len(target[1]) == 0 {
target[1] = "latest"
}
//uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1])
repo = url[0]
item = target[0]
ds.Tag = target[1]
ds.DestName = ds.Tag
ds.Datapool = args[1]
}
//fmt.Println("uri:", uri)
jsonData, err := json.Marshal(ds)
if err != nil {
return
}
resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData)
if err != nil {
fmt.Println(err)
os.Exit(2)
}
defer resp.Body.Close()
if resp.StatusCode == 200 {
//body, _ := ioutil.ReadAll(resp.Body)
//fmt.Println(body)
body, _ := ioutil.ReadAll(resp.Body)
ShowMsgResp(body, true)
//fmt.Printf("%s/%s:%s will be download to %s\n.", repo, item, ds.Tag, ds.Datapool)
} else if resp.StatusCode == 401 {
if err := Login(false, nil); err == nil {
Pull(login, args)
} else {
fmt.Println(err)
return err
}
} else {
fmt.Println(resp.StatusCode)
}
//body, _ := ioutil.ReadAll(resp.Body)
//fmt.Println(body)
return nil // dl(uri)
//return nil
}
func Pull(login bool, args []string) (err error) {
var repo, item string
dstruc := ds.DsPull{}
f := mflag.NewFlagSet("pull", mflag.ContinueOnError)
f.StringVar(&dstruc.DestName, []string{"-destname", "d"}, "", "Indicates the name that tag will be stored as ")
pbAutomatic := f.Bool([]string{"-automatic", "a"}, false, "Pull the new tags of a dataitem automatically")
pbCancelAutomatic := f.Bool([]string{"-cancel", "c"}, false, "Cancel the automatical pulling of a dataitem")
if len(args) < 2 || (len(args) >= 2 && (args[0][0] == '-' || args[1][0] == '-')) {
//fmt.Println(ErrMsgArgument)
pullUsage()
return
}
f.Usage = pullUsage
if err = f.Parse(args[2:]); err != nil {
fmt.Println(err)
return err
}
u, err := url.Parse(args[0])
if err != nil {
fmt.Println(err)
return
}
dstruc.Automatic = *pbAutomatic
dstruc.CancelAutomatic = *pbCancelAutomatic
source := strings.Trim(u.Path, "/")
if url := strings.Split(source, "/"); len(url) != 2 {
fmt.Println(ErrMsgArgument)
pullUsage()
return
} else {
target := strings.Split(url[1], ":")
if len(target) == 1 {
target = append(target, "latest")
} else if len(target[1]) == 0 {
target[1] = "latest"
}
//uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1])
repo = url[0]
item = target[0]
dstruc.Tag = target[1]
if len(dstruc.DestName) == 0 {
dstruc.DestName = dstruc.Tag
}
}
//get datapool and itemdesc
if store := strings.Split(strings.Trim(args[1], "/"), "://"); len(store) == 1 {
dstruc.Datapool = store[0]
dstruc.ItemDesc = repo + "_" + item
} else if len(store) == 2 {
dstruc.Datapool = store[0]
dstruc.ItemDesc = strings.Trim(store[1], "/")
if len(dstruc.Datapool) == 0 {
fmt.Println("Datahub : DATAPOOL://LOCATION are required!")
pullUsage()
return
}
if len(dstruc.ItemDesc) == 0 {
dstruc.ItemDesc = repo + "_" + item
}
} else {
fmt.Println("Error : DATAPOOL://LOCATION format error!")
pullUsage()
return
}
jsonData, err := json.Marshal(dstruc)
if err != nil {
fmt.Println("Error")
return
}
resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData)
if err != nil {
fmt.Println(err)
os.Exit(2)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
//body, _ := ioutil.ReadAll(resp.Body)
//ShowMsgResp(body, true)
showResponse(resp)
} else if resp.StatusCode == http.StatusUnauthorized {
if err := Login(false, nil); err == nil {
return Pull(login, args)
} else {
fmt.Println(err)
return err
}
} else {
result := ds.Response{}
respbody, _ := ioutil.ReadAll(resp.Body)
//fmt.Println(string(respbody))
unmarshalerr := json.Unmarshal(respbody, &result)
if unmarshalerr != nil {
fmt.Println("Error : Pull error.", unmarshalerr)
return unmarshalerr
}
if result.Code == ServerErrResultCode5009 {
fmt.Println("DataHub : Failed to get subscription")
} else if result.Code == ServerErrResultCode5012 {
fmt.Println("DataHub : Permission denied,you have not subscribed current dataitem yet.")
} else if result.Code == ServerErrResultCode5023 {
fmt.Println("DataHub : Currently the data is unavaliable.")
} else {
//showError(resp)
fmt.Println("Error : ", result.Msg)
}
}
//showError(resp)
return nil
}
/*download routine, supports resuming broken downloads.*/
func download(url string, p ds.DsPull) (int64, error) {
log.Printf("we are going to download %s, save to dp=%s,name=%s\n", url, p.Datapool, p.DestName)
var out *os.File
var err error
var destfilename string
dpexist := CheckDataPoolExist(p.Datapool)
if dpexist == false {
destfilename = g_strDpPath + p.DestName
} else {
dpconn := GetDataPoolDpconn(p.Datapool)
if len(dpconn) == 0 {
destfilename = g_strDpPath + p.DestName
} else {
os.MkdirAll(dpconn+"/"+p.Datapool+"/"+p.Repository+"/"+p.Dataitem, 0755)
destfilename = dpconn + "/" + p.Datapool + "/" + p.Repository + "/" + p.Dataitem + "/" + p.DestName
}
}
fmt.Println(destfilename)
out, err = os.OpenFile(destfilename, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return 0, err
}
stat, err := out.Stat()
if err != nil {
out.Close()
return 0, err
}
out.Seek(stat.Size(), 0)
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "go-downloader")
/* Set download starting position with 'Range' in HTTP header*/
req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
log.Printf("%v bytes had already been downloaded.\n", stat.Size())
resp, err := http.DefaultClient.Do(req)
/*Save response body to file only when HTTP 2xx received. TODO*/
if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
if resp != nil {
log.Println("http status code:", resp.StatusCode, err)
body, _ := ioutil.ReadAll(resp.Body)
log.Println("response Body:", string(body))
}
filesize := stat.Size()
out.Close()
if filesize == 0 {
os.Remove(destfilename)
}
return 0, err
}
defer resp.Body.Close()
fname := resp.Header.Get("Source-Filename")
if len(fname) > 0 {
p.DestName = fname
}
n, err := io.Copy(out, resp.Body)
if err != nil {
out.Close()
return 0, err
}
out.Close()
log.Printf("%d bytes downloaded.", n)
InsertTagToDb(dpexist, p)
return n, nil
}
func pullHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
MetaFile := "meta.txt"
SampleFile := "sample.txt"
log.Println(r.URL.Path + "(pull)")
result, _ := ioutil.ReadAll(r.Body)
p := ds.DsPull{}
p.ItemDesc = strings.Trim(p.ItemDesc, "/")
if strings.Contains(p.ItemDesc, "/") == true {
log.Println("The path of item can't contain '/'.", p.ItemDesc)
w.Write([]byte(`{"msg":"The path of item can't contain '/'."}`))
return
}
if err := json.Unmarshal(result, &p); err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
p.Repository = ps.ByName("repo")
p.Dataitem = ps.ByName("item")
dpexist := CheckDataPoolExist(p.Datapool)
if dpexist == false {
e := fmt.Sprintf("Datapool '%s' does not exist.", p.Datapool)
l := log.Error("Code:", cmd.ErrorDatapoolNotExits, e)
logq.LogPutqueue(l)
msgret := ds.Result{Code: cmd.ErrorDatapoolNotExits, Msg: e}
resp, _ := json.Marshal(msgret)
w.WriteHeader(http.StatusBadRequest)
w.Write(resp)
return
}
alItemdesc, err := GetItemDesc(p.Repository, p.Dataitem)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
if len(alItemdesc) != 0 && p.ItemDesc != alItemdesc {
p.ItemDesc = alItemdesc
//TODO add log tishi
} else if len(p.ItemDesc) == 0 && len(alItemdesc) == 0 {
p.ItemDesc = p.Repository + "_" + p.Dataitem
}
if dpconn := GetDataPoolDpconn(p.Datapool); len(dpconn) == 0 {
strret = p.Datapool + " not found. " + p.Tag + " will be pulled into " + g_strDpPath + "/" + p.ItemDesc
} else {
if len(p.DestName) == 0 {
strret = p.Repository + "/" + p.Dataitem + ":" + p.Tag + " will be pulled soon and can be found as " +
strings.Split(dpconn, "##")[0] + "/" + p.ItemDesc + "/" + p.Tag
} else {
strret = p.Repository + "/" + p.Dataitem + ":" + p.Tag + " will be pulled soon and can be found as " +
strings.Split(dpconn, "##")[0] + "/" + p.ItemDesc + "/" + p.DestName
}
}
//add to automatic pull list
if p.Automatic == true {
if true == CheckExistInQueue(p) {
strret = p.Repository + "/" + p.Dataitem + " is being pulled automatically."
} else {
AutomaticPullPutqueue(p)
strret = p.Repository + "/" + p.Dataitem + " will be pulled automatically."
}
msgret := ds.MsgResp{Msg: strret}
resp, _ := json.Marshal(msgret)
w.WriteHeader(http.StatusOK)
w.Write(resp)
return
}
if p.CancelAutomatic == true {
if true == AutomaticPullRmqueue(p) {
strret = "Cancel the automatical pulling of " + p.Repository + "/" + p.Dataitem + " successfully."
} else {
strret = "you have already cancelled the automatical pulling of " + p.Repository + "/" + p.Dataitem
}
msgret := ds.MsgResp{Msg: strret}
resp, _ := json.Marshal(msgret)
w.WriteHeader(http.StatusOK)
w.Write(resp)
return
}
uri := "/transaction/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + p.Tag
token, entrypoint, err := getAccessToken(uri, w)
if err != nil {
log.Println(err)
strret = err.Error()
return
} else {
uri = "/pull/" + ps.ByName("repo") + "/" + ps.ByName("item") + "/" + p.Tag +
"?token=" + token + "&username="******"Metafile and Samplefile is already exist")
} else {
dpconn := GetDataPoolDpconn(p.Datapool)
os.MkdirAll(dpconn+"/"+p.ItemDesc+"/", 0777)
m, err := os.OpenFile(dpconn+"/"+p.ItemDesc+"/"+MetaFile, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return
}
defer m.Close()
s, err := os.OpenFile(dpconn+"/"+p.ItemDesc+"/"+SampleFile, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return
}
defer s.Close()
path := "/api/repositories/" + ps.ByName("repo") + "/" + ps.ByName("item")
resp, err := commToServerGetRsp("get", path, nil)
if err != nil {
log.Error(err)
return
}
body, err := ioutil.ReadAll(resp.Body)
ms := ds.ItemMs{}
retResp := ds.Response{Data: &ms}
json.Unmarshal(body, &retResp)
m.WriteString(ms.Meta)
s.WriteString(ms.Sample)
defer resp.Body.Close()
}
}
log.Println(strret)
return
}