func PullItemAutomatic(Tags map[int]string, v ds.DsPull) { var d ds.DsPull = v for id, tag := range Tags { var chn = make(chan int) d.Tag = tag d.DestName = d.Tag go PullOneTagAutomatic(d, chn) <-chn UpdateStatMsgTagadded(id, ALREADYREAD) } }
func Pull(login bool, args []string) (err error) { if len(args) != 2 { fmt.Println("invalid argument..") pullUsage() return } u, err := url.Parse(args[0]) if err != nil { panic(err) } source := u.Path if u.Path[0] == '/' { source = u.Path[1:] } var repo, item string ds := ds.DsPull{} if url := strings.Split(source, "/"); len(url) != 2 { fmt.Println("invalid argument..") pullUsage() return } else { target := strings.Split(url[1], ":") if len(target) == 1 { target = append(target, "latest") } else if len(target[1]) == 0 { target[1] = "latest" } //uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1]) repo = url[0] item = target[0] ds.Tag = target[1] ds.DestName = ds.Tag ds.Datapool = args[1] } //fmt.Println("uri:", uri) jsonData, err := json.Marshal(ds) if err != nil { return } resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData) if err != nil { fmt.Println(err) os.Exit(2) } defer resp.Body.Close() if resp.StatusCode == 200 { //body, _ := ioutil.ReadAll(resp.Body) //fmt.Println(body) body, _ := ioutil.ReadAll(resp.Body) ShowMsgResp(body, true) //fmt.Printf("%s/%s:%s will be download to %s\n.", repo, item, ds.Tag, ds.Datapool) } else if resp.StatusCode == 401 { if err := Login(false, nil); err == nil { Pull(login, args) } else { fmt.Println(err) return err } } else { fmt.Println(resp.StatusCode) } //body, _ := ioutil.ReadAll(resp.Body) //fmt.Println(body) return nil // dl(uri) //return nil }
func Pull(login bool, args []string) (err error) { var repo, item string dstruc := ds.DsPull{} f := mflag.NewFlagSet("pull", mflag.ContinueOnError) f.StringVar(&dstruc.DestName, []string{"-destname", "d"}, "", "Indicates the name that tag will be stored as ") pbAutomatic := f.Bool([]string{"-automatic", "a"}, false, "Pull the new tags of a dataitem automatically") pbCancelAutomatic := f.Bool([]string{"-cancel", "c"}, false, "Cancel the automatical pulling of a dataitem") if len(args) < 2 || (len(args) >= 2 && (args[0][0] == '-' || args[1][0] == '-')) { //fmt.Println(ErrMsgArgument) pullUsage() return } f.Usage = pullUsage if err = f.Parse(args[2:]); err != nil { fmt.Println(err) return err } u, err := url.Parse(args[0]) if err != nil { fmt.Println(err) return } dstruc.Automatic = *pbAutomatic dstruc.CancelAutomatic = *pbCancelAutomatic source := strings.Trim(u.Path, "/") if url := strings.Split(source, "/"); len(url) != 2 { fmt.Println(ErrMsgArgument) pullUsage() return } else { target := strings.Split(url[1], ":") if len(target) == 1 { target = append(target, "latest") } else if len(target[1]) == 0 { target[1] = "latest" } //uri = fmt.Sprintf("%s/%s:%s", url[0], target[0], target[1]) repo = url[0] item = target[0] dstruc.Tag = target[1] if len(dstruc.DestName) == 0 { dstruc.DestName = dstruc.Tag } } //get datapool and itemdesc if store := strings.Split(strings.Trim(args[1], "/"), "://"); len(store) == 1 { dstruc.Datapool = store[0] dstruc.ItemDesc = repo + "_" + item } else if len(store) == 2 { dstruc.Datapool = store[0] dstruc.ItemDesc = strings.Trim(store[1], "/") if len(dstruc.Datapool) == 0 { fmt.Println("Datahub : DATAPOOL://LOCATION are required!") pullUsage() return } if len(dstruc.ItemDesc) == 0 { dstruc.ItemDesc = repo + "_" + item } } else { fmt.Println("Error : DATAPOOL://LOCATION format error!") pullUsage() return } jsonData, err := json.Marshal(dstruc) if err != nil { fmt.Println("Error") return } resp, err := commToDaemon("post", "/subscriptions/"+repo+"/"+item+"/pull", jsonData) if err != nil { fmt.Println(err) os.Exit(2) } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { //body, _ := ioutil.ReadAll(resp.Body) //ShowMsgResp(body, true) showResponse(resp) } else if resp.StatusCode == http.StatusUnauthorized { if err := Login(false, nil); err == nil { return Pull(login, args) } else { fmt.Println(err) return err } } else { result := ds.Response{} respbody, _ := ioutil.ReadAll(resp.Body) //fmt.Println(string(respbody)) unmarshalerr := json.Unmarshal(respbody, &result) if unmarshalerr != nil { fmt.Println("Error : Pull error.", unmarshalerr) return unmarshalerr } if result.Code == ServerErrResultCode5009 { fmt.Println("DataHub : Failed to get subscription") } else if result.Code == ServerErrResultCode5012 { fmt.Println("DataHub : Permission denied,you have not subscribed current dataitem yet.") } else if result.Code == ServerErrResultCode5023 { fmt.Println("DataHub : Currently the data is unavaliable.") } else { //showError(resp) fmt.Println("Error : ", result.Msg) } } //showError(resp) return nil }
/*download routine, supports resuming broken downloads.*/ func download(url string, p ds.DsPull) (int64, error) { log.Printf("we are going to download %s, save to dp=%s,name=%s\n", url, p.Datapool, p.DestName) var out *os.File var err error var destfilename string dpexist := CheckDataPoolExist(p.Datapool) if dpexist == false { destfilename = g_strDpPath + p.DestName } else { dpconn := GetDataPoolDpconn(p.Datapool) if len(dpconn) == 0 { destfilename = g_strDpPath + p.DestName } else { os.MkdirAll(dpconn+"/"+p.Datapool+"/"+p.Repository+"/"+p.Dataitem, 0755) destfilename = dpconn + "/" + p.Datapool + "/" + p.Repository + "/" + p.Dataitem + "/" + p.DestName } } fmt.Println(destfilename) out, err = os.OpenFile(destfilename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return 0, err } stat, err := out.Stat() if err != nil { out.Close() return 0, err } out.Seek(stat.Size(), 0) req, err := http.NewRequest("GET", url, nil) req.Header.Set("User-Agent", "go-downloader") /* Set download starting position with 'Range' in HTTP header*/ req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-") log.Printf("%v bytes had already been downloaded.\n", stat.Size()) resp, err := http.DefaultClient.Do(req) /*Save response body to file only when HTTP 2xx received. TODO*/ if err != nil || (resp != nil && resp.StatusCode/100 != 2) { if resp != nil { log.Println("http status code:", resp.StatusCode, err) body, _ := ioutil.ReadAll(resp.Body) log.Println("response Body:", string(body)) } filesize := stat.Size() out.Close() if filesize == 0 { os.Remove(destfilename) } return 0, err } defer resp.Body.Close() fname := resp.Header.Get("Source-Filename") if len(fname) > 0 { p.DestName = fname } n, err := io.Copy(out, resp.Body) if err != nil { out.Close() return 0, err } out.Close() log.Printf("%d bytes downloaded.", n) InsertTagToDb(dpexist, p) return n, nil }