func retryableHTTP(name string, st int, req *http.Request, jd interface{}) {
var err error
for i := 0; i < 3; i++ {
if i > 0 {
log.Printf("Retrying %v to %v", req.Method, req.URL)
time.Sleep(time.Second * time.Duration(i))
}
var res *http.Response
res, err = http.DefaultClient.Do(req)
if err != nil {
continue
}
defer res.Body.Close()
if res.StatusCode == st {
if jd != nil {
d := json.NewDecoder(res.Body)
err = d.Decode(jd)
if err != nil {
continue
}
}
return
}
err = httputil.HTTPError(res)
}
log.Fatalf("Couldn't do %v against %s: %v", req.Method, req.URL, err)
}
// Dump a database or range of a database to a Writer.
//
// db is the name of the db to dump
// from and to are both optional and will be parsed as a seriesly
// timestamp.
func (s *SerieslyDB) Dump(w io.Writer, from, to string) (int64, error) {
u := s.URL()
u.Path += "/_dump"
params := url.Values{}
if err := setTimeParam(params, "from", from); err != nil {
return 0, err
}
if err := setTimeParam(params, "to", to); err != nil {
return 0, err
}
u.RawQuery = params.Encode()
res, err := s.s.client.Get(u.String())
if err != nil {
return 0, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return 0, httputil.HTTPError(res)
}
return io.Copy(w, res.Body)
}
// Get a reference to the file at the given path.
func (c Client) OpenFile(path string) (*FileHandle, error) {
res, err := http.Get(c.URLFor("/.cbfs/info/file/" + noSlash(path)))
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, httputil.HTTPError(res)
}
j := struct {
Meta FileMeta
Path string
}{}
d := json.NewDecoder(res.Body)
err = d.Decode(&j)
if err != nil {
return nil, err
}
h := j.Meta.OID
infos, err := c.GetBlobInfos(h)
if err != nil {
return nil, err
}
return &FileHandle{c, h, 0, j.Meta.Length, j.Meta,
infos[h].Nodes}, nil
}
// Grab a file.
//
// This ensures the request is coming directly from a node that
// already has the blob vs. proxying.
func (c Client) Get(path string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", c.URLFor(path), nil)
if err != nil {
return nil, err
}
req.Header.Set("X-CBFS-LocalOnly", "true")
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
return res.Body, nil
case 300:
defer res.Body.Close()
res, err = http.Get(res.Header.Get("Location"))
if err != nil {
return nil, err
}
return res.Body, nil
default:
defer res.Body.Close()
return nil, httputil.HTTPError(res)
}
}
// Set a configuration parameter by name.
func (c Client) SetConfigParam(key, val string) error {
conf, err := c.GetConfig()
if err != nil {
return err
}
err = conf.SetParameter(key, val)
if err != nil {
return err
}
data, err := json.Marshal(&conf)
if err != nil {
return err
}
req, err := http.NewRequest("PUT", c.confURL(),
bytes.NewBuffer(data))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 204 {
return httputil.HTTPError(res)
}
return nil
}
func (nma *NMA) handleResponse(res *http.Response) error {
if res.StatusCode > 300 || res.StatusCode < 200 {
return httputil.HTTPError(res)
}
_, err := decodeResponse(res.Body)
if err != nil {
// Fill response stuff here.
}
return err
}
func checkFrameClient(addr string) {
fc := findExistingFrameClient(addr)
if fc == nil {
return
}
info := fc.conn.GetInfo()
sufficient := false
if (info.BytesRead-fc.prevInfo.BytesRead > minFrameRead) ||
(info.BytesWritten-fc.prevInfo.BytesWritten > minFrameWritten) {
fc.lastActivity = time.Now()
sufficient = true
}
if time.Since(fc.lastActivity) > frameMaxIdle {
log.Printf("Too long with insufficient activity on %v, shutting down",
addr)
destroyFrameClient(addr)
return
}
// If we're not naturally moving enough data, send a ping.
if !sufficient {
ch := make(chan error)
go func() {
res, err := fc.client.Get("http://" +
addr + "/.cbfs/ping/")
if err == nil {
res.Body.Close()
if res.StatusCode != 204 {
err = httputil.HTTPError(res)
}
}
ch <- err
}()
var err error
select {
case err = <-ch:
case <-time.After(time.Minute):
err = errors.New("ping timeout")
}
if err != nil {
log.Printf("Ping error on %v: %v", addr, err)
destroyFrameClient(addr)
return
}
}
fc.prevInfo = info
fc.checker = time.AfterFunc(frameCheckFreq, func() {
checkFrameClient(addr)
})
}
func (fw fetchWorker) Work(i interface{}) error {
oid := i.(string)
res, err := http.Get(fw.n.BlobURL(oid))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return httputil.HTTPError(res)
}
return fw.cb(oid, res.Body)
}
func getJsonData(u string, into interface{}) error {
res, err := http.Get(u)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return httputil.HTTPError(res)
}
d := json.NewDecoder(res.Body)
return d.Decode(into)
}
// Info returns the info for a database.
func (s *SerieslyDB) Info() (*DBInfo, error) {
rv := &DBInfo{}
res, err := s.s.client.Get(s.URL().String())
if err != nil {
return rv, err
}
if res.StatusCode != 200 {
return rv, httputil.HTTPError(res)
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(rv)
return rv, err
}
func induceTask(ustr, taskname string) error {
u := cbfstool.ParseURL(ustr)
u.Path = "/.cbfs/tasks/" + taskname
res, err := http.PostForm(u.String(), nil)
if err != nil {
return err
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
return httputil.HTTPError(res)
}
return nil
}
func tryURL(url string) error {
client := &http.Client{
Timeout: *timeout,
}
res, err := client.Get(url)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
return nil
}
return httputil.HTTPError(res)
}
func (c *bitfogClient) deleteFile(dest string) error {
req, err := http.NewRequest("DELETE", dest, nil)
if err != nil {
return err
}
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 204 {
return httputil.HTTPError(resp)
}
return nil
}
// List all databases.
func (s *Seriesly) List() ([]string, error) {
u := *s.u
u.Path = "/_all_dbs"
res, err := s.client.Get(u.String())
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, httputil.HTTPError(res)
}
rv := []string{}
err = json.NewDecoder(res.Body).Decode(&rv)
return rv, err
}
func (c *bitfogClient) createSymlink(target, dest string) error {
req, err := http.NewRequest("PUT", dest, strings.NewReader(target))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/symlink")
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 204 {
return httputil.HTTPError(resp)
}
return nil
}
func storeFile(db *couch.Database, pf photoFile) error {
u := pf.url
if u == "" {
u = fmt.Sprintf("%s/%s/%s", db.DBURL(), pf.p.ID, pf.fn)
}
outfile := fmt.Sprintf("%s/%s", pf.p.filepath(), pf.fn)
res, err := http.Get(u)
if err != nil {
return err
}
if res.StatusCode != 200 {
return httputil.HTTPError(res)
}
defer res.Body.Close()
f, err := os.Create(outfile)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, res.Body)
if err != nil {
return err
}
if err := updateExif(outfile, pf.p); err != nil {
log.Printf("Failed to update exif of %v: %v", pf.p.ID, err)
failedf := fmt.Sprintf("%s/failed/%s.%s", *outdir, pf.p.ID, pf.p.Extension)
if err := os.Rename(outfile, failedf); err != nil {
log.Printf("Error renaming failed %v to %vfile: %v",
outfile, failedf, err)
}
if err := storeDetails(pf.p); err != nil {
log.Printf("Error writing out detail json: %v", err)
}
jfi := fmt.Sprintf("%s/%s/details.json", *outdir, pf.p.ID)
jfo := fmt.Sprintf("%s/failed/%s.json", *outdir, pf.p.ID)
if err := os.Rename(jfi, jfo); err != nil {
log.Printf("Error moving json stuff into failed dir: %v", err)
}
return err
}
return nil
}
// Grab a file.
//
// This ensures the request is coming directly from a node that
// already has the blob vs. proxying.
func (c Client) Get(path string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", c.URLFor(path), nil)
if err != nil {
return nil, err
}
req.Header.Set("X-CBFS-LocalOnly", "true")
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
return res.Body, nil
case 300:
defer res.Body.Close()
redirectTarget := res.Header.Get("Location")
log.Printf("Redirecting to %v", redirectTarget)
resRedirect, err := http.Get(redirectTarget)
if err != nil {
return nil, err
}
// if we follow the redirect, make sure response code == 200
switch resRedirect.StatusCode {
case 200:
return resRedirect.Body, nil
default:
return nil, fmt.Errorf(
"Got %v response following redirect to %v",
resRedirect.StatusCode,
redirectTarget,
)
}
default:
defer res.Body.Close()
return nil, httputil.HTTPError(res)
}
}
func sendJsonRequest(name string, st int, req *http.Request, jd interface{}) {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("PRIVATE-TOKEN", *private_token)
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatalf("Couldn't execute %v against %s: %v", req.Method, req.URL, err)
return
}
defer res.Body.Close()
if res.StatusCode == st {
if jd != nil {
d := json.NewDecoder(res.Body)
err = d.Decode(jd)
if err != nil {
log.Fatalf("Error decoding json payload %v", err)
}
}
return
}
log.Fatalf("Couldn't execute %v against %s: %v", req.Method, req.URL, httputil.HTTPError(res))
}
func (c *bitfogClient) uploadFile(src, dest string) error {
srcfile, err := c.fs.Open(src)
if err != nil {
return err
}
defer srcfile.Close()
req, err := http.NewRequest("PUT", dest, srcfile)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 204 {
return httputil.HTTPError(resp)
}
return nil
}
// Parses json stuff into a thing. Returns the next URL if any
func getJSON(name, subu string, out interface{}) string {
u := subu
if !strings.HasPrefix(u, "http") {
u = base + subu
}
req, err := http.NewRequest("GET", u, nil)
maybeFatal(name, err)
req.SetBasicAuth(*username, *password)
for i := 0; i < 3; i++ {
if i > 0 {
log.Printf("Retrying JSON req to %v", req.URL)
}
var res *http.Response
res, err = http.DefaultClient.Do(req)
if err != nil {
continue
}
defer res.Body.Close()
if res.StatusCode != 200 {
err = httputil.HTTPError(res)
continue
}
links := parseLink(res.Header.Get("Link"))
d := json.NewDecoder(res.Body)
maybeFatal(name, d.Decode(out))
return links["next"]
}
log.Fatalf("Error getting JSON from %v: %v", u, err)
panic("unreachable")
}
func process(u string, in []byte, ch chan result) {
var latestError error
for i := 0; i < *retries; i++ {
latestError = nil
if *verbose {
log.Printf("Trying %v", u)
}
req, err := http.NewRequest("POST", u, bytes.NewReader(in))
if err != nil {
log.Fatalf("Error creating request for %v: %v", u, err)
}
for k, v := range headerProto {
req.Header[k] = v
}
res, err := http.DefaultClient.Do(req)
if err == nil {
res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
err = httputil.HTTPError(res)
}
}
if err != nil {
latestError = err
log.Printf("Error on %v: %v", u, err)
time.Sleep(*backoff)
continue
}
break
}
ch <- result{u, latestError}
}
