func (im *imp) doAPI(ctx *context.Context, result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments. must be even.")
}
if im.creds() == nil {
return fmt.Errorf("No authentication creds")
}
form := url.Values{}
for i := 0; i < len(keyval); i += 2 {
if keyval[i+1] != "" {
form.Set(keyval[i], keyval[i+1])
}
}
fullURL := apiURL + apiPath
res, err := im.doGet(ctx, fullURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
return fmt.Errorf("could not parse response for %s: %v", fullURL, err)
}
return nil
}
func (im *imp) flickrAPIRequest(result interface{}, method string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
}
if im.user == nil {
return fmt.Errorf("No authenticated user")
}
form := url.Values{}
form.Set("method", method)
form.Set("format", "json")
form.Set("nojsoncallback", "1")
form.Set("user_id", im.user.Id)
for i := 0; i < len(keyval); i += 2 {
form.Set(keyval[i], keyval[i+1])
}
res, err := im.flickrRequest(apiURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
log.Printf("Error parsing response for %s: %s", apiURL, err)
}
return err
}
// SearchExistingFileSchema does a search query looking for an
// existing file with entire contents of wholeRef, then does a HEAD
// request to verify the file still exists on the server. If so,
// it returns that file schema's blobref.
//
// May return (zero, nil) on ENOENT. A non-nil error is only returned
// if there were problems searching.
func (c *Client) SearchExistingFileSchema(wholeRef blob.Ref) (blob.Ref, error) {
sr, err := c.SearchRoot()
if err != nil {
return blob.Ref{}, err
}
url := sr + "camli/search/files?wholedigest=" + wholeRef.String()
req := c.newRequest("GET", url)
res, err := c.doReqGated(req)
if err != nil {
return blob.Ref{}, err
}
if res.StatusCode != 200 {
body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
res.Body.Close()
return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
}
var ress struct {
Files []blob.Ref `json:"files"`
}
if err := httputil.DecodeJSON(res, &ress); err != nil {
return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
}
if len(ress.Files) == 0 {
return blob.Ref{}, nil
}
for _, f := range ress.Files {
if c.FileHasContents(f, wholeRef) {
return f, nil
}
}
return blob.Ref{}, nil
}
func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
}
token, err := im.tokenCache.Token()
if err != nil {
return fmt.Errorf("Token error: %v", err)
}
form := url.Values{}
form.Set("v", "20140225") // 4sq requires this to version their API
form.Set("oauth_token", token.AccessToken)
for i := 0; i < len(keyval); i += 2 {
form.Set(keyval[i], keyval[i+1])
}
fullURL := apiURL + apiPath
res, err := im.doGet(fullURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
log.Printf("Error parsing response for %s: %v", fullURL, err)
}
return err
}
func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
}
if im.cred == nil {
return fmt.Errorf("No authentication creds")
}
if im.userid == "" {
return fmt.Errorf("No user id")
}
form := url.Values{}
form.Set("user_id", im.userid)
for i := 0; i < len(keyval); i += 2 {
if keyval[i+1] != "" {
form.Set(keyval[i], keyval[i+1])
}
}
res, err := im.doGet(apiURL+apiPath, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
log.Printf("Error parsing response for %s: %s", apiURL, err)
}
return err
}
// GetJSON sends a GET request to url, and unmarshals the returned
// JSON response into data. The URL's host must match the client's
// configured server.
func (c *Client) GetJSON(url string, data interface{}) error {
if !strings.HasPrefix(url, c.discoRoot()) {
return fmt.Errorf("wrong URL (%q) for this server", url)
}
hreq := c.newRequest("GET", url)
resp, err := c.expect2XX(hreq)
if err != nil {
return err
}
return httputil.DecodeJSON(resp, data)
}
// TODO(bradfitz): delete most of this. use new camlistore.org/pkg/blobserver/protocol types instead
// of a map[string]interface{}.
func (c *Client) responseJSONMap(requestName string, resp *http.Response) (map[string]interface{}, error) {
if resp.StatusCode != 200 {
log.Printf("After %s request, failed to JSON from response; status code is %d", requestName, resp.StatusCode)
io.Copy(os.Stderr, resp.Body)
return nil, fmt.Errorf("After %s request, HTTP response code is %d; no JSON to parse.", requestName, resp.StatusCode)
}
jmap := make(map[string]interface{})
if err := httputil.DecodeJSON(resp, &jmap); err != nil {
return nil, err
}
return jmap, nil
}
// Remove the list of blobs. An error is returned if the server failed to
// remove a blob. Removing a non-existent blob isn't an error.
func (c *Client) RemoveBlobs(blobs []blob.Ref) error {
if c.sto != nil {
return c.sto.RemoveBlobs(blobs)
}
pfx, err := c.prefix()
if err != nil {
return err
}
url_ := fmt.Sprintf("%s/camli/remove", pfx)
params := make(url.Values) // "blobN" -> BlobRefStr
needsDelete := make(map[string]bool) // BlobRefStr -> true
for n, b := range blobs {
if !b.Valid() {
return errors.New("Cannot delete invalid blobref")
}
key := fmt.Sprintf("blob%v", n+1)
params.Add(key, b.String())
needsDelete[b.String()] = true
}
req, err := http.NewRequest("POST", url_, strings.NewReader(params.Encode()))
if err != nil {
return fmt.Errorf("Error creating RemoveBlobs POST request: %v", err)
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
c.authMode.AddAuthHeader(req)
resp, err := c.httpClient.Do(req)
if err != nil {
resp.Body.Close()
return fmt.Errorf("Got status code %d from blobserver for remove %s", resp.StatusCode, params.Encode())
}
if resp.StatusCode != 200 {
resp.Body.Close()
return fmt.Errorf("Invalid http response %d in remove response", resp.StatusCode)
}
var remResp removeResponse
if err := httputil.DecodeJSON(resp, &remResp); err != nil {
return fmt.Errorf("Failed to parse remove response: %v", err)
}
for _, value := range remResp.Removed {
delete(needsDelete, value)
}
if len(needsDelete) > 0 {
return fmt.Errorf("Failed to remove blobs %s", strings.Join(stringKeys(needsDelete), ", "))
}
return nil
}
func parseStatResponse(res *http.Response) (*statResponse, error) {
var s = &statResponse{HaveMap: make(map[string]blob.SizedRef)}
var pres protocol.StatResponse
if err := httputil.DecodeJSON(res, &pres); err != nil {
return nil, ResponseFormatError(err)
}
s.canLongPoll = pres.CanLongPoll
for _, statItem := range pres.Stat {
br := statItem.Ref
if !br.Valid() {
continue
}
s.HaveMap[br.String()] = blob.SizedRef{br, int64(statItem.Size)}
}
return s, nil
}
func (c *Client) GetClaims(req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
sr, err := c.SearchRoot()
if err != nil {
return nil, err
}
url := sr + req.URLSuffix()
hreq := c.newRequest("GET", url)
hres, err := c.expect2XX(hreq)
if err != nil {
return nil, err
}
res := new(search.ClaimsResponse)
if err := httputil.DecodeJSON(hres, res); err != nil {
return nil, err
}
return res, nil
}
// PopulateJSONFromURL makes a GET call at apiURL, using keyval as parameters of
// the associated form. The JSON response is decoded into result.
func (ctx OAuthContext) PopulateJSONFromURL(result interface{}, apiURL string, keyval ...string) error {
if len(keyval)%2 == 1 {
return errors.New("Incorrect number of keyval arguments. must be even.")
}
form := url.Values{}
for i := 0; i < len(keyval); i += 2 {
form.Set(keyval[i], keyval[i+1])
}
hres, err := ctx.Get(apiURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(hres, result)
if err != nil {
return fmt.Errorf("could not parse response for %s: %v", apiURL, err)
}
return err
}
func (im *imp) doAPI(ctx context.Context, form url.Values, result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
}
form.Set("v", apiVersion) // 4sq requires this to version their API
for i := 0; i < len(keyval); i += 2 {
form.Set(keyval[i], keyval[i+1])
}
fullURL := apiURL + apiPath
res, err := doGet(ctx, fullURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
log.Printf("Error parsing response for %s: %v", fullURL, err)
}
return err
}
func (c *Client) Query(req *search.SearchQuery) (*search.SearchResult, error) {
sr, err := c.SearchRoot()
if err != nil {
return nil, err
}
url := sr + req.URLSuffix()
body, err := json.MarshalIndent(req, "", "\t")
if err != nil {
return nil, err
}
hreq := c.newRequest("POST", url, bytes.NewReader(body))
hres, err := c.expect2XX(hreq)
if err != nil {
return nil, err
}
res := new(search.SearchResult)
if err := httputil.DecodeJSON(hres, res); err != nil {
return nil, err
}
return res, nil
}
func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
// TODO: use ctx (wait for Go 1.7?)
sr, err := c.SearchRoot()
if err != nil {
return nil, err
}
url := sr + req.URLSuffixPost()
body, err := json.MarshalIndent(req, "", "\t")
if err != nil {
return nil, err
}
hreq := c.newRequest("POST", url, bytes.NewReader(body))
hres, err := c.expect2XX(hreq)
if err != nil {
return nil, err
}
res := new(search.DescribeResponse)
if err := httputil.DecodeJSON(hres, res); err != nil {
return nil, err
}
return res, nil
}
func (c *Client) doDiscovery() error {
root, err := url.Parse(c.discoRoot())
if err != nil {
return err
}
res, err := c.discoveryResp()
if err != nil {
return err
}
var disco camtypes.Discovery
if err := httputil.DecodeJSON(res, &disco); err != nil {
return err
}
u, err := root.Parse(disco.SearchRoot)
if err != nil {
return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
}
c.searchRoot = u.String()
u, err = root.Parse(disco.HelpRoot)
if err != nil {
return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
}
c.helpRoot = u.String()
c.storageGen = disco.StorageGeneration
u, err = root.Parse(disco.BlobRoot)
if err != nil {
return fmt.Errorf("client: error resolving blobRoot: %v", err)
}
c.prefixv = strings.TrimRight(u.String(), "/")
if disco.UIDiscovery != nil {
u, err = root.Parse(disco.DownloadHelper)
if err != nil {
return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
}
c.downloadHelper = u.String()
}
if disco.SyncHandlers != nil {
for _, v := range disco.SyncHandlers {
ufrom, err := root.Parse(v.From)
if err != nil {
return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
}
uto, err := root.Parse(v.To)
if err != nil {
return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
}
c.syncHandlers = append(c.syncHandlers, &SyncInfo{
From: ufrom.String(),
To: uto.String(),
ToIndex: v.ToIndex,
})
}
}
if disco.Signing != nil {
c.serverKeyID = disco.Signing.PublicKeyID
}
return nil
}
func (c *Client) doDiscovery() error {
root, err := url.Parse(c.discoRoot())
if err != nil {
return err
}
// If the path is just "" or "/", do discovery against
// the URL to see which path we should actually use.
req := c.newRequest("GET", c.discoRoot(), nil)
req.Header.Set("Accept", "text/x-camli-configuration")
res, err := c.doReqGated(req)
if err != nil {
return err
}
if res.StatusCode != 200 {
res.Body.Close()
return fmt.Errorf("Got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
}
// TODO(bradfitz): little weird in retrospect that we request
// text/x-camli-configuration and expect to get back
// text/javascript. Make them consistent.
if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
res.Body.Close()
return fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
}
// TODO: make a proper struct type for this in another package somewhere:
m := make(map[string]interface{})
if err := httputil.DecodeJSON(res, &m); err != nil {
return err
}
searchRoot, ok := m["searchRoot"].(string)
if ok {
u, err := root.Parse(searchRoot)
if err != nil {
return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", searchRoot)
}
c.searchRoot = u.String()
}
downloadHelper, ok := m["downloadHelper"].(string)
if ok {
u, err := root.Parse(downloadHelper)
if err != nil {
return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", downloadHelper)
}
c.downloadHelper = u.String()
}
c.storageGen, _ = m["storageGeneration"].(string)
blobRoot, ok := m["blobRoot"].(string)
if !ok {
return fmt.Errorf("No blobRoot in config discovery response")
}
u, err := root.Parse(blobRoot)
if err != nil {
return fmt.Errorf("client: error resolving blobRoot: %v", err)
}
c.prefixv = strings.TrimRight(u.String(), "/")
syncHandlers, ok := m["syncHandlers"].([]interface{})
if ok {
for _, v := range syncHandlers {
vmap := v.(map[string]interface{})
from := vmap["from"].(string)
ufrom, err := root.Parse(from)
if err != nil {
return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", from)
}
to := vmap["to"].(string)
uto, err := root.Parse(to)
if err != nil {
return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", to)
}
toIndex, _ := vmap["toIndex"].(bool)
c.syncHandlers = append(c.syncHandlers, &SyncInfo{
From: ufrom.String(),
To: uto.String(),
ToIndex: toIndex,
})
}
}
return nil
}
func (c *Client) doDiscovery() error {
root, err := url.Parse(c.discoRoot())
if err != nil {
return err
}
res, err := c.discoveryResp()
if err != nil {
return err
}
// TODO: make a proper struct type for this in another package somewhere:
m := make(map[string]interface{})
if err := httputil.DecodeJSON(res, &m); err != nil {
return err
}
searchRoot, ok := m["searchRoot"].(string)
if ok {
u, err := root.Parse(searchRoot)
if err != nil {
return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", searchRoot)
}
c.searchRoot = u.String()
}
downloadHelper, ok := m["downloadHelper"].(string)
if ok {
u, err := root.Parse(downloadHelper)
if err != nil {
return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", downloadHelper)
}
c.downloadHelper = u.String()
}
c.storageGen, _ = m["storageGeneration"].(string)
blobRoot, ok := m["blobRoot"].(string)
if !ok {
return fmt.Errorf("No blobRoot in config discovery response")
}
u, err := root.Parse(blobRoot)
if err != nil {
return fmt.Errorf("client: error resolving blobRoot: %v", err)
}
c.prefixv = strings.TrimRight(u.String(), "/")
syncHandlers, ok := m["syncHandlers"].([]interface{})
if ok {
for _, v := range syncHandlers {
vmap := v.(map[string]interface{})
from := vmap["from"].(string)
ufrom, err := root.Parse(from)
if err != nil {
return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", from)
}
to := vmap["to"].(string)
uto, err := root.Parse(to)
if err != nil {
return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", to)
}
toIndex, _ := vmap["toIndex"].(bool)
c.syncHandlers = append(c.syncHandlers, &SyncInfo{
From: ufrom.String(),
To: uto.String(),
ToIndex: toIndex,
})
}
}
serverSigning, ok := m["signing"].(map[string]interface{})
if ok {
c.serverKeyID = serverSigning["publicKeyId"].(string)
}
return nil
}
// Upload uploads a blob, as described by the provided UploadHandle parameters.
func (c *Client) Upload(h *UploadHandle) (*PutResult, error) {
errorf := func(msg string, arg ...interface{}) (*PutResult, error) {
err := fmt.Errorf(msg, arg...)
c.log.Print(err.Error())
return nil, err
}
bodyReader, bodySize, err := h.readerAndSize()
if err != nil {
return nil, fmt.Errorf("client: error slurping upload handle to find its length: %v", err)
}
if bodySize > constants.MaxBlobSize {
return nil, errors.New("client: body is bigger then max blob size")
}
c.statsMutex.Lock()
c.stats.UploadRequests.Blobs++
c.stats.UploadRequests.Bytes += bodySize
c.statsMutex.Unlock()
pr := &PutResult{BlobRef: h.BlobRef, Size: uint32(bodySize)}
if c.sto != nil {
// TODO: stat first so we can show skipped?
_, err := blobserver.Receive(c.sto, h.BlobRef, bodyReader)
if err != nil {
return nil, err
}
return pr, nil
}
if !h.Vivify {
if _, ok := c.haveCache.StatBlobCache(h.BlobRef); ok {
pr.Skipped = true
return pr, nil
}
}
blobrefStr := h.BlobRef.String()
// Pre-upload. Check whether the blob already exists on the
// server and if not, the URL to upload it to.
pfx, err := c.prefix()
if err != nil {
return nil, err
}
if !h.SkipStat {
url_ := fmt.Sprintf("%s/camli/stat", pfx)
req := c.newRequest("POST", url_, strings.NewReader("camliversion=1&blob1="+blobrefStr))
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.doReqGated(req)
if err != nil {
return errorf("stat http error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errorf("stat response had http status %d", resp.StatusCode)
}
stat, err := parseStatResponse(resp)
if err != nil {
return nil, err
}
for _, sbr := range stat.HaveMap {
c.haveCache.NoteBlobExists(sbr.Ref, uint32(sbr.Size))
}
_, serverHasIt := stat.HaveMap[blobrefStr]
if env.DebugUploads() {
log.Printf("HTTP Stat(%s) = %v", blobrefStr, serverHasIt)
}
if !h.Vivify && serverHasIt {
pr.Skipped = true
if closer, ok := h.Contents.(io.Closer); ok {
// TODO(bradfitz): I did this
// Close-if-possible thing early on, before I
// knew better. Fix the callers instead, and
// fix the docs.
closer.Close()
}
c.haveCache.NoteBlobExists(h.BlobRef, uint32(bodySize))
return pr, nil
}
}
if env.DebugUploads() {
log.Printf("Uploading: %s (%d bytes)", blobrefStr, bodySize)
}
pipeReader, pipeWriter := io.Pipe()
multipartWriter := multipart.NewWriter(pipeWriter)
copyResult := make(chan error, 1)
go func() {
defer pipeWriter.Close()
part, err := multipartWriter.CreateFormFile(blobrefStr, blobrefStr)
if err != nil {
copyResult <- err
return
}
_, err = io.Copy(part, bodyReader)
if err == nil {
err = multipartWriter.Close()
}
copyResult <- err
}()
// TODO(bradfitz): verbosity levels. make this VLOG(2) or something. it's noisy:
// c.log.Printf("Uploading %s", br)
uploadURL := fmt.Sprintf("%s/camli/upload", pfx)
req := c.newRequest("POST", uploadURL)
req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
if h.Vivify {
req.Header.Add("X-Camlistore-Vivify", "1")
}
req.Body = ioutil.NopCloser(pipeReader)
req.ContentLength = multipartOverhead + bodySize + int64(len(blobrefStr))*2
resp, err := c.doReqGated(req)
if err != nil {
return errorf("upload http error: %v", err)
}
defer resp.Body.Close()
// check error from earlier copy
if err := <-copyResult; err != nil {
return errorf("failed to copy contents into multipart writer: %v", err)
}
// The only valid HTTP responses are 200 and 303.
if resp.StatusCode != 200 && resp.StatusCode != 303 {
return errorf("invalid http response %d in upload response", resp.StatusCode)
}
if resp.StatusCode == 303 {
otherLocation := resp.Header.Get("Location")
if otherLocation == "" {
return errorf("303 without a Location")
}
baseURL, _ := url.Parse(uploadURL)
absURL, err := baseURL.Parse(otherLocation)
if err != nil {
return errorf("303 Location URL relative resolve error: %v", err)
}
otherLocation = absURL.String()
resp, err = http.Get(otherLocation)
if err != nil {
return errorf("error following 303 redirect after upload: %v", err)
}
}
var ures protocol.UploadResponse
if err := httputil.DecodeJSON(resp, &ures); err != nil {
return errorf("error in upload response: %v", err)
}
if ures.ErrorText != "" {
c.log.Printf("Blob server reports error: %s", ures.ErrorText)
}
expectedSize := uint32(bodySize)
for _, sb := range ures.Received {
if sb.Ref != h.BlobRef {
continue
}
if sb.Size != expectedSize {
return errorf("Server got blob %v, but reports wrong length (%v; we sent %d)",
sb.Ref, sb.Size, expectedSize)
}
c.statsMutex.Lock()
c.stats.Uploads.Blobs++
c.stats.Uploads.Bytes += bodySize
c.statsMutex.Unlock()
if pr.Size <= 0 {
pr.Size = sb.Size
}
c.haveCache.NoteBlobExists(pr.BlobRef, pr.Size)
return pr, nil
}
return nil, errors.New("Server didn't receive blob.")
}