// FullPath returns the action full path computed by concatenating the API and resource base paths
// with the action specific path.
func (r *RouteDefinition) FullPath(version *APIVersionDefinition) string {
if strings.HasPrefix(r.Path, "//") {
return httprouter.CleanPath(r.Path[1:])
}
var base string
if r.Parent != nil && r.Parent.Parent != nil {
base = r.Parent.Parent.FullPath(version)
}
return httprouter.CleanPath(path.Join(base, r.Path))
}
// Reverse would reverse the named routes with params supported. E.g. we have a routes "/hello/:name" named "Hello",
// then we can call s.Reverse("Hello", "world") gives us "/hello/world"
func (s *Server) Reverse(name string, params ...interface{}) string {
path, ok := s.namedRoutes[name]
if !ok {
log.Warnf("Server routes reverse failed, cannot find named routes %q", name)
return "/no_such_named_routes_defined"
}
if len(params) == 0 || path == "/" {
return path
}
strParams := make([]string, len(params))
for i, param := range params {
strParams[i] = fmt.Sprint(param)
}
parts := strings.Split(path, "/")[1:]
paramIndex := 0
for i, part := range parts {
if part[0] == ':' || part[0] == '*' {
if paramIndex < len(strParams) {
parts[i] = strParams[paramIndex]
paramIndex++
}
}
}
return httprouter.CleanPath("/" + strings.Join(parts, "/"))
}
func (h *HTTPStore) getRefs(refs map[ref.Ref]bool, cs ChunkSink) {
// POST http://<host>/getRefs/. Post body: ref=sha1---&ref=sha1---& Response will be chunk data if present, 404 if absent.
u := *h.host
u.Path = httprouter.CleanPath(h.host.Path + constants.GetRefsPath)
values := &url.Values{}
for r, _ := range refs {
values.Add("ref", r.String())
}
req := h.newRequest("POST", u.String(), strings.NewReader(values.Encode()), http.Header{
"Accept-Encoding": {"gzip"},
"Content-Type": {"application/x-www-form-urlencoded"},
})
res, err := h.httpClient.Do(req)
d.Chk.NoError(err)
defer closeResponse(res)
d.Chk.Equal(http.StatusOK, res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
reader := res.Body
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
gr, err := gzip.NewReader(reader)
d.Chk.NoError(err)
defer gr.Close()
reader = gr
}
rl := make(chan struct{}, 1) // Rate limit to 1 because there are already N goroutines waiting on responses, all we need to do is send the Chunks back through their channels.
Deserialize(reader, cs, rl)
}
func (bhcs *httpBatchStore) hasRefs(hashes hashSet, batch chunks.ReadBatch) {
// POST http://<host>/hasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|".
u := *bhcs.host
u.Path = httprouter.CleanPath(bhcs.host.Path + constants.HasRefsPath)
req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
"Accept-Encoding": {"x-snappy-framed"},
"Content-Type": {"application/x-www-form-urlencoded"},
})
res, err := bhcs.httpClient.Do(req)
d.Chk.NoError(err)
reader := resBodyReader(res)
defer closeResponse(reader)
d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
scanner := bufio.NewScanner(reader)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
h := hash.Parse(scanner.Text())
d.Chk.True(scanner.Scan())
if scanner.Text() == "true" {
for _, outstanding := range batch[h] {
// This is a little gross, but OutstandingHas.Satisfy() expects a chunk. It ignores it, though, and just sends 'true' over the channel it's holding.
outstanding.Satisfy(chunks.EmptyChunk)
}
} else {
for _, outstanding := range batch[h] {
outstanding.Fail()
}
}
delete(batch, h)
}
}
func (bhcs *httpBatchStore) sendWriteRequests(hashes hashSet, hints types.Hints) {
if len(hashes) == 0 {
return
}
bhcs.rateLimit <- struct{}{}
go func() {
defer func() {
<-bhcs.rateLimit
bhcs.unwrittenPuts.Clear(hashes)
bhcs.requestWg.Add(-len(hashes))
}()
var res *http.Response
var err error
for tryAgain := true; tryAgain; {
serializedChunks, pw := io.Pipe()
errChan := make(chan error)
go func() {
err := bhcs.unwrittenPuts.ExtractChunks(hashes, pw)
// The ordering of these is important. Close the pipe so that the HTTP stack which is reading from serializedChunks knows it has everything, and only THEN block on errChan.
pw.Close()
errChan <- err
close(errChan)
}()
body := buildWriteValueRequest(serializedChunks, hints)
url := *bhcs.host
url.Path = httprouter.CleanPath(bhcs.host.Path + constants.WriteValuePath)
// TODO: Make this accept snappy encoding
req := newRequest("POST", bhcs.auth, url.String(), body, http.Header{
"Accept-Encoding": {"gzip"},
"Content-Encoding": {"x-snappy-framed"},
"Content-Type": {"application/octet-stream"},
})
res, err = bhcs.httpClient.Do(req)
d.Exp.NoError(err)
d.Exp.NoError(<-errChan)
defer closeResponse(res.Body)
if tryAgain = res.StatusCode == httpStatusTooManyRequests; tryAgain {
reader := res.Body
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
gr, err := gzip.NewReader(reader)
d.Exp.NoError(err)
defer gr.Close()
reader = gr
}
/*hashes :=*/ deserializeHashes(reader)
// TODO: BUG 1259 Since the client must currently send all chunks in one batch, the only thing to do in response to backpressure is send EVERYTHING again. Once batching is again possible, this code should figure out how to resend the chunks indicated by hashes.
}
}
d.Exp.True(http.StatusCreated == res.StatusCode, "Unexpected response: %s", formatErrorResponse(res))
}()
}
// createRequestContext creates new request context.
func createRequestContext(request *http.Request, response http.ResponseWriter) *RequestContext {
context := &RequestContext{
Path: httprouter.CleanPath(request.URL.Path),
Method: strings.ToLower(request.Method),
request: request,
response: response,
}
// Format request headers
if len(request.Header) > 0 {
context.Header = make(map[string]string)
for k, v := range request.Header {
if header := strings.ToLower(k); header == "authorization" {
context.Header[header] = v[0]
} else {
context.Header[header] = strings.ToLower(v[0])
}
}
}
// Parse body context if neccessary
var params url.Values
switch context.Method {
case Get:
params = request.URL.Query()
break
case Patch, Post:
if contentType := context.Header["content-type"]; contentType == "application/x-www-form-urlencoded" {
if err := request.ParseForm(); err == nil {
params = request.Form
}
} else if strings.HasPrefix(contentType, "multipart/form-data; boundary") {
if err := request.ParseMultipartForm(Cfg.MultipartSize); err == nil {
params = request.MultipartForm.Value
}
}
break
default:
break
}
// Process params
if len(params) > 0 {
context.QueryParams = make(map[string]string)
for k, v := range params {
context.QueryParams[k] = v[0]
}
}
return context
}
func Test_CreateRequestContext_GetRequest(t *testing.T) {
u := new(TestUnit)
defer u.Teardown()
u.Setup()
// Create test server
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
context := createRequestContext(r, w)
if context.Path != httprouter.CleanPath(r.URL.Path) {
t.Errorf(test.ExpectedStringButFoundString, httprouter.CleanPath(r.URL.Path), context.Path)
}
if context.Header == nil {
t.Error(test.ExpectedNotNil)
} else {
if len(context.Header) != 2 {
t.Errorf(test.ExpectedNumberButFoundNumber, 2, len(context.Header))
} else {
if context.Header["user-agent"] != "go-http-client/1.1" {
t.Errorf(test.ExpectedStringButFoundString, "go-http-client/1.1", context.Header["user-agent"])
}
if context.Header["accept-encoding"] != "gzip" {
t.Errorf(test.ExpectedStringButFoundString, "gzip", context.Header["accept-encoding"])
}
}
}
if context.PathParams != nil {
t.Error(test.ExpectedNil)
}
if context.QueryParams != nil {
t.Error(test.ExpectedNil)
}
}))
defer ts.Close()
http.Get(ts.URL)
}
// FullPath computes the base path to the resource actions concatenating the API and parent resource
// base paths as needed.
func (r *ResourceDefinition) FullPath(version *APIVersionDefinition) string {
var basePath string
if p := r.Parent(); p != nil {
if ca := p.CanonicalAction(); ca != nil {
if routes := ca.Routes; len(routes) > 0 {
// Note: all these tests should be true at code generation time
// as DSL validation makes sure that parent resources have a
// canonical path.
basePath = path.Join(routes[0].FullPath(version))
}
}
} else {
basePath = version.BasePath
}
return httprouter.CleanPath(path.Join(basePath, r.BasePath))
}
func (h *HTTPStore) requestRef(r ref.Ref, method string, body io.Reader) *http.Response {
url := *h.host
url.Path = httprouter.CleanPath(h.host.Path + constants.RefPath)
if !r.IsEmpty() {
url.Path = path.Join(url.Path, r.String())
}
header := http.Header{}
if body != nil {
header.Set("Content-Type", "application/octet-stream")
}
req := h.newRequest(method, url.String(), body, header)
res, err := h.httpClient.Do(req)
d.Chk.NoError(err)
return res
}
func (bhcs *httpBatchStore) requestRoot(method string, current, last hash.Hash) *http.Response {
u := *bhcs.host
u.Path = httprouter.CleanPath(bhcs.host.Path + constants.RootPath)
if method == "POST" {
d.Exp.False(current.IsEmpty())
params := u.Query()
params.Add("last", last.String())
params.Add("current", current.String())
u.RawQuery = params.Encode()
}
req := newRequest(method, bhcs.auth, u.String(), nil, nil)
res, err := bhcs.httpClient.Do(req)
d.Chk.NoError(err)
return res
}
func (h *HTTPStore) requestRoot(method string, current, last ref.Ref) *http.Response {
u := *h.host
u.Path = httprouter.CleanPath(h.host.Path + constants.RootPath)
if method == "POST" {
d.Exp.False(current.IsEmpty())
params := url.Values{}
params.Add("last", last.String())
params.Add("current", current.String())
u.RawQuery = params.Encode()
}
req := h.newRequest(method, u.String(), nil, nil)
res, err := h.httpClient.Do(req)
d.Chk.NoError(err)
return res
}
func parsePattern(p string) (pattern, error) {
if httprouter.CleanPath(p) != p {
return pattern{}, fmt.Errorf("path is not clean")
}
var pat pattern
if !strings.HasPrefix(p, "/") {
return pattern{}, fmt.Errorf("path must start with /")
}
for len(p) > 0 {
i := strings.IndexAny(p, ":*")
if i == -1 {
pat.static = append(pat.static, p)
return pat, nil
}
if i == 0 {
panic("unexpected empty path segment")
}
pat.static = append(pat.static, p[0:i])
if p[i-1] != '/' {
return pattern{}, fmt.Errorf("no / before wildcard segment")
}
p = p[i:]
i = strings.Index(p, "/")
if i == -1 {
pat.static = append(pat.static, "")
pat.vars = append(pat.vars, p[1:])
pat.catchAll = p[0] == '*'
return pat, nil
}
if p[0] == '*' {
return pattern{}, fmt.Errorf("catch-all route not at end of path")
}
v := p[1:i]
if strings.IndexAny(v, ":*") != -1 {
return pattern{}, fmt.Errorf("no / before wildcard segment")
}
pat.static = append(pat.static, "")
pat.vars = append(pat.vars, v)
p = p[i:]
}
return pat, nil
}
func (bhcs *httpBatchStore) getRefs(hashes hashSet, batch chunks.ReadBatch) {
// POST http://<host>/getRefs/. Post body: ref=sha1---&ref=sha1---& Response will be chunk data if present, 404 if absent.
u := *bhcs.host
u.Path = httprouter.CleanPath(bhcs.host.Path + constants.GetRefsPath)
req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
"Accept-Encoding": {"x-snappy-framed"},
"Content-Type": {"application/x-www-form-urlencoded"},
})
res, err := bhcs.httpClient.Do(req)
d.Chk.NoError(err)
reader := resBodyReader(res)
defer closeResponse(reader)
d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
rl := make(chan struct{}, 16)
chunks.Deserialize(reader, &readBatchChunkSink{&batch, &sync.RWMutex{}}, rl)
}
func (h *HTTPStore) getHasRefs(refs map[ref.Ref]bool, reqs hasBatch) {
// POST http://<host>/getHasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|"
u := *h.host
u.Path = httprouter.CleanPath(h.host.Path + constants.GetHasPath)
values := &url.Values{}
for r, _ := range refs {
values.Add("ref", r.String())
}
req := h.newRequest("POST", u.String(), strings.NewReader(values.Encode()), http.Header{
"Accept-Encoding": {"gzip"},
"Content-Type": {"application/x-www-form-urlencoded"},
})
res, err := h.httpClient.Do(req)
d.Chk.NoError(err)
defer closeResponse(res)
d.Chk.Equal(http.StatusOK, res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))
reader := res.Body
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
gr, err := gzip.NewReader(reader)
d.Chk.NoError(err)
defer gr.Close()
reader = gr
}
scanner := bufio.NewScanner(reader)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
r := ref.Parse(scanner.Text())
scanner.Scan()
has := scanner.Text() == "true"
for _, ch := range reqs[r] {
ch <- has
}
}
}
func (h *HTTPStore) postRefs(chs []Chunk) {
body := &bytes.Buffer{}
gw := gzip.NewWriter(body)
sz := NewSerializer(gw)
for _, chunk := range chs {
sz.Put(chunk)
}
sz.Close()
gw.Close()
url := *h.host
url.Path = httprouter.CleanPath(h.host.Path + constants.PostRefsPath)
req := h.newRequest("POST", url.String(), body, http.Header{
"Content-Encoding": {"gzip"},
"Content-Type": {"application/octet-stream"},
})
res, err := h.httpClient.Do(req)
d.Chk.NoError(err)
d.Chk.Equal(res.StatusCode, http.StatusCreated, "Unexpected response: %s", http.StatusText(res.StatusCode))
closeResponse(res)
h.unwrittenPuts.Clear(chs)
}
// GetRouteParameter implemention will extract the param the julienschmidt way
func (h HTTPRouter) GetRouteParameter(r http.Request, param string) string {
path := httprouter.CleanPath(r.URL.Path)
_, params, _ := h.router.Lookup(r.Method, path)
return params.ByName(param)
}
func (h HTTPStoreFlags) CreateStore(ns string) ChunkStore {
if h.check() {
return NewHTTPStore(*h.host+httprouter.CleanPath(ns), *h.auth)
}
return nil
}
func (f RemoteStoreFactory) CreateStore(ns string) Database {
return NewRemoteDatabase(f.host+httprouter.CleanPath(ns), f.auth)
}
