Пример #1
0
// FullPath returns the action full path computed by concatenating the API and resource base paths
// with the action specific path.
func (r *RouteDefinition) FullPath(version *APIVersionDefinition) string {
	if strings.HasPrefix(r.Path, "//") {
		return httprouter.CleanPath(r.Path[1:])
	}
	var base string
	if r.Parent != nil && r.Parent.Parent != nil {
		base = r.Parent.Parent.FullPath(version)
	}
	return httprouter.CleanPath(path.Join(base, r.Path))
}
Пример #2
0
// Reverse would reverse the named routes with params supported. E.g. we have a routes "/hello/:name" named "Hello",
// then we can call s.Reverse("Hello", "world") gives us "/hello/world"
func (s *Server) Reverse(name string, params ...interface{}) string {
	path, ok := s.namedRoutes[name]
	if !ok {
		log.Warnf("Server routes reverse failed, cannot find named routes %q", name)
		return "/no_such_named_routes_defined"
	}
	if len(params) == 0 || path == "/" {
		return path
	}
	strParams := make([]string, len(params))
	for i, param := range params {
		strParams[i] = fmt.Sprint(param)
	}
	parts := strings.Split(path, "/")[1:]
	paramIndex := 0
	for i, part := range parts {
		if part[0] == ':' || part[0] == '*' {
			if paramIndex < len(strParams) {
				parts[i] = strParams[paramIndex]
				paramIndex++
			}
		}
	}
	return httprouter.CleanPath("/" + strings.Join(parts, "/"))
}
Пример #3
0
func (h *HTTPStore) getRefs(refs map[ref.Ref]bool, cs ChunkSink) {
	// POST http://<host>/getRefs/. Post body: ref=sha1---&ref=sha1---& Response will be chunk data if present, 404 if absent.
	u := *h.host
	u.Path = httprouter.CleanPath(h.host.Path + constants.GetRefsPath)
	values := &url.Values{}
	for r, _ := range refs {
		values.Add("ref", r.String())
	}

	req := h.newRequest("POST", u.String(), strings.NewReader(values.Encode()), http.Header{
		"Accept-Encoding": {"gzip"},
		"Content-Type":    {"application/x-www-form-urlencoded"},
	})

	res, err := h.httpClient.Do(req)
	d.Chk.NoError(err)
	defer closeResponse(res)
	d.Chk.Equal(http.StatusOK, res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))

	reader := res.Body
	if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(reader)
		d.Chk.NoError(err)
		defer gr.Close()
		reader = gr
	}

	rl := make(chan struct{}, 1) // Rate limit to 1 because there are already N goroutines waiting on responses, all we need to do is send the Chunks back through their channels.
	Deserialize(reader, cs, rl)
}
Пример #4
0
func (bhcs *httpBatchStore) hasRefs(hashes hashSet, batch chunks.ReadBatch) {
	// POST http://<host>/hasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|".
	u := *bhcs.host
	u.Path = httprouter.CleanPath(bhcs.host.Path + constants.HasRefsPath)

	req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
		"Accept-Encoding": {"x-snappy-framed"},
		"Content-Type":    {"application/x-www-form-urlencoded"},
	})

	res, err := bhcs.httpClient.Do(req)
	d.Chk.NoError(err)
	reader := resBodyReader(res)
	defer closeResponse(reader)

	d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))

	scanner := bufio.NewScanner(reader)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		h := hash.Parse(scanner.Text())
		d.Chk.True(scanner.Scan())
		if scanner.Text() == "true" {
			for _, outstanding := range batch[h] {
				// This is a little gross, but OutstandingHas.Satisfy() expects a chunk. It ignores it, though, and just sends 'true' over the channel it's holding.
				outstanding.Satisfy(chunks.EmptyChunk)
			}
		} else {
			for _, outstanding := range batch[h] {
				outstanding.Fail()
			}
		}
		delete(batch, h)
	}
}
Пример #5
0
func (bhcs *httpBatchStore) sendWriteRequests(hashes hashSet, hints types.Hints) {
	if len(hashes) == 0 {
		return
	}
	bhcs.rateLimit <- struct{}{}
	go func() {
		defer func() {
			<-bhcs.rateLimit
			bhcs.unwrittenPuts.Clear(hashes)
			bhcs.requestWg.Add(-len(hashes))
		}()

		var res *http.Response
		var err error
		for tryAgain := true; tryAgain; {
			serializedChunks, pw := io.Pipe()
			errChan := make(chan error)
			go func() {
				err := bhcs.unwrittenPuts.ExtractChunks(hashes, pw)
				// The ordering of these is important. Close the pipe so that the HTTP stack which is reading from serializedChunks knows it has everything, and only THEN block on errChan.
				pw.Close()
				errChan <- err
				close(errChan)
			}()
			body := buildWriteValueRequest(serializedChunks, hints)

			url := *bhcs.host
			url.Path = httprouter.CleanPath(bhcs.host.Path + constants.WriteValuePath)
			// TODO: Make this accept snappy encoding
			req := newRequest("POST", bhcs.auth, url.String(), body, http.Header{
				"Accept-Encoding":  {"gzip"},
				"Content-Encoding": {"x-snappy-framed"},
				"Content-Type":     {"application/octet-stream"},
			})

			res, err = bhcs.httpClient.Do(req)
			d.Exp.NoError(err)
			d.Exp.NoError(<-errChan)
			defer closeResponse(res.Body)

			if tryAgain = res.StatusCode == httpStatusTooManyRequests; tryAgain {
				reader := res.Body
				if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
					gr, err := gzip.NewReader(reader)
					d.Exp.NoError(err)
					defer gr.Close()
					reader = gr
				}
				/*hashes :=*/ deserializeHashes(reader)
				// TODO: BUG 1259 Since the client must currently send all chunks in one batch, the only thing to do in response to backpressure is send EVERYTHING again. Once batching is again possible, this code should figure out how to resend the chunks indicated by hashes.
			}
		}

		d.Exp.True(http.StatusCreated == res.StatusCode, "Unexpected response: %s", formatErrorResponse(res))
	}()
}
Пример #6
0
// createRequestContext creates new request context.
func createRequestContext(request *http.Request, response http.ResponseWriter) *RequestContext {
	context := &RequestContext{
		Path:     httprouter.CleanPath(request.URL.Path),
		Method:   strings.ToLower(request.Method),
		request:  request,
		response: response,
	}

	// Format request headers
	if len(request.Header) > 0 {
		context.Header = make(map[string]string)

		for k, v := range request.Header {
			if header := strings.ToLower(k); header == "authorization" {
				context.Header[header] = v[0]
			} else {
				context.Header[header] = strings.ToLower(v[0])
			}
		}
	}

	// Parse body context if neccessary
	var params url.Values
	switch context.Method {

	case Get:
		params = request.URL.Query()
		break

	case Patch, Post:
		if contentType := context.Header["content-type"]; contentType == "application/x-www-form-urlencoded" {
			if err := request.ParseForm(); err == nil {
				params = request.Form
			}
		} else if strings.HasPrefix(contentType, "multipart/form-data; boundary") {
			if err := request.ParseMultipartForm(Cfg.MultipartSize); err == nil {
				params = request.MultipartForm.Value
			}
		}
		break

	default:
		break
	}

	// Process params
	if len(params) > 0 {
		context.QueryParams = make(map[string]string)

		for k, v := range params {
			context.QueryParams[k] = v[0]
		}
	}
	return context
}
Пример #7
0
func Test_CreateRequestContext_GetRequest(t *testing.T) {
	u := new(TestUnit)
	defer u.Teardown()
	u.Setup()

	// Create test server
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		context := createRequestContext(r, w)

		if context.Path != httprouter.CleanPath(r.URL.Path) {
			t.Errorf(test.ExpectedStringButFoundString, httprouter.CleanPath(r.URL.Path), context.Path)
		}
		if context.Header == nil {
			t.Error(test.ExpectedNotNil)
		} else {
			if len(context.Header) != 2 {
				t.Errorf(test.ExpectedNumberButFoundNumber, 2, len(context.Header))
			} else {
				if context.Header["user-agent"] != "go-http-client/1.1" {
					t.Errorf(test.ExpectedStringButFoundString, "go-http-client/1.1", context.Header["user-agent"])
				}
				if context.Header["accept-encoding"] != "gzip" {
					t.Errorf(test.ExpectedStringButFoundString, "gzip", context.Header["accept-encoding"])
				}
			}
		}
		if context.PathParams != nil {
			t.Error(test.ExpectedNil)
		}
		if context.QueryParams != nil {
			t.Error(test.ExpectedNil)
		}
	}))
	defer ts.Close()
	http.Get(ts.URL)
}
Пример #8
0
// FullPath computes the base path to the resource actions concatenating the API and parent resource
// base paths as needed.
func (r *ResourceDefinition) FullPath(version *APIVersionDefinition) string {
	var basePath string
	if p := r.Parent(); p != nil {
		if ca := p.CanonicalAction(); ca != nil {
			if routes := ca.Routes; len(routes) > 0 {
				// Note: all these tests should be true at code generation time
				// as DSL validation makes sure that parent resources have a
				// canonical path.
				basePath = path.Join(routes[0].FullPath(version))
			}
		}
	} else {
		basePath = version.BasePath
	}
	return httprouter.CleanPath(path.Join(basePath, r.BasePath))
}
Пример #9
0
func (h *HTTPStore) requestRef(r ref.Ref, method string, body io.Reader) *http.Response {
	url := *h.host
	url.Path = httprouter.CleanPath(h.host.Path + constants.RefPath)
	if !r.IsEmpty() {
		url.Path = path.Join(url.Path, r.String())
	}

	header := http.Header{}
	if body != nil {
		header.Set("Content-Type", "application/octet-stream")
	}
	req := h.newRequest(method, url.String(), body, header)

	res, err := h.httpClient.Do(req)
	d.Chk.NoError(err)
	return res
}
Пример #10
0
func (bhcs *httpBatchStore) requestRoot(method string, current, last hash.Hash) *http.Response {
	u := *bhcs.host
	u.Path = httprouter.CleanPath(bhcs.host.Path + constants.RootPath)
	if method == "POST" {
		d.Exp.False(current.IsEmpty())
		params := u.Query()
		params.Add("last", last.String())
		params.Add("current", current.String())
		u.RawQuery = params.Encode()
	}

	req := newRequest(method, bhcs.auth, u.String(), nil, nil)

	res, err := bhcs.httpClient.Do(req)
	d.Chk.NoError(err)

	return res
}
Пример #11
0
func (h *HTTPStore) requestRoot(method string, current, last ref.Ref) *http.Response {
	u := *h.host
	u.Path = httprouter.CleanPath(h.host.Path + constants.RootPath)
	if method == "POST" {
		d.Exp.False(current.IsEmpty())
		params := url.Values{}
		params.Add("last", last.String())
		params.Add("current", current.String())
		u.RawQuery = params.Encode()
	}

	req := h.newRequest(method, u.String(), nil, nil)

	res, err := h.httpClient.Do(req)
	d.Chk.NoError(err)

	return res
}
Пример #12
0
func parsePattern(p string) (pattern, error) {
	if httprouter.CleanPath(p) != p {
		return pattern{}, fmt.Errorf("path is not clean")
	}
	var pat pattern
	if !strings.HasPrefix(p, "/") {
		return pattern{}, fmt.Errorf("path must start with /")
	}
	for len(p) > 0 {
		i := strings.IndexAny(p, ":*")
		if i == -1 {
			pat.static = append(pat.static, p)
			return pat, nil
		}
		if i == 0 {
			panic("unexpected empty path segment")
		}
		pat.static = append(pat.static, p[0:i])
		if p[i-1] != '/' {
			return pattern{}, fmt.Errorf("no / before wildcard segment")
		}
		p = p[i:]
		i = strings.Index(p, "/")
		if i == -1 {
			pat.static = append(pat.static, "")
			pat.vars = append(pat.vars, p[1:])
			pat.catchAll = p[0] == '*'
			return pat, nil
		}
		if p[0] == '*' {
			return pattern{}, fmt.Errorf("catch-all route not at end of path")
		}
		v := p[1:i]
		if strings.IndexAny(v, ":*") != -1 {
			return pattern{}, fmt.Errorf("no / before wildcard segment")
		}
		pat.static = append(pat.static, "")
		pat.vars = append(pat.vars, v)
		p = p[i:]
	}
	return pat, nil
}
Пример #13
0
func (bhcs *httpBatchStore) getRefs(hashes hashSet, batch chunks.ReadBatch) {
	// POST http://<host>/getRefs/. Post body: ref=sha1---&ref=sha1---& Response will be chunk data if present, 404 if absent.
	u := *bhcs.host
	u.Path = httprouter.CleanPath(bhcs.host.Path + constants.GetRefsPath)

	req := newRequest("POST", bhcs.auth, u.String(), buildHashesRequest(hashes), http.Header{
		"Accept-Encoding": {"x-snappy-framed"},
		"Content-Type":    {"application/x-www-form-urlencoded"},
	})

	res, err := bhcs.httpClient.Do(req)
	d.Chk.NoError(err)
	reader := resBodyReader(res)
	defer closeResponse(reader)

	d.Chk.True(http.StatusOK == res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))

	rl := make(chan struct{}, 16)
	chunks.Deserialize(reader, &readBatchChunkSink{&batch, &sync.RWMutex{}}, rl)
}
Пример #14
0
func (h *HTTPStore) getHasRefs(refs map[ref.Ref]bool, reqs hasBatch) {
	// POST http://<host>/getHasRefs/. Post body: ref=sha1---&ref=sha1---& Response will be text of lines containing "|ref| |bool|"
	u := *h.host
	u.Path = httprouter.CleanPath(h.host.Path + constants.GetHasPath)
	values := &url.Values{}
	for r, _ := range refs {
		values.Add("ref", r.String())
	}

	req := h.newRequest("POST", u.String(), strings.NewReader(values.Encode()), http.Header{
		"Accept-Encoding": {"gzip"},
		"Content-Type":    {"application/x-www-form-urlencoded"},
	})

	res, err := h.httpClient.Do(req)
	d.Chk.NoError(err)
	defer closeResponse(res)
	d.Chk.Equal(http.StatusOK, res.StatusCode, "Unexpected response: %s", http.StatusText(res.StatusCode))

	reader := res.Body
	if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(reader)
		d.Chk.NoError(err)
		defer gr.Close()
		reader = gr
	}

	scanner := bufio.NewScanner(reader)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		r := ref.Parse(scanner.Text())
		scanner.Scan()
		has := scanner.Text() == "true"
		for _, ch := range reqs[r] {
			ch <- has
		}
	}
}
Пример #15
0
func (h *HTTPStore) postRefs(chs []Chunk) {
	body := &bytes.Buffer{}
	gw := gzip.NewWriter(body)
	sz := NewSerializer(gw)
	for _, chunk := range chs {
		sz.Put(chunk)
	}
	sz.Close()
	gw.Close()

	url := *h.host
	url.Path = httprouter.CleanPath(h.host.Path + constants.PostRefsPath)
	req := h.newRequest("POST", url.String(), body, http.Header{
		"Content-Encoding": {"gzip"},
		"Content-Type":     {"application/octet-stream"},
	})

	res, err := h.httpClient.Do(req)
	d.Chk.NoError(err)

	d.Chk.Equal(res.StatusCode, http.StatusCreated, "Unexpected response: %s", http.StatusText(res.StatusCode))
	closeResponse(res)
	h.unwrittenPuts.Clear(chs)
}
Пример #16
0
// GetRouteParameter implemention will extract the param the julienschmidt way
func (h HTTPRouter) GetRouteParameter(r http.Request, param string) string {
	path := httprouter.CleanPath(r.URL.Path)
	_, params, _ := h.router.Lookup(r.Method, path)
	return params.ByName(param)
}
Пример #17
0
func (h HTTPStoreFlags) CreateStore(ns string) ChunkStore {
	if h.check() {
		return NewHTTPStore(*h.host+httprouter.CleanPath(ns), *h.auth)
	}
	return nil
}
Пример #18
0
func (f RemoteStoreFactory) CreateStore(ns string) Database {
	return NewRemoteDatabase(f.host+httprouter.CleanPath(ns), f.auth)
}