示例#1
23
文件: site.go 项目: fanyang01/crawler
func siteRoot(u *url.URL) string {
	uu := url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
	}
	return uu.String()
}
示例#2
3
文件: db.go 项目: puspesh/cdfs
func (s *Client) doGet(u url.URL) (*FileReader, error) {
	r, e := http.NewRequest("GET", u.String(), nil)
	if e != nil {
		return nil, e
	}
	return doRequest(r, s.AppToken, s.AccessToken)
}
示例#3
1
文件: deploy.go 项目: Wind0r/toni
func streamDeploymentLog(c *Configuration, deploymentLogURL *url.URL) error {
	urlStr := deploymentLogURL.String()
	wsHeaders := http.Header{"Origin": {c.Host}, apiTokenHeaderName: {c.ApiToken}}

	wsConn, _, err := websocket.DefaultDialer.Dial(urlStr, wsHeaders)
	if err != nil {
		return err
	}

	logs := make(chan deploy.LogEntry)
	defer func() {
		close(logs)
	}()

	go deploy.ConsoleLogger(logs)

	for {
		entry := deploy.LogEntry{}
		err := wsConn.ReadJSON(&entry)
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		logs <- entry
	}

	return nil
}
示例#4
1
文件: router.go 项目: zhenruyan/tsuru
func (r *fakeRouter) RemoveRoute(name string, address *url.URL) error {
	backendName, err := router.Retrieve(name)
	if err != nil {
		return err
	}
	if !r.HasBackend(backendName) {
		return router.ErrBackendNotFound
	}
	r.mutex.Lock()
	defer r.mutex.Unlock()
	if r.failuresByIp[address.String()] {
		return ErrForcedFailure
	}
	index := -1
	routes := r.backends[backendName]
	for i := range routes {
		if routes[i] == address.String() {
			index = i
			break
		}
	}
	if index < 0 {
		return router.ErrRouteNotFound
	}
	routes[index] = routes[len(routes)-1]
	r.backends[backendName] = routes[:len(routes)-1]
	return nil
}
示例#5
1
文件: deploy.go 项目: Wind0r/toni
func createDeployment(c *Configuration, deploymentURL *url.URL, data url.Values) (string, error) {
	req, err := http.NewRequest("POST", deploymentURL.String(), bytes.NewBufferString(data.Encode()))
	req.Header.Set(apiTokenHeaderName, c.ApiToken)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	resp, err := http.DefaultTransport.RoundTrip(req)
	if err != nil {
		return "", err
	}

	if resp.StatusCode != http.StatusSeeOther {
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}
		return "", UnexpectedResponse{resp.StatusCode, string(body)}
	}

	location := resp.Header.Get("Location")
	if location == "" {
		return "", fmt.Errorf("createDeployment: no Location header is response")
	}
	return location, nil
}
示例#6
1
func tcpAddr(host string) string {
	u := url.URL{
		Scheme: "tcp",
		Host:   host,
	}
	return u.String()
}
示例#7
1
文件: main.go 项目: GamerockSA/dex
func NewClientHandler(c *oidc.Client, issuer string, cbURL url.URL) http.Handler {
	mux := http.NewServeMux()

	oob := cbURL.String() == client.OOBRedirectURI

	issuerURL, err := url.Parse(issuer)
	if err != nil {
		log.Fatalf("Could not parse issuer url: %v", err)
	}

	mux.HandleFunc("/", handleIndexFunc(oob))
	mux.HandleFunc("/login", handleLoginFunc(c))
	mux.HandleFunc("/register", handleRegisterFunc(c))
	if cbURL.String() != client.OOBRedirectURI {
		mux.HandleFunc(cbURL.Path, handleCallbackFunc(c))
	} else {
		mux.HandleFunc("/callback", handleCallbackFunc(c))
	}

	resendURL := *issuerURL
	resendURL.Path = "/resend-verify-email"

	mux.HandleFunc("/resend", handleResendFunc(c, *issuerURL, resendURL, cbURL))
	return mux
}
示例#8
1
文件: apiclient.go 项目: bac/juju
// connectStream is the internal version of ConnectStream. It differs from
// ConnectStream only in that it will not retry the connection if it encounters
// discharge-required error.
func (st *state) connectStream(path string, attrs url.Values) (base.Stream, error) {
	path, err := apiPath(st.modelTag, path)
	if err != nil {
		return nil, errors.Trace(err)
	}
	target := url.URL{
		Scheme:   "wss",
		Host:     st.addr,
		Path:     path,
		RawQuery: attrs.Encode(),
	}
	cfg, err := websocket.NewConfig(target.String(), "http://localhost/")
	if st.tag != "" {
		cfg.Header = utils.BasicAuthHeader(st.tag, st.password)
	}
	if st.nonce != "" {
		cfg.Header.Set(params.MachineNonceHeader, st.nonce)
	}
	// Add any cookies because they will not be sent to websocket
	// connections by default.
	st.addCookiesToHeader(cfg.Header)

	cfg.TlsConfig = st.tlsConfig
	connection, err := websocketDialConfig(cfg)
	if err != nil {
		return nil, err
	}
	if err := readInitialStreamError(connection); err != nil {
		return nil, errors.Trace(err)
	}
	return connection, nil
}
示例#9
1
func (f *dockerFetcher) fetchImageFrom(u *url.URL, latest bool) (string, error) {
	if !f.InsecureFlags.SkipImageCheck() {
		return "", fmt.Errorf("signature verification for docker images is not supported (try --insecure-options=image)")
	}

	if f.Debug {
		log.Printf("fetching image from %s", u.String())
	}

	aciFile, err := f.fetch(u)
	if err != nil {
		return "", err
	}
	// At this point, the ACI file is removed, but it is kept
	// alive, because we have an fd to it opened.
	defer aciFile.Close()

	key, err := f.S.WriteACI(aciFile, latest)
	if err != nil {
		return "", err
	}

	// TODO(krnowak): Consider dropping the signature URL part
	// from store.Remote. It is not used anywhere and the data
	// stored here is useless.
	newRem := store.NewRemote(u.String(), ascURLFromImgURL(u).String())
	newRem.BlobKey = key
	newRem.DownloadTime = time.Now()
	err = f.S.WriteRemote(newRem)
	if err != nil {
		return "", err
	}

	return key, nil
}
示例#10
1
文件: deploy.go 项目: Wind0r/toni
func killDeployment(c *Configuration, deploymentKillURL *url.URL) error {
	req, err := http.NewRequest("POST", deploymentKillURL.String(), &bytes.Buffer{})
	req.Header.Set(apiTokenHeaderName, c.ApiToken)

	_, err = http.DefaultTransport.RoundTrip(req)
	return err
}
示例#11
1
func SignAmazonUrl(origUrl *url.URL, api AmazonProductAPI) (signedUrl string, err error) {

	escapeUrl := strings.Replace(origUrl.RawQuery, ",", "%2C", -1)
	escapeUrl = strings.Replace(escapeUrl, ":", "%3A", -1)

	params := strings.Split(escapeUrl, "&")
	sort.Strings(params)
	sortedParams := strings.Join(params, "&")

	toSign := fmt.Sprintf("GET\n%s\n%s\n%s", origUrl.Host, origUrl.Path, sortedParams)

	hasher := hmac.New(sha256.New, []byte(api.SecretKey))
	_, err = hasher.Write([]byte(toSign))
	if err != nil {
		return "", err
	}

	hash := base64.StdEncoding.EncodeToString(hasher.Sum(nil))

	hash = url.QueryEscape(hash)

	newParams := fmt.Sprintf("%s&Signature=%s", sortedParams, hash)

	origUrl.RawQuery = newParams

	return origUrl.String(), nil
}
示例#12
1
文件: client.go 项目: CNDonny/scope
func main() {
	flag.Parse()
	log.SetFlags(0)

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	u := url.URL{Scheme: "ws", Host: *addr, Path: "/echo"}
	log.Printf("connecting to %s", u.String())

	c, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
	if err != nil {
		log.Fatal("dial:", err)
	}
	defer c.Close()

	done := make(chan struct{})

	go func() {
		defer c.Close()
		defer close(done)
		for {
			_, message, err := c.ReadMessage()
			if err != nil {
				log.Println("read:", err)
				return
			}
			log.Printf("recv: %s", message)
		}
	}()

	ticker := time.NewTicker(time.Second)
	defer ticker.Stop()

	for {
		select {
		case t := <-ticker.C:
			err := c.WriteMessage(websocket.TextMessage, []byte(t.String()))
			if err != nil {
				log.Println("write:", err)
				return
			}
		case <-interrupt:
			log.Println("interrupt")
			// To cleanly close a connection, a client should send a close
			// frame and wait for the server to close the connection.
			err := c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
			if err != nil {
				log.Println("write close:", err)
				return
			}
			select {
			case <-done:
			case <-time.After(time.Second):
			}
			c.Close()
			return
		}
	}
}
示例#13
1
func (f *dockerFetcher) fetch(u *url.URL) (*os.File, error) {
	tmpDir, err := f.getTmpDir()
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(tmpDir)

	registryURL := strings.TrimPrefix(u.String(), "docker://")
	user, password := f.getCreds(registryURL)
	config := docker2aci.RemoteConfig{
		Username: user,
		Password: password,
		Insecure: f.InsecureFlags.AllowHTTP(),
		CommonConfig: docker2aci.CommonConfig{
			Squash:      true,
			OutputDir:   tmpDir,
			TmpDir:      tmpDir,
			Compression: d2acommon.NoCompression,
		},
	}
	acis, err := docker2aci.ConvertRemoteRepo(registryURL, config)
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error converting docker image to ACI"), err)
	}

	aciFile, err := os.Open(acis[0])
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error opening squashed ACI file"), err)
	}

	return aciFile, nil
}
示例#14
1
文件: client.go 项目: cgenuity/dex
func (c Client) ValidRedirectURL(u *url.URL) (url.URL, error) {
	if c.Public {
		if u == nil {
			return url.URL{}, ErrorInvalidRedirectURL
		}
		if u.String() == OOBRedirectURI {
			return *u, nil
		}

		if u.Scheme != "http" {
			return url.URL{}, ErrorInvalidRedirectURL
		}

		hostPort := strings.Split(u.Host, ":")
		if len(hostPort) != 2 {
			return url.URL{}, ErrorInvalidRedirectURL
		}

		if hostPort[0] != "localhost" || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.Fragment != "" {
			return url.URL{}, ErrorInvalidRedirectURL
		}

		return *u, nil
	}

	return ValidRedirectURL(u, c.Metadata.RedirectURIs)
}
示例#15
0
func (o *OpenTsdbOutput) Init(config interface{}) (err error) {
	o.OpenTsdbOutputConfig = config.(*OpenTsdbOutputConfig)
	//todo: check address validity
	// if o.url, err = url.Parse(o.Address); err != nil {
	//     return fmt.Errorf("Can't parse URL '%s': %s", o.Address, err.Error())
	// }
	//
	o.client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}

	var u *url.URL
	if u, err = url.Parse(o.Url); err == nil {
	}

	o.logMsgChan = make(chan []byte, o.LogMsgChSize)

	u, err = u.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	o.Url = u.String()

	if err != nil {
		log.Printf("initialize OpenTsdbOutput failed, %s", err.Error())
		return err
	}
	return
}
示例#16
0
文件: api.go 项目: jkassemi/gochimp
// see https://mandrillapp.com/api/docs/
// currently supporting json output formats
func NewMandrill(apiKey string) (*MandrillAPI, error) {
	u := url.URL{}
	u.Scheme = "https"
	u.Host = mandrill_uri
	u.Path = mandrill_version
	return &MandrillAPI{apiKey, u.String()}, nil
}
示例#17
0
func (cfg *OIDCConnectorConfig) Connector(ns url.URL, lf oidc.LoginFunc, tpls *template.Template) (Connector, error) {
	ns.Path = path.Join(ns.Path, httpPathCallback)

	ccfg := oidc.ClientConfig{
		RedirectURL: ns.String(),
		Credentials: oidc.ClientCredentials{
			ID:     cfg.ClientID,
			Secret: cfg.ClientSecret,
		},
	}

	cl, err := oidc.NewClient(ccfg)
	if err != nil {
		return nil, err
	}

	idpc := &OIDCConnector{
		id:                   cfg.ID,
		issuerURL:            cfg.IssuerURL,
		cbURL:                ns,
		loginFunc:            lf,
		client:               cl,
		trustedEmailProvider: cfg.TrustedEmailProvider,
		emailClaim:           cfg.EmailClaim,
	}
	return idpc, nil
}
示例#18
0
func getGameList(scheme, host string) (gameList []*Game, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("%v", r)
		}
	}()
	u := url.URL{Scheme: scheme, Host: host}
	u.Path = "games/"
	resp, err := client(*useTLS).Get(u.String())
	if err != nil {
		return gameList, err
	}
	defer resp.Body.Close()
	contents := make([]byte, 131072) // should be enough, but will fail poorly if game list is longer
	n, err := resp.Body.Read(contents)
	if err != nil && err != io.EOF {
		return gameList, err
	}
	gameList = make([]*Game, 0)
	err = json.Unmarshal(contents[:n], &gameList)
	if err != nil {
		return make([]*Game, 0), err
	}
	return gameList, err
}
示例#19
0
文件: http.go 项目: CNDonny/scope
// DoHTTPProbe checks if a GET request to the url succeeds.
// If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
// If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
// This is exported because some other packages may want to do direct HTTP probes.
func DoHTTPProbe(url *url.URL, headers http.Header, client HTTPGetInterface) (probe.Result, string, error) {
	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		// Convert errors into failures to catch timeouts.
		return probe.Failure, err.Error(), nil
	}
	req.Header = headers
	res, err := client.Do(req)
	if err != nil {
		// Convert errors into failures to catch timeouts.
		return probe.Failure, err.Error(), nil
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return probe.Failure, "", err
	}
	body := string(b)
	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
		glog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res)
		return probe.Success, body, nil
	}
	glog.V(4).Infof("Probe failed for %s with request headers %v, response body: %v", url.String(), headers, body)
	return probe.Failure, fmt.Sprintf("HTTP probe failed with statuscode: %d", res.StatusCode), nil
}
示例#20
0
文件: tsdb.go 项目: Skyscanner/bosun
// QueryResponse performs a v2 OpenTSDB request to the given host. host should
// be of the form hostname:port. A nil client uses DefaultClient.
func (r *Request) QueryResponse(host string, client *http.Client) (*http.Response, error) {
	u := url.URL{
		Scheme: "http",
		Host:   host,
		Path:   "/api/query",
	}
	b, err := json.Marshal(&r)
	if err != nil {
		return nil, err
	}
	if client == nil {
		client = DefaultClient
	}
	resp, err := client.Post(u.String(), "application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		e := RequestError{Request: string(b)}
		defer resp.Body.Close()
		body, _ := ioutil.ReadAll(resp.Body)
		if err := json.NewDecoder(bytes.NewBuffer(body)).Decode(&e); err == nil {
			return nil, &e
		}
		s := fmt.Sprintf("opentsdb: %s", resp.Status)
		if len(body) > 0 {
			s = fmt.Sprintf("%s: %s", s, body)
		}
		return nil, errors.New(s)
	}
	return resp, nil
}
示例#21
0
func (p *Page) Fetch(token string) error {
	u := url.URL{
		Path:     p.plan.Path,
		RawQuery: p.plan.Query.Encode(),
	}

	resp, err := newNetworkClient(p.config).MakeRequest(network.Request{
		Method:                "GET",
		Path:                  u.String(),
		Authorization:         network.NewTokenAuthorization(token),
		AcceptableStatusCodes: []int{http.StatusOK},
	})
	if err != nil {
		return err
	}

	var response documents.PageResponse
	err = json.Unmarshal(resp.Body, &response)
	if err != nil {
		panic(err)
	}

	updatedPage := newPageFromResponse(p.config, p.plan, response)
	p.TotalResults = updatedPage.TotalResults
	p.TotalPages = updatedPage.TotalPages
	p.NextURL = updatedPage.NextURL
	p.PrevURL = updatedPage.PrevURL
	p.Resources = updatedPage.Resources

	return nil
}
示例#22
0
文件: session.go 项目: SemQuery/web
func Login(session sessions.Session, re render.Render, r *http.Request) {
	client_id := common.Config.OAuth2Client_ID

	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

	b := make([]rune, 10)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	session.AddFlash(string(b), "state")

	redirectBack := r.URL.Query().Get("redirect_back")
	ref := r.Referer()

	if redirectBack == "true" && ref != "" {
		session.Set("redirect_to", ref)
	} else {
		session.Set("redirect_to", nil)
	}

	query := url.Values{}
	query.Set("client_id", client_id)
	query.Set("state", string(b))
	query.Set("scope", "repo")

	dest := url.URL{
		Scheme:   "https",
		Host:     "github.com",
		Path:     "/login/oauth/authorize",
		RawQuery: query.Encode(),
	}
	re.Redirect(dest.String())
}
示例#23
0
func Detect_language(q string) string {
	var Url *url.URL
	Url, err := url.Parse("https://www.googleapis.com")
	if err != nil {
		fmt.Println(err)
	}

	Url.Path += "/language/translate/v2/detect"
	parameters := url.Values{}
	parameters.Add("q", q)
	parameters.Add("key", Google_key())
	Url.RawQuery = parameters.Encode()

	resp, err := http.Get(Url.String())
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	var data gtresponse
	json.Unmarshal(body, &data)

	lang := data.Data.Detections[0][0].Language
	return lang
}
示例#24
0
// Address constructs a URL from the given network and hostname.
func Address(network, host string) string {
	u := url.URL{
		Scheme: network,
		Host:   host,
	}
	return u.String()
}
示例#25
0
func (c *ShuttleConfig) OutletURL() string {
	var err error
	var oUrl *url.URL

	if len(c.LogsURL) > 0 {
		oUrl, err = url.Parse(c.LogsURL)
		if err != nil {
			log.Fatalf("Unable to parse logs-url")
		}
	}
	if len(LogplexUrl) > 0 {
		oUrl, err = url.Parse(LogplexUrl)
		if err != nil {
			log.Fatalf("Unable to parse $LOGPLEX_URL")
		}
	}

	if oUrl == nil {
		log.Fatalf("Must set -logs-url or $LOGPLEX_URL.")
	}

	if oUrl.User == nil {
		oUrl.User = url.UserPassword("token", c.Appname)
	}
	return oUrl.String()
}
示例#26
0
文件: oidc_test.go 项目: Tecsisa/dex
func verifyUserClaims(claims jose.Claims, ci *client.Client, user *user.User, issuerURL url.URL) error {
	expectedSub, expectedName := ci.Credentials.ID, ci.Credentials.ID
	if user != nil {
		expectedSub, expectedName = user.ID, user.DisplayName
	}

	if aud, ok := claims["aud"].(string); !ok {
		return fmt.Errorf("unexpected claim value for aud, got=nil, want=%v", ci.Credentials.ID)
	} else if aud != ci.Credentials.ID {
		return fmt.Errorf("unexpected claim value for aud, got=%v, want=%v", aud, ci.Credentials.ID)
	}

	if sub, ok := claims["sub"].(string); !ok {
		return fmt.Errorf("unexpected claim value for sub, got=nil, want=%v", expectedSub)
	} else if sub != expectedSub {
		return fmt.Errorf("unexpected claim value for sub, got=%v, want=%v", sub, expectedSub)
	}

	if name, ok := claims["name"].(string); !ok {
		return fmt.Errorf("unexpected claim value for aud, got=nil, want=%v", expectedName)
	} else if name != expectedName {
		return fmt.Errorf("unexpected claim value for name, got=%v, want=%v", name, expectedName)
	}

	wantIss := issuerURL.String()
	if iss := claims["iss"].(string); iss != wantIss {
		return fmt.Errorf("unexpected claim value for iss, got=%v, want=%v", iss, wantIss)
	}

	return nil
}
示例#27
0
func (a *attempt) fillReferences(workUnit *workUnit, worker *worker) error {
	var err error
	var url *url.URL

	if err == nil {
		url, err = a.Template(a.Representation.WorkUnitURL, map[string]interface{}{})
	}
	if err == nil && workUnit != nil && workUnit.URL.String() == url.String() {
		a.workUnit = workUnit
	}
	if err == nil && a.workUnit == nil {
		a.workUnit, err = workUnitFromURL(&a.resource, a.Representation.WorkUnitURL, nil)
	}

	if err == nil {
		url, err = a.Template(a.Representation.WorkerURL, map[string]interface{}{})
	}
	if err == nil && worker != nil && worker.URL.String() == url.String() {
		a.worker = worker
	}
	if err == nil && a.worker == nil {
		a.worker, err = workerFromURL(&a.resource, a.Representation.WorkerURL)
	}

	return err
}
示例#28
0
func (ea *endpointAuthorizer) ModifyRequest(req *http.Request) error {
	v2Root := strings.Index(req.URL.Path, "/v2/")
	if v2Root == -1 {
		return nil
	}

	ping := url.URL{
		Host:   req.URL.Host,
		Scheme: req.URL.Scheme,
		Path:   req.URL.Path[:v2Root+4],
	}

	pingEndpoint := ping.String()

	challenges, err := ea.challenges.GetChallenges(pingEndpoint)
	if err != nil {
		return err
	}

	if len(challenges) > 0 {
		for _, handler := range ea.handlers {
			for _, challenge := range challenges {
				if challenge.Scheme != handler.Scheme() {
					continue
				}
				if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
示例#29
0
func (b *BlobStore) doListRequest(baseURL *url.URL) (listResponse, error) {
	req, err := http.NewRequest("PROPFIND", baseURL.String(), nil)
	if err != nil {
		return listResponse{}, err
	}

	req.Header.Add("Depth", "1")

	resp, err := b.Client.Do(req)
	if err != nil {
		return listResponse{}, err
	}

	defer resp.Body.Close()

	if resp.StatusCode != 207 {
		return listResponse{}, errors.New(resp.Status)
	}

	decoder := xml.NewDecoder(resp.Body)

	var listResp listResponse
	if err := decoder.Decode(&listResp); err != nil {
		return listResponse{}, err
	}

	return listResp, nil
}
示例#30
0
文件: pages.go 项目: 5webvn/crawler
func (p Pages) Value(key *url.URL) *Page {
	ret := p.safeMap.Value(key.String())
	if ret == nil {
		return nil
	}
	return ret.(*Page)
}