func lookup(uri *url.URL, q string, suffix bool) (refResult *RefererResult) {
refResult = &RefererResult{URI: uri, Medium: "unknown"}
for medium, mediumData := range data {
for refName, refconfig := range mediumData {
for _, domain := range refconfig["domains"] {
if (!suffix && q == domain) || (suffix && (strings.HasSuffix(q, domain) || strings.HasPrefix(q, domain))) {
refResult.Known = true
refResult.Referer = refName
refResult.Medium = medium
params, paramExists := refconfig["parameters"]
if paramExists {
for _, param := range params {
sterm := uri.Query().Get(param)
if sterm != "" {
refResult.SearchParameter = param
refResult.SearchTerm = sterm
}
}
}
return refResult
}
}
}
}
return
}
func (d *UDPClient) Start(uri *url.URL) error {
d.url = uri
d.stop = make(chan struct{})
params := uri.Query()
// The address must not have a port, as otherwise both announce and lookup
// sockets would try to bind to the same port.
addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
if err != nil {
return err
}
d.listenAddress = addr
broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
if err != nil {
d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
} else {
d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
}
retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
if err != nil {
d.errorRetryInterval = DefaultErrorRetryInternval
} else {
d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
}
d.wg.Add(1)
go d.broadcast()
return nil
}
func sortQuery(u *url.URL) {
q := u.Query()
if len(q) == 0 {
return
}
arKeys := make([]string, len(q))
i := 0
for k, _ := range q {
arKeys[i] = k
i++
}
sort.Strings(arKeys)
buf := new(bytes.Buffer)
for _, k := range arKeys {
sort.Strings(q[k])
for _, v := range q[k] {
if buf.Len() > 0 {
buf.WriteRune('&')
}
buf.WriteString(fmt.Sprintf("%s=%s", k, url.QueryEscape(v)))
}
}
// Rebuild the raw query string
u.RawQuery = buf.String()
}
func getConfigOverrides(uri *url.URL) (*kube_client_cmd.ConfigOverrides, error) {
kubeConfigOverride := kube_client_cmd.ConfigOverrides{
ClusterInfo: kube_client_cmd_api.Cluster{
APIVersion: APIVersion,
},
}
if len(uri.Scheme) != 0 && len(uri.Host) != 0 {
kubeConfigOverride.ClusterInfo.Server = fmt.Sprintf("%s://%s", uri.Scheme, uri.Host)
}
opts := uri.Query()
if len(opts["apiVersion"]) >= 1 {
kubeConfigOverride.ClusterInfo.APIVersion = opts["apiVersion"][0]
}
if len(opts["insecure"]) > 0 {
insecure, err := strconv.ParseBool(opts["insecure"][0])
if err != nil {
return nil, err
}
kubeConfigOverride.ClusterInfo.InsecureSkipTLSVerify = insecure
}
return &kubeConfigOverride, nil
}
func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error {
if err := conn.Handshake(); err != nil {
return err
}
cs := conn.ConnectionState()
if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName {
return fmt.Errorf("protocol negotiation error")
}
q := uri.Query()
relayIDs := q.Get("id")
if relayIDs != "" {
relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs)
if err != nil {
return fmt.Errorf("relay address contains invalid verification id: %s", err)
}
certs := cs.PeerCertificates
if cl := len(certs); cl != 1 {
return fmt.Errorf("unexpected certificate count: %d", cl)
}
remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw)
if remoteID != relayID {
return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID)
}
}
return nil
}
func getCursor(u *url.URL) interface{} {
vals, ok := u.Query()["next"]
if !ok {
return nil
}
return vals[0]
}
func DownloadOneStockPerDay(symbol string, date string) error {
u := url.URL{Scheme: "http", Host: "market.finance.sina.com.cn", Path: "/downxls.php"}
q := u.Query()
q.Set("date", date)
q.Set("symbol", symbol)
u.RawQuery = q.Encode()
log.Println(u.String())
td := GetStockInOneDay(u.String())
if len(td) > 0 {
var ops DBOps
ops.Init()
err := ops.Open()
if err != nil {
log.Println("open the database failed.")
log.Fatal(err)
}
defer ops.Close()
ops.CopyDataIntoTable(td)
} else {
log.Printf("%s:%s#No DATA\n", symbol, date)
}
return nil
}
func GetNameBySymbol(s string) (string, bool) {
u := url.URL{Scheme: "http", Host: "hq.sinajs.cn", Path: "/"}
q := u.Query()
q.Set("list", s)
u.RawQuery = q.Encode()
log.Println(u.String())
res, err := http.Get(u.String())
if err != nil {
log.Println(err.Error())
return "", false
}
body, err := ioutil.ReadAll(res.Body)
defer res.Body.Close()
if err != nil {
log.Println(err.Error())
return "", false
}
// log.Println(GBKtoUTF8(string(body)))
content := string(body)
if strings.Contains(content, ",") != true {
log.Println("Invalid symbol")
return "", false
}
name := content[strings.Index(content, "\"")+1 : strings.Index(content, ",")]
log.Println(GBKtoUTF8(name))
// log.Printf("length:%d\n", len(name))
return GBKtoUTF8(name), true
}
func TestRefreshParam(t *testing.T) {
var requrl *url.URL
InitTests(true)
c := NewTestConn()
c.RequestTracer = func(method, urlStr, body string) {
requrl, _ = url.Parse(urlStr)
}
date := time.Unix(1257894000, 0)
data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
// Now tests small batches
indexer := c.NewBulkIndexer(1)
indexer.Refresh = true
indexer.Start()
<-time.After(time.Millisecond * 20)
indexer.Index("users", "user", "2", "", "", &date, data)
<-time.After(time.Millisecond * 200)
// indexer.Flush()
indexer.Stop()
assert.T(t, requrl.Query().Get("refresh") == "true", "Should have set refresh query param to true")
}
func hotpFromURL(u *url.URL) (*HOTP, string, error) {
label := u.Path[1:]
v := u.Query()
secret := v.Get("secret")
if secret == "" {
return nil, "", ErrInvalidURL
}
var digits = 6
if sdigit := v.Get("digits"); sdigit != "" {
tmpDigits, err := strconv.ParseInt(sdigit, 10, 8)
if err != nil {
return nil, "", err
}
digits = int(tmpDigits)
}
var counter uint64 = 0
if scounter := v.Get("counter"); scounter != "" {
var err error
counter, err = strconv.ParseUint(scounter, 10, 64)
if err != nil {
return nil, "", err
}
}
key, err := base32.StdEncoding.DecodeString(secret)
if err != nil {
return nil, "", err
}
otp := NewHOTP(key, counter, digits)
return otp, label, nil
}
func (lp *LinkParser) Parse(newPost *post.Post, raw []byte) post.Flow {
if !lp.ready {
lp.ready = true
return post.Continue
}
if bytes.HasPrefix(raw, []byte("post/view?")) {
newPost.Link = "http://www.javaworld.com.tw/jute/" + string(raw)
var postLink *url.URL
postLink, lp.err = url.Parse(newPost.Link)
if lp.err != nil {
return post.Error
}
values := postLink.Query()
newPost.Id, lp.err = strconv.Atoi(values.Get("id"))
if lp.err != nil {
return post.Error
}
newPost.Bid, lp.err = strconv.Atoi(values.Get("bid"))
if lp.err != nil {
return post.Error
}
return post.Stop
}
return post.Continue
}
func queryValue(u *url.URL, key string) string {
vals, ok := u.Query()[key]
if !ok || len(vals) != 1 {
return ""
}
return vals[0]
}
func parsePrefix(u *url.URL) string {
t := u.Query().Get("prefix")
if t != "" {
return t
}
return SyslogPrefix
}
func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) {
api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2013-08-15", true)
c.Assert(err, chk.IsNil)
cli := api.GetBlobService()
expiry := time.Time{}
expectedParts := url.URL{
Scheme: "https",
Host: "foo.blob.core.windows.net",
Path: "container/name",
RawQuery: url.Values{
"sv": {"2013-08-15"},
"sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="},
"sr": {"b"},
"sp": {"r"},
"se": {"0001-01-01T00:00:00Z"},
}.Encode()}
u, err := cli.GetBlobSASURI("container", "name", expiry, "r")
c.Assert(err, chk.IsNil)
sasParts, err := url.Parse(u)
c.Assert(err, chk.IsNil)
c.Assert(expectedParts.String(), chk.Equals, sasParts.String())
c.Assert(expectedParts.Query(), chk.DeepEquals, sasParts.Query())
}
func queryHTTPTracker(infoHashes []string, u *url.URL) (tr []InfoHashDetails, err error) {
uq := u.Query()
for _, infoHash := range infoHashes {
uq.Add("info_hash", infoHash)
}
log.Println("HTTP INFO HASH DEBUG REMOVE THIS", uq)
// Don't report IPv6 address, the user might prefer to keep
// that information private when communicating with IPv4 hosts.
if false {
ipv6Address, err := findLocalIPV6AddressFor(u.Host)
if err == nil {
log.Println("our ipv6", ipv6Address)
uq.Add("ipv6", ipv6Address)
}
}
// This might reorder the existing query string in the Announce url
// This might break some broken trackers that don't parse URLs properly.
u.RawQuery = uq.Encode()
tr, err = getTrackerInfo(u.String())
if tr == nil || err != nil {
log.Println("Error: Could not fetch tracker info:", err)
}
return
}
func NewCursorFromUrl(u *url.URL, defaults *Defaults, options *Options) *Cursor {
c := NewCursorFromDefaultsAndOptions(defaults, options)
c.Url = u
values := u.Query()
if v, ok := values["value"]; ok {
c.Value = v[0]
}
if v, ok := values["offset"]; ok {
offset, err := strconv.Atoi(v[0])
if err != nil {
c.Offset = offset
}
}
if v, ok := values["count"]; ok {
count, err := strconv.Atoi(v[0])
if err == nil {
c.Count = count
}
}
if v, ok := values["order"]; ok {
c.Order = v[0]
}
if v, ok := values["direction"]; ok {
direction, err := strconv.Atoi(v[0])
if err == nil && (direction == ASC || direction == DESC) {
c.Direction = direction
}
}
return c
}
func TestAuthCode(t *testing.T) {
var callbackURL *url.URL
router := mux.NewRouter()
ts := httptest.NewUnstartedServer(router)
callbackCalled := false
handler.SetRoutes(router, mockAuthorization("", new(jwt.Token)))
router.HandleFunc("/remote/oauth2/auth", authHandlerMock(t, ts))
router.HandleFunc("/callback", func(w http.ResponseWriter, r *http.Request) {
callbackURL = r.URL
callbackCalled = true
})
ts.Start()
defer ts.Close()
for _, c := range []struct{ config *oauth2.Config }{{configs["working"]}} {
config := *c.config
config.Endpoint = oauth2.Endpoint{AuthURL: ts.URL + "/oauth2/auth?provider=mockprovider", TokenURL: ts.URL + "/oauth2/token"}
authURL := config.AuthCodeURL(uuid.New())
t.Logf("Auth code URL: %s", authURL)
resp, err := http.Get(authURL)
require.Nil(t, err)
defer resp.Body.Close()
require.True(t, callbackCalled)
callbackCalled = false
token, err := config.Exchange(oauth2.NoContext, callbackURL.Query().Get("code"))
require.Nil(t, err)
require.NotEmpty(t, token.AccessToken)
require.NotEmpty(t, token.RefreshToken)
testTokenRefresh(t, ts.URL, config.ClientID, config.ClientSecret, token.RefreshToken, true)
}
}
func removeQueryArgument(u *url.URL, arguments ...string) {
query := u.Query()
for _, argument := range arguments {
query.Del(argument)
}
u.RawQuery = query.Encode()
}
func (s *SocialTencent) UserInfo(token *oauth.Token, URL *url.URL) (*BasicUserInfo, error) {
var data struct {
Data struct {
Id string `json:"openid"`
Name string `json:"name"`
Email string `json:"email"`
} `json:"data"`
}
var err error
// https://open.t.qq.com/api/user/info?
//oauth_consumer_key=APP_KEY&
//access_token=ACCESSTOKEN&openid=openid
//clientip=CLIENTIP&oauth_version=2.a
//scope=all
var urls = url.Values{
"oauth_consumer_key": {s.Transport.Config.ClientId},
"access_token": {token.AccessToken},
"openid": URL.Query()["openid"],
"oauth_version": {"2.a"},
"scope": {"all"},
}
r, err := http.Get(s.reqUrl + "?" + urls.Encode())
if err != nil {
return nil, err
}
defer r.Body.Close()
if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
return nil, err
}
return &BasicUserInfo{
Identity: data.Data.Id,
Name: data.Data.Name,
Email: data.Data.Email,
}, nil
}
// parseParams parses the parameters from the URL, and uses defaultParams if
// there are missing values.
func parseParams(href *url.URL) parameters {
params := defaultParams
path, ext := splitPath(href.Path)
if len(path) >= 1 {
params.name = path[0]
}
if len(path) >= 2 {
params.depth = path[1]
}
if ext == "svg" {
params.onlySVG = true
}
query := href.Query()
if thickness := query.Get("t"); thickness != "" {
params.thickness = thickness
}
if color := query.Get("c"); color != "" {
params.color = color
}
if precision := query.Get("p"); precision != "" {
params.precision = precision
}
return params
}
// resourceForURL returns a resource object for the given URL.
func (srv *SwiftServer) resourceForURL(u *url.URL) (r resource) {
m := pathRegexp.FindStringSubmatch(u.Path)
if m == nil {
fatalf(404, "InvalidURI", "Couldn't parse the specified URI")
}
containerName := m[2]
objectName := m[4]
if containerName == "" {
return rootResource{}
}
b := containerResource{
name: containerName,
container: srv.Containers[containerName],
}
if objectName == "" {
return b
}
if b.container == nil {
fatalf(404, "NoSuchContainer", "The specified container does not exist")
}
objr := objectResource{
name: objectName,
version: u.Query().Get("versionId"),
container: b.container,
}
if obj := objr.container.objects[objr.name]; obj != nil {
objr.object = obj
}
return objr
}
// URL returns redirect url for application authentication
func (a Auth) URL() string {
u := url.URL{}
u.Host = oauthHost
u.Scheme = oauthScheme
u.Path = oauthPath
if len(a.RedirectURI) == 0 {
a.RedirectURI = oauthRedirectURI
}
if len(a.ResponseType) == 0 {
a.ResponseType = oauthResponseType
}
if len(a.Display) == 0 {
a.Display = oauthDisplay
}
values := u.Query()
values.Add(paramResponseType, a.ResponseType)
values.Add(paramScope, a.Scope.String())
values.Add(paramAppID, int64s(a.ID))
values.Add(paramRedirectURI, a.RedirectURI)
values.Add(paramVersion, defaultVersion)
values.Add(paramDisplay, a.Display)
u.RawQuery = values.Encode()
return u.String()
}
func NewGraphiteSink(uri *url.URL) (core.DataSink, error) {
host, portString, err := net.SplitHostPort(uri.Host)
if err != nil {
return nil, err
}
if host == "" {
host = DefaultHost
}
port := DefaultPort
if portString != "" {
if port, err = strconv.Atoi(portString); err != nil {
return nil, err
}
}
prefix := uri.Query().Get("prefix")
if prefix == "" {
prefix = DefaultPrefix
}
client, err := graphite.GraphiteFactory(uri.Scheme, host, port, prefix)
if err != nil {
return nil, err
}
return &Sink{client: client}, nil
}
// GetCommandURI return uri part which created by command
func (c *Command) GetCommandURI(m *model.APIMessage, text string) string {
u := url.URL{}
q := u.Query()
q.Set("chat_id", strconv.FormatInt(m.Message.Chat.ID, 10))
q.Set("text", text)
return "/" + c.GetName() + "?" + q.Encode()
}
// filterURL raises an error if the provided url.URL has any
// questionable attributes
func filterURL(u *url.URL) error {
if !(u.Scheme == "http" || u.Scheme == "https") {
return fmt.Errorf("unable to use endpoint scheme %s, http/https only", u.Scheme)
}
if u.Path != "/" {
return fmt.Errorf("unable to use endpoint with non-root path: %s", u)
}
if len(u.Query()) > 0 {
return fmt.Errorf("unable to use endpoint with query parameters: %s", u)
}
if len(u.Opaque) > 0 {
return fmt.Errorf("malformed endpoint: %s", u)
}
if u.User != nil {
return fmt.Errorf("unable to use endpoint with user info: %s", u)
}
if len(u.Fragment) > 0 {
return fmt.Errorf("unable to use endpoint with fragment: %s", u)
}
return nil
}
func newFilters(u url.URL, whitelist []string) []Filter {
var filters []Filter
queryValues := u.Query()
for _, key := range whitelist {
var filtersToAdd []Filter
filterStr := queryValues.Get(key)
if filterStr != "" {
fieldName := translation.PropertyToField(key)
csv := strings.Split(filterStr, ",")
for _, s := range csv {
if strings.HasPrefix(s, "gt:") {
filtersToAdd = append(filtersToAdd, Filter{Field: fieldName, Operator: ">", Value: s[3:len(s)]})
} else if strings.HasPrefix(s, "gte:") {
filtersToAdd = append(filtersToAdd, Filter{Field: fieldName, Operator: ">=", Value: s[4:len(s)]})
} else if strings.HasPrefix(s, "lt:") {
filtersToAdd = append(filtersToAdd, Filter{Field: fieldName, Operator: "<", Value: s[3:len(s)]})
} else if strings.HasPrefix(s, "lte:") {
filtersToAdd = append(filtersToAdd, Filter{Field: fieldName, Operator: "<=", Value: s[4:len(s)]})
}
}
// No comparison operator so default to equals
if len(filtersToAdd) == 0 {
filtersToAdd = append(filtersToAdd, Filter{Field: fieldName, Operator: "=", Value: filterStr})
}
filters = append(filters, filtersToAdd...)
}
}
return filters
}
func Test_buildRequestUrl(t *testing.T) {
user, _ := user.Current()
url1 := url.URL{Scheme: "http", Host: "localhost:8080", Path: "/webhdfs/v1/test"}
q := url1.Query()
q.Set("user.name", user.Username)
url1.RawQuery = q.Encode()
conf := Configuration{Addr: url1.Host}
u, err := buildRequestUrl(conf, &Path{Name: "/test"}, nil)
if err != nil {
t.Fatal(err)
}
if url1 != *u {
t.Errorf("Expecting url [%v], but got [%v]", url1.String(), u.String())
}
// test with params
v := url.Values{}
v.Add("op1", "OP_1")
v.Add("op2", "OP_2")
v.Add("user.name", user.Username)
url1.RawQuery = v.Encode()
params := map[string]string{
"op1": "OP_1",
"op2": "OP_2",
}
u, err = buildRequestUrl(conf, &Path{Name: "/test"}, ¶ms)
if url1 != *u {
t.Errorf("Expecting url [%v], but got [%v]", url1.String(), u.String())
}
}
// Download and parse OAuth2 JSON access request
func downloadData(method string, u *url.URL, auth *BasicAuth, transport http.RoundTripper, output ResponseData) error {
var postdata url.Values
var body io.Reader
var contenttype string
if method == "POST" {
// Convert query parameters to post data
postdata = u.Query()
u.RawQuery = ""
body = strings.NewReader(postdata.Encode())
contenttype = "application/x-www-form-urlencoded"
}
// Create a new request
preq, err := http.NewRequest(method, u.String(), body)
if err != nil {
return err
}
if auth != nil {
preq.SetBasicAuth(auth.Username, auth.Password)
}
// Set content type for post request
if contenttype != "" {
preq.Header.Set("Content-Type", contenttype)
}
// Explicitly set accept header to JSON
preq.Header.Set("Accept", "application/json")
// do request
client := &http.Client{}
if transport != nil {
client.Transport = transport
}
presp, err := client.Do(preq)
if err != nil {
return err
}
// must close body
defer presp.Body.Close()
// decode JSON and detect OAuth error
jdec := json.NewDecoder(presp.Body)
if err = jdec.Decode(&output); err == nil {
if em, eok := output["error"]; eok {
return NewError(fmt.Sprintf("%v", em), fmt.Sprintf("%v", output["error_description"]),
fmt.Sprintf("%v", output["error_uri"]), fmt.Sprintf("%v", output["state"]))
}
}
// If no OAuth error was detected, make sure we don't return success in an error case
if err == nil && presp.StatusCode != 200 {
return errors.New(fmt.Sprintf("Invalid status code (%d): %s", presp.StatusCode, presp.Status))
}
return err
}
// Parses the querystring (q= parameter) and moves special tokens such as
// "lang:c" from the querystring into separate arguments.
func RewriteQuery(u url.URL) url.URL {
// query is a copy which we will modify using Set() and use in the result
query := u.Query()
querystr := query.Get("q")
queryWords := []string{}
for _, word := range strings.Split(querystr, " ") {
fmt.Printf("word = %v\n", word)
lower := strings.ToLower(word)
if strings.HasPrefix(lower, "filetype:") {
query.Add("filetype", strings.ToLower(word[len("filetype:"):]))
} else if strings.HasPrefix(lower, "-filetype:") {
query.Add("nfiletype", strings.ToLower(word[len("-filetype:"):]))
} else if strings.HasPrefix(lower, "package:") {
query.Set("package", word[len("package:"):])
} else if strings.HasPrefix(lower, "pkg:") {
query.Set("package", word[len("pkg:"):])
} else if strings.HasPrefix(lower, "-package:") {
query.Add("npackage", word[len("-package:"):])
} else if strings.HasPrefix(lower, "-pkg:") {
query.Add("npackage", word[len("-pkg:"):])
} else if strings.HasPrefix(lower, "path:") || strings.HasPrefix(lower, "file:") {
query.Add("path", word[len("path:"):])
} else if strings.HasPrefix(lower, "-path:") || strings.HasPrefix(lower, "-file:") {
query.Add("npath", word[len("-path:"):])
} else {
queryWords = append(queryWords, word)
}
}
query.Set("q", strings.Join(queryWords, " "))
u.RawQuery = query.Encode()
return u
}
func ForwardToErrorHandler(w http.ResponseWriter, r *http.Request, err error, errorHandlerURL url.URL) {
q := errorHandlerURL.Query()
q.Set("error", err.Error())
errorHandlerURL.RawQuery = q.Encode()
http.Redirect(w, r, errorHandlerURL.String(), http.StatusFound)
}