Exemplo n.º 1
0
func parse(s string, base url.URL) (ln []string, as []string) {
	z := html.NewTokenizer(strings.NewReader(s))
	lnm := make(map[string]struct{})
	asm := make(map[string]struct{})

	// anonymous func used to get attribute values
	attr := func(a string) string {
		var av string
		for {
			if k, v, ha := z.TagAttr(); string(k) == a {
				av = string(v)
				break
			} else if ha == false {
				break
			}
		}
		return av
	}

	// convert map to slice
	slc := func(m map[string]struct{}) []string {
		var v []string

		for k := range m {
			v = append(v, k)
		}
		sort.Strings(v)
		return v
	}

	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			return slc(lnm), slc(asm)
		case html.StartTagToken, html.EndTagToken:
			if tn, ha := z.TagName(); ha {
				tg := string(tn)
				if av := attr(attrs[tg]); av != "" {
					switch tg {
					case atag:
						if url, err := base.Parse(av); err == nil {
							if url.Host == base.Host && url.RequestURI() != base.RequestURI() {
								lnm[url.RequestURI()] = struct{}{}
							}
						}
					case stag, itag, ltag:
						asm[av] = struct{}{}
					}
				}
			}
		}
	}
}
Exemplo n.º 2
0
func Fetch(imageUrl string) ([]byte, error) {
	url, err := url.Parse(imageUrl)
	if err != nil {
		return nil, fmt.Errorf("Invalid image URL: (url=%s)", url.RequestURI())
	}
	return fetchImage(url)
}
Exemplo n.º 3
0
func regKnownDSNTranslators() {

	// TODO
	knownDrivers := map[string]func(*url.URL) string{
		"postgres": func(url *url.URL) string {
			password, _ := url.User.Password()
			return fmt.Sprintf("user=%s password=%s dbname=%s %s", url.User.Username(), password, strings.TrimPrefix(url.Path, "/"), url.RawQuery)
		},
		// "sqlite3": func(url *url.URL) string { return url.Path },
		"mysql": func(url *url.URL) string {
			password, _ := url.User.Password()
			return fmt.Sprintf("%v:%v@%v", url.User.Username(), password, url.RequestURI())
		},
	}

	for driverName, v := range knownDrivers {
		_, err := sql.Open(driverName, "")
		if err == nil {
			if dsnTranslators != nil { // !nashtsai! does not override
				if _, has := dsnTranslators[driverName]; has {
					continue
				}
			}
			RegisterDSNTranslator(driverName, v)
		} else {
			// fmt.Printf("driver failed: %v | err: %v\n", driverName, err)
		}
	}
}
Exemplo n.º 4
0
func (this *Cache) generateKey(r *http.Request) string {
	if key, ok := r.Header["X-Memcache-Key"]; ok {
		return key[0]
	}

	// Use the request URI as a key.
	url, _ := url.Parse(r.URL.String())
	return url.RequestURI()
}
Exemplo n.º 5
0
func TestAddingRedirects(t *testing.T) {
	db := getDB(t)
	ds := getDS(t)

	link := func(index int) string {
		return fmt.Sprintf("http://subdom.dom.com/page%d.html", index)
	}

	fr := walker.FetchResults{
		URL:            parse(link(1)),
		RedirectedFrom: []*walker.URL{parse(link(2)), parse(link(3))},
		FetchTime:      time.Unix(0, 0),
	}

	ds.StoreURLFetchResults(&fr)

	expected := []struct {
		link  string
		redto string
	}{
		{link: link(1), redto: link(2)},
		{link: link(2), redto: link(3)},
		{link: link(3), redto: ""},
	}

	for _, exp := range expected {
		url := parse(exp.link)

		dom, subdom, _ := url.TLDPlusOneAndSubdomain()
		itr := db.Query("SELECT redto_url FROM links WHERE dom = ? AND subdom = ? AND path = ? AND proto = ?",
			dom,
			subdom,
			url.RequestURI(),
			url.Scheme).Iter()
		var redto string
		if !itr.Scan(&redto) {
			t.Errorf("Failed to find link %q", exp.link)
			continue
		}

		err := itr.Close()
		if err != nil {
			t.Errorf("Iterator returned error %v", err)
			continue
		}

		if redto != exp.redto {
			t.Errorf("Redirect mismatch: got %q, expected %q", redto, exp.redto)
		}
	}
}
Exemplo n.º 6
0
func openMongoSession(mongoUrl string) (session *mgo.Session, db *mgo.Database, err error) {
	session, err = mgo.Dial(mongoUrl)
	if err != nil {
		return
	}

	url, err := url.Parse(mongoUrl)
	if err != nil {
		return
	}

	dbName := strings.TrimLeft(url.RequestURI(), "/")
	db = session.DB(dbName)
	return
}
Exemplo n.º 7
0
func SiteCache(store CacheStore, expire time.Duration) gin.HandlerFunc {

	return func(c *gin.Context) {
		var cache responseCache
		url := c.Request.URL
		key := urlEscape(PageCachePrefix, url.RequestURI())
		if err := store.Get(key, &cache); err != nil {
			c.Next()
		} else {
			c.Writer.WriteHeader(cache.status)
			for k, vals := range cache.header {
				for _, v := range vals {
					c.Writer.Header().Add(k, v)
				}
			}
			c.Writer.Write(cache.data)
		}
	}
}
Exemplo n.º 8
0
func openMongoSession() (session *mgo.Session, db *mgo.Database, err error) {
	mongoUrl := os.Getenv("MONGO_URL")
	if mongoUrl == "" {
		mongoUrl = "mongodb://10.0.2.2:27017/minefold_development"
	}
	session, err = mgo.Dial(mongoUrl)
	if err != nil {
		return
	}

	url, err := url.Parse(mongoUrl)
	if err != nil {
		return
	}

	dbName := strings.TrimLeft(url.RequestURI(), "/")
	db = session.DB(dbName)
	return
}
Exemplo n.º 9
0
func (m *goMysqlDB) RunTest(t *testing.T, fn func(params)) {
	if !m.Running() {
		t.Logf("skipping test; no MySQL running on localhost:3306")
		return
	}
	user := os.Getenv("GOSQLTEST_MYSQL_USER")
	if user == "" {
		user = "******"
	}
	pass, ok := getenvOk("GOSQLTEST_MYSQL_PASS")
	if !ok {
		pass = "******"
	}
	dbName := "gosqltest"

	gosqlproxy.RegisterDSNTranslator("mysql", func(url *url.URL) string {
		password, _ := url.User.Password()
		return fmt.Sprintf("%v:%v@%v", url.User.Username(), password, url.RequestURI())
	})

	// db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s", user, pass, dbName))
	db, err := sql.Open("gosqlproxy", fmt.Sprintf("mysql://%s:%s@localhost/%s;params://localhost/ddd?batchsize=50&asyncwrite=true", user, pass, dbName))
	if err != nil {
		t.Fatalf("error connecting: %v", err)
	}

	params := params{goMysql, t, db}

	// Drop all tables in the test database.
	rows, err := db.Query("SHOW TABLES")
	if err != nil {
		t.Fatalf("failed to enumerate tables: %v", err)
	}
	for rows.Next() {
		var table string
		if rows.Scan(&table) == nil &&
			strings.HasPrefix(strings.ToLower(table), strings.ToLower(TablePrefix)) {
			params.mustExec("DROP TABLE " + table)
		}
	}

	fn(params)
}
Exemplo n.º 10
0
Arquivo: conn.go Projeto: vonwenm/spdy
// newStream is used to create a new serverStream from a SYN_STREAM frame.
func (c *Conn) newStream(frame *frames.SYN_STREAM) *ResponseStream {
	header := frame.Header
	rawUrl := header.Get(":scheme") + "://" + header.Get(":host") + header.Get(":path")

	url, err := url.Parse(rawUrl)
	if c.check(err != nil, "Received SYN_STREAM with invalid request URL (%v)", err) {
		return nil
	}

	vers := header.Get(":version")
	major, minor, ok := http.ParseHTTPVersion(vers)
	if c.check(!ok, "Invalid HTTP version: "+vers) {
		return nil
	}

	method := header.Get(":method")

	// Build this into a request to present to the Handler.
	request := &http.Request{
		Method:     method,
		URL:        url,
		Proto:      vers,
		ProtoMajor: major,
		ProtoMinor: minor,
		RemoteAddr: c.remoteAddr,
		Header:     header,
		Host:       url.Host,
		RequestURI: url.RequestURI(),
		TLS:        c.tlsState,
	}

	output := c.output[frame.Priority]
	c.streamCreation.Lock()
	out := NewResponseStream(c, frame, output, c.server.Handler, request)
	c.streamCreation.Unlock()
	c.flowControlLock.Lock()
	f := c.flowControl
	c.flowControlLock.Unlock()
	out.AddFlowControl(f)

	return out
}
Exemplo n.º 11
0
func sendWebHook(config account.HookConfig, body interface{}) {
	if config.Address != "" {
		url, err := url.Parse(config.Address)
		if config.Method == "" {
			config.Method = "POST"
		}
		b := body.([]byte)
		httpClient := requests.NewHTTPClient(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
		_, _, _, err = httpClient.MakeRequest(requests.Args{
			AcceptableCode: http.StatusOK,
			Method:         config.Method,
			Path:           url.RequestURI(),
			Body:           string(b),
		})

		if err != nil {
			Logger.Warn(fmt.Sprintf("Failed to call WebHook for %s: %s.", config.Address, err.Error()))
		}
	}
}
Exemplo n.º 12
0
// Cache Decorator
func CachePage(store CacheStore, expire time.Duration, handle gin.HandlerFunc) gin.HandlerFunc {

	return func(c *gin.Context) {
		var cache responseCache
		url := c.Request.URL
		key := urlEscape(PageCachePrefix, url.RequestURI())
		if err := store.Get(key, &cache); err != nil {
			// replace writer
			writer := newCachedWriter(store, expire, c.Writer, key)
			c.Writer = writer
			handle(c)
		} else {
			c.Writer.WriteHeader(cache.status)
			for k, vals := range cache.header {
				for _, v := range vals {
					c.Writer.Header().Add(k, v)
				}
			}
			c.Writer.Write(cache.data)
		}
	}
}
Exemplo n.º 13
0
// Parses the xml and returns what we need from the xml
func (episode *CrunchyrollEpisode) GetEpisodeInfo(quality string, cookies []*http.Cookie) error {
	episode.Quality = quality // Sets the quality to the passed quality string

	// Gets the HTML of the episode page
	episodeReqHeaders := http.Header{}
	episodeReqHeaders.Add("referer", "http://www.crunchyroll.com/"+strings.Split(episode.Path, "/")[1])
	episodeResponse, err := anirip.GetHTTPResponse("GET",
		episode.URL,
		nil,
		episodeReqHeaders,
		cookies)
	if err != nil {
		return err
	}

	// Creates the goquery document that will be used to scrape for episode metadata
	episodeDoc, err := goquery.NewDocumentFromResponse(episodeResponse)
	if err != nil {
		return anirip.Error{Message: "There was an error while reading the episode doc", Err: err}
	}

	// Scrapes the episode metadata from the episode page
	episodeMetaDataJSON := episodeDoc.Find("script#liftigniter-metadata").First().Text()

	// Parses the metadata json to a MetaData object
	episodeMetaData := new(EpisodeMetaData)
	if err := json.Unmarshal([]byte(episodeMetaDataJSON), episodeMetaData); err != nil {
		return anirip.Error{Message: "There was an error while parsing episode metadata", Err: err}
	}

	// Formdata to indicate the source page
	formData := url.Values{
		"current_page": {episode.URL},
	}

	// Querystring for getting the crunchyroll standard config
	queryString := url.Values{
		"req":           {"RpcApiVideoPlayer_GetStandardConfig"},
		"media_id":      {strconv.Itoa(episode.ID)},
		"video_format":  {getVideoFormat(episode.Quality)},
		"video_quality": {getVideoQuality(episode.Quality)},
		"auto_play":     {"1"},
		"aff":           {"crunchyroll-website"},
		"show_pop_out_controls":   {"1"},
		"pop_out_disable_message": {""},
		"click_through":           {"0"},
	}

	// Performs the HTTP Request that will get the XML
	standardConfigReqHeaders := http.Header{}
	standardConfigReqHeaders.Add("Host", "www.crunchyroll.com")
	standardConfigReqHeaders.Add("Origin", "http://static.ak.crunchyroll.com")
	standardConfigReqHeaders.Add("Content-type", "application/x-www-form-urlencoded")
	standardConfigReqHeaders.Add("Referer", "http://static.ak.crunchyroll.com/versioned_assets/StandardVideoPlayer.f3770232.swf")
	standardConfigReqHeaders.Add("X-Requested-With", "ShockwaveFlash/22.0.0.192")
	standardConfigResponse, err := anirip.GetHTTPResponse("POST",
		"http://www.crunchyroll.com/xml/?"+queryString.Encode(),
		bytes.NewBufferString(formData.Encode()),
		standardConfigReqHeaders,
		cookies)
	if err != nil {
		return err
	}

	// Gets the xml string from the recieved xml response body
	standardConfigResponseBody, err := ioutil.ReadAll(standardConfigResponse.Body)
	if err != nil {
		return anirip.Error{Message: "There was an error reading the xml response", Err: err}
	}
	xmlString := string(standardConfigResponseBody)

	// Checks for an unsupported region first
	if strings.Contains(xmlString, "<code>") && strings.Contains(xmlString, "</code>") {
		if strings.SplitN(strings.SplitN(xmlString, "<code>", 2)[1], "</code>", 2)[0] == "4" {
			return anirip.Error{Message: "This video is not available in your region", Err: err}
		}
	}

	// Next performs some really basic parsing of the host url
	xmlHostURL := ""
	if strings.Contains(xmlString, "<host>") && strings.Contains(xmlString, "</host>") {
		xmlHostURL = strings.SplitN(strings.SplitN(xmlString, "<host>", 2)[1], "</host>", 2)[0]
	} else {
		return anirip.Error{Message: "No hosts were found for the episode", Err: err}
	}

	// Same type of xml parsing to get the file
	episodeFile := ""
	if strings.Contains(xmlString, "<file>") && strings.Contains(xmlString, "</file>") {
		episodeFile = strings.SplitN(strings.SplitN(xmlString, "<file>", 2)[1], "</file>", 2)[0]
	} else {
		return anirip.Error{Message: "No hosts were found for the episode", Err: err}
	}

	// Parses the URL in order to break out the two urls required for dumping
	url, err := url.Parse(xmlHostURL)
	if err != nil {
		return anirip.Error{Message: "There was an error parsing episode information", Err: err}
	}

	// Sets the RTMP info recieved before returning
	episode.Title = episodeMetaData.Name
	episode.FileName = anirip.CleanFileName(episode.FileName + episode.Title) // Updates filename with title that we just scraped
	episode.MediaInfo = RTMPInfo{
		File:   episodeFile,
		URLOne: url.Scheme + "://" + url.Host + url.Path,
		URLTwo: strings.Trim(url.RequestURI(), "/"),
	}
	return nil
}
Exemplo n.º 14
0
func (e Entry) ProxyLink() (h template.HTML, err error) {
	//Retrieve url content
	//	res, err := http.Get(html.UnescapeString(e.Link))
	//Get the actual domain and attempt to replace any img links that are relative
	url, err := url.Parse(e.Link)
	if err != nil {
		glog.Errorf("url.Parse(%s): %s", e.Link, err)
	}

	proxy_username := "******"
	proxy_pass := "******"

	link := fmt.Sprintf("https://%s:%[email protected]/nph-proxy.pl/en/20/%s/%s%s", proxy_username, proxy_pass, url.Scheme, url.Host, url.RequestURI())
	fmt.Printf("link: %s", link)

	ht := fmt.Sprintf("<iframe id='view_iframe' src=\"%s\" style='overflow:auto;height:1000%%;frameborder:0;width:100%%' ></iframe>", link)
	return template.HTML(ht), err
	/*
		res, err := http.Get(html.UnescapeString(link))
		if err != nil {
			glog.Errorf("http.Get(html.UnescapeString(%s)): %s", link, err)
			return h, err
		}
	*/

	/*
		client := &http.Client{}

		link := fmt.Sprintf("https://proxy.chriskaufmann.com/nph-proxy.pl/en/20/%s/%s%s",  url.Scheme, url.Host, url.RequestURI() )
		fmt.Printf("link: %s", link)
		req, err := http.NewRequest("GET", link, nil)
		if err != nil {
			glog.Errorf("http.NewRuquest(%s): %s", e.Link, err)
		}
		req.SetBasicAuth(proxy_username, proxy_pass)
		res, err := client.Do(req)
		if err != nil {
			glog.Errorf("http.Get(%s): %s", e.Link, err)
			return h, err
		}

		content, err := ioutil.ReadAll(res.Body)
		if err != nil {
			glog.Errorf("ioutil.ReadAll(): %s", err)
			return h, err
		}
		res.Body.Close()
		h = template.HTML(content)
		return h, err

	*/
	/*
		ht := fmt.Sprintf("<iframe id='view_iframe' src=\"%s\" style='overflow:auto;height:1000%%;frameborder:0;width:100%%' ></iframe>", link)
		return template.HTML(ht), err

		client := &http.Client{}
		req, err := http.NewRequest("GET", e.Link, nil)
		if err != nil {
			glog.Errorf("http.NewRuquest(%s): %s", e.Link, err)
		}
		req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36")
		res, err := client.Do(req)
		if err != nil {
			glog.Errorf("http.Get(%s): %s", e.Link, err)
			return h, err
		}
	*/

	/*
		content, err := ioutil.ReadAll(res.Body)
		if err != nil {
			glog.Errorf("ioutil.ReadAll(): %s", err)
			return h, err
		}
		res.Body.Close()
		c := fmt.Sprintf("url: %s\n Content: %s", e.Link, content)
		imgregex, err := regexp.Compile("img\\s+src\\s{0,}=\\s{0,}(['\"])/")
		if err != nil {
			glog.Errorf("regexp.compile(): %s")
		}
		fmt.Printf("host: %s", url.Host)
		c = imgregex.ReplaceAllString(c, "img src=$1")

		excludes := []string{"<head(.|\n)*?/head>", "<script(.|\n)*?/script>", "<noscript(.|\n)*?/noscript>"}
		for _, e := range excludes {
			reg, err := regexp.CompilePOSIX(e)
			if err != nil {
				glog.Errorf("rgexp.compile: %s", err)
				return h, err
			}
			c = reg.ReplaceAllString(c, " ")
		}
		fmt.Printf("html: %s", fmt.Sprintf(" %s ", c))
		h = template.HTML(c)
		return h, err
	*/
}
Exemplo n.º 15
0
// handlePush performs the processing of SYN_STREAM frames forming a server push.
func (c *Conn) handlePush(frame *frames.SYN_STREAM) {
	sid := frame.StreamID

	// Check stream creation is allowed.
	c.goawayLock.Lock()
	goaway := c.goawayReceived || c.goawaySent
	c.goawayLock.Unlock()
	if goaway || c.Closed() {
		return
	}

	if c.server != nil {
		log.Println("Error: Only clients can receive server pushes.")
		return
	}

	if c.check(sid&1 != 0, "Received SYN_STREAM with odd Stream ID %d", sid) {
		return
	}

	c.lastPushStreamIDLock.Lock()
	lsid := c.lastPushStreamID
	c.lastPushStreamIDLock.Unlock()
	if c.check(sid <= lsid, "Received SYN_STREAM with Stream ID %d, less than %d", sid, lsid) {
		return
	}

	if c.criticalCheck(!sid.Valid(), sid, "Received SYN_STREAM with excessive Stream ID %d", sid) {
		return
	}

	// Stream ID is fine.

	// Check stream limit would allow the new stream.
	if !c.pushStreamLimit.Add() {
		c._RST_STREAM(sid, common.RST_STREAM_REFUSED_STREAM)
		return
	}

	ok := frame.Priority.Valid(3)
	if c.criticalCheck(!ok, sid, "Received SYN_STREAM with invalid priority %d", frame.Priority) {
		return
	}

	// Parse the request.
	header := frame.Header
	rawUrl := header.Get(":scheme") + "://" + header.Get(":host") + header.Get(":path")
	url, err := url.Parse(rawUrl)
	if c.check(err != nil, "Received SYN_STREAM with invalid request URL (%v)", err) {
		return
	}

	vers := header.Get(":version")
	major, minor, ok := http.ParseHTTPVersion(vers)
	if c.check(!ok, "Invalid HTTP version: "+vers) {
		return
	}

	method := header.Get(":method")
	request := &http.Request{
		Method:     method,
		URL:        url,
		Proto:      vers,
		ProtoMajor: major,
		ProtoMinor: minor,
		RemoteAddr: c.remoteAddr,
		Header:     header,
		Host:       url.Host,
		RequestURI: url.RequestURI(),
		TLS:        c.tlsState,
	}

	// Check whether the receiver wants this resource.
	if c.PushReceiver != nil && !c.PushReceiver.ReceiveRequest(request) {
		c._RST_STREAM(sid, common.RST_STREAM_REFUSED_STREAM)
		return
	}

	if c.PushReceiver != nil {
		c.pushRequests[sid] = request
		c.lastPushStreamIDLock.Lock()
		c.lastPushStreamID = sid
		c.lastPushStreamIDLock.Unlock()
		c.PushReceiver.ReceiveHeader(request, frame.Header)
	}
}