func parse(s string, base url.URL) (ln []string, as []string) {
z := html.NewTokenizer(strings.NewReader(s))
lnm := make(map[string]struct{})
asm := make(map[string]struct{})
// anonymous func used to get attribute values
attr := func(a string) string {
var av string
for {
if k, v, ha := z.TagAttr(); string(k) == a {
av = string(v)
break
} else if ha == false {
break
}
}
return av
}
// convert map to slice
slc := func(m map[string]struct{}) []string {
var v []string
for k := range m {
v = append(v, k)
}
sort.Strings(v)
return v
}
for {
tt := z.Next()
switch tt {
case html.ErrorToken:
return slc(lnm), slc(asm)
case html.StartTagToken, html.EndTagToken:
if tn, ha := z.TagName(); ha {
tg := string(tn)
if av := attr(attrs[tg]); av != "" {
switch tg {
case atag:
if url, err := base.Parse(av); err == nil {
if url.Host == base.Host && url.RequestURI() != base.RequestURI() {
lnm[url.RequestURI()] = struct{}{}
}
}
case stag, itag, ltag:
asm[av] = struct{}{}
}
}
}
}
}
}
func Fetch(imageUrl string) ([]byte, error) {
url, err := url.Parse(imageUrl)
if err != nil {
return nil, fmt.Errorf("Invalid image URL: (url=%s)", url.RequestURI())
}
return fetchImage(url)
}
func regKnownDSNTranslators() {
// TODO
knownDrivers := map[string]func(*url.URL) string{
"postgres": func(url *url.URL) string {
password, _ := url.User.Password()
return fmt.Sprintf("user=%s password=%s dbname=%s %s", url.User.Username(), password, strings.TrimPrefix(url.Path, "/"), url.RawQuery)
},
// "sqlite3": func(url *url.URL) string { return url.Path },
"mysql": func(url *url.URL) string {
password, _ := url.User.Password()
return fmt.Sprintf("%v:%v@%v", url.User.Username(), password, url.RequestURI())
},
}
for driverName, v := range knownDrivers {
_, err := sql.Open(driverName, "")
if err == nil {
if dsnTranslators != nil { // !nashtsai! does not override
if _, has := dsnTranslators[driverName]; has {
continue
}
}
RegisterDSNTranslator(driverName, v)
} else {
// fmt.Printf("driver failed: %v | err: %v\n", driverName, err)
}
}
}
func (this *Cache) generateKey(r *http.Request) string {
if key, ok := r.Header["X-Memcache-Key"]; ok {
return key[0]
}
// Use the request URI as a key.
url, _ := url.Parse(r.URL.String())
return url.RequestURI()
}
func TestAddingRedirects(t *testing.T) {
db := getDB(t)
ds := getDS(t)
link := func(index int) string {
return fmt.Sprintf("http://subdom.dom.com/page%d.html", index)
}
fr := walker.FetchResults{
URL: parse(link(1)),
RedirectedFrom: []*walker.URL{parse(link(2)), parse(link(3))},
FetchTime: time.Unix(0, 0),
}
ds.StoreURLFetchResults(&fr)
expected := []struct {
link string
redto string
}{
{link: link(1), redto: link(2)},
{link: link(2), redto: link(3)},
{link: link(3), redto: ""},
}
for _, exp := range expected {
url := parse(exp.link)
dom, subdom, _ := url.TLDPlusOneAndSubdomain()
itr := db.Query("SELECT redto_url FROM links WHERE dom = ? AND subdom = ? AND path = ? AND proto = ?",
dom,
subdom,
url.RequestURI(),
url.Scheme).Iter()
var redto string
if !itr.Scan(&redto) {
t.Errorf("Failed to find link %q", exp.link)
continue
}
err := itr.Close()
if err != nil {
t.Errorf("Iterator returned error %v", err)
continue
}
if redto != exp.redto {
t.Errorf("Redirect mismatch: got %q, expected %q", redto, exp.redto)
}
}
}
func openMongoSession(mongoUrl string) (session *mgo.Session, db *mgo.Database, err error) {
session, err = mgo.Dial(mongoUrl)
if err != nil {
return
}
url, err := url.Parse(mongoUrl)
if err != nil {
return
}
dbName := strings.TrimLeft(url.RequestURI(), "/")
db = session.DB(dbName)
return
}
func SiteCache(store CacheStore, expire time.Duration) gin.HandlerFunc {
return func(c *gin.Context) {
var cache responseCache
url := c.Request.URL
key := urlEscape(PageCachePrefix, url.RequestURI())
if err := store.Get(key, &cache); err != nil {
c.Next()
} else {
c.Writer.WriteHeader(cache.status)
for k, vals := range cache.header {
for _, v := range vals {
c.Writer.Header().Add(k, v)
}
}
c.Writer.Write(cache.data)
}
}
}
func openMongoSession() (session *mgo.Session, db *mgo.Database, err error) {
mongoUrl := os.Getenv("MONGO_URL")
if mongoUrl == "" {
mongoUrl = "mongodb://10.0.2.2:27017/minefold_development"
}
session, err = mgo.Dial(mongoUrl)
if err != nil {
return
}
url, err := url.Parse(mongoUrl)
if err != nil {
return
}
dbName := strings.TrimLeft(url.RequestURI(), "/")
db = session.DB(dbName)
return
}
func (m *goMysqlDB) RunTest(t *testing.T, fn func(params)) {
if !m.Running() {
t.Logf("skipping test; no MySQL running on localhost:3306")
return
}
user := os.Getenv("GOSQLTEST_MYSQL_USER")
if user == "" {
user = "******"
}
pass, ok := getenvOk("GOSQLTEST_MYSQL_PASS")
if !ok {
pass = "******"
}
dbName := "gosqltest"
gosqlproxy.RegisterDSNTranslator("mysql", func(url *url.URL) string {
password, _ := url.User.Password()
return fmt.Sprintf("%v:%v@%v", url.User.Username(), password, url.RequestURI())
})
// db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s", user, pass, dbName))
db, err := sql.Open("gosqlproxy", fmt.Sprintf("mysql://%s:%s@localhost/%s;params://localhost/ddd?batchsize=50&asyncwrite=true", user, pass, dbName))
if err != nil {
t.Fatalf("error connecting: %v", err)
}
params := params{goMysql, t, db}
// Drop all tables in the test database.
rows, err := db.Query("SHOW TABLES")
if err != nil {
t.Fatalf("failed to enumerate tables: %v", err)
}
for rows.Next() {
var table string
if rows.Scan(&table) == nil &&
strings.HasPrefix(strings.ToLower(table), strings.ToLower(TablePrefix)) {
params.mustExec("DROP TABLE " + table)
}
}
fn(params)
}
// newStream is used to create a new serverStream from a SYN_STREAM frame.
func (c *Conn) newStream(frame *frames.SYN_STREAM) *ResponseStream {
header := frame.Header
rawUrl := header.Get(":scheme") + "://" + header.Get(":host") + header.Get(":path")
url, err := url.Parse(rawUrl)
if c.check(err != nil, "Received SYN_STREAM with invalid request URL (%v)", err) {
return nil
}
vers := header.Get(":version")
major, minor, ok := http.ParseHTTPVersion(vers)
if c.check(!ok, "Invalid HTTP version: "+vers) {
return nil
}
method := header.Get(":method")
// Build this into a request to present to the Handler.
request := &http.Request{
Method: method,
URL: url,
Proto: vers,
ProtoMajor: major,
ProtoMinor: minor,
RemoteAddr: c.remoteAddr,
Header: header,
Host: url.Host,
RequestURI: url.RequestURI(),
TLS: c.tlsState,
}
output := c.output[frame.Priority]
c.streamCreation.Lock()
out := NewResponseStream(c, frame, output, c.server.Handler, request)
c.streamCreation.Unlock()
c.flowControlLock.Lock()
f := c.flowControl
c.flowControlLock.Unlock()
out.AddFlowControl(f)
return out
}
func sendWebHook(config account.HookConfig, body interface{}) {
if config.Address != "" {
url, err := url.Parse(config.Address)
if config.Method == "" {
config.Method = "POST"
}
b := body.([]byte)
httpClient := requests.NewHTTPClient(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
_, _, _, err = httpClient.MakeRequest(requests.Args{
AcceptableCode: http.StatusOK,
Method: config.Method,
Path: url.RequestURI(),
Body: string(b),
})
if err != nil {
Logger.Warn(fmt.Sprintf("Failed to call WebHook for %s: %s.", config.Address, err.Error()))
}
}
}
// Cache Decorator
func CachePage(store CacheStore, expire time.Duration, handle gin.HandlerFunc) gin.HandlerFunc {
return func(c *gin.Context) {
var cache responseCache
url := c.Request.URL
key := urlEscape(PageCachePrefix, url.RequestURI())
if err := store.Get(key, &cache); err != nil {
// replace writer
writer := newCachedWriter(store, expire, c.Writer, key)
c.Writer = writer
handle(c)
} else {
c.Writer.WriteHeader(cache.status)
for k, vals := range cache.header {
for _, v := range vals {
c.Writer.Header().Add(k, v)
}
}
c.Writer.Write(cache.data)
}
}
}
// Parses the xml and returns what we need from the xml
func (episode *CrunchyrollEpisode) GetEpisodeInfo(quality string, cookies []*http.Cookie) error {
episode.Quality = quality // Sets the quality to the passed quality string
// Gets the HTML of the episode page
episodeReqHeaders := http.Header{}
episodeReqHeaders.Add("referer", "http://www.crunchyroll.com/"+strings.Split(episode.Path, "/")[1])
episodeResponse, err := anirip.GetHTTPResponse("GET",
episode.URL,
nil,
episodeReqHeaders,
cookies)
if err != nil {
return err
}
// Creates the goquery document that will be used to scrape for episode metadata
episodeDoc, err := goquery.NewDocumentFromResponse(episodeResponse)
if err != nil {
return anirip.Error{Message: "There was an error while reading the episode doc", Err: err}
}
// Scrapes the episode metadata from the episode page
episodeMetaDataJSON := episodeDoc.Find("script#liftigniter-metadata").First().Text()
// Parses the metadata json to a MetaData object
episodeMetaData := new(EpisodeMetaData)
if err := json.Unmarshal([]byte(episodeMetaDataJSON), episodeMetaData); err != nil {
return anirip.Error{Message: "There was an error while parsing episode metadata", Err: err}
}
// Formdata to indicate the source page
formData := url.Values{
"current_page": {episode.URL},
}
// Querystring for getting the crunchyroll standard config
queryString := url.Values{
"req": {"RpcApiVideoPlayer_GetStandardConfig"},
"media_id": {strconv.Itoa(episode.ID)},
"video_format": {getVideoFormat(episode.Quality)},
"video_quality": {getVideoQuality(episode.Quality)},
"auto_play": {"1"},
"aff": {"crunchyroll-website"},
"show_pop_out_controls": {"1"},
"pop_out_disable_message": {""},
"click_through": {"0"},
}
// Performs the HTTP Request that will get the XML
standardConfigReqHeaders := http.Header{}
standardConfigReqHeaders.Add("Host", "www.crunchyroll.com")
standardConfigReqHeaders.Add("Origin", "http://static.ak.crunchyroll.com")
standardConfigReqHeaders.Add("Content-type", "application/x-www-form-urlencoded")
standardConfigReqHeaders.Add("Referer", "http://static.ak.crunchyroll.com/versioned_assets/StandardVideoPlayer.f3770232.swf")
standardConfigReqHeaders.Add("X-Requested-With", "ShockwaveFlash/22.0.0.192")
standardConfigResponse, err := anirip.GetHTTPResponse("POST",
"http://www.crunchyroll.com/xml/?"+queryString.Encode(),
bytes.NewBufferString(formData.Encode()),
standardConfigReqHeaders,
cookies)
if err != nil {
return err
}
// Gets the xml string from the recieved xml response body
standardConfigResponseBody, err := ioutil.ReadAll(standardConfigResponse.Body)
if err != nil {
return anirip.Error{Message: "There was an error reading the xml response", Err: err}
}
xmlString := string(standardConfigResponseBody)
// Checks for an unsupported region first
if strings.Contains(xmlString, "<code>") && strings.Contains(xmlString, "</code>") {
if strings.SplitN(strings.SplitN(xmlString, "<code>", 2)[1], "</code>", 2)[0] == "4" {
return anirip.Error{Message: "This video is not available in your region", Err: err}
}
}
// Next performs some really basic parsing of the host url
xmlHostURL := ""
if strings.Contains(xmlString, "<host>") && strings.Contains(xmlString, "</host>") {
xmlHostURL = strings.SplitN(strings.SplitN(xmlString, "<host>", 2)[1], "</host>", 2)[0]
} else {
return anirip.Error{Message: "No hosts were found for the episode", Err: err}
}
// Same type of xml parsing to get the file
episodeFile := ""
if strings.Contains(xmlString, "<file>") && strings.Contains(xmlString, "</file>") {
episodeFile = strings.SplitN(strings.SplitN(xmlString, "<file>", 2)[1], "</file>", 2)[0]
} else {
return anirip.Error{Message: "No hosts were found for the episode", Err: err}
}
// Parses the URL in order to break out the two urls required for dumping
url, err := url.Parse(xmlHostURL)
if err != nil {
return anirip.Error{Message: "There was an error parsing episode information", Err: err}
}
// Sets the RTMP info recieved before returning
episode.Title = episodeMetaData.Name
episode.FileName = anirip.CleanFileName(episode.FileName + episode.Title) // Updates filename with title that we just scraped
episode.MediaInfo = RTMPInfo{
File: episodeFile,
URLOne: url.Scheme + "://" + url.Host + url.Path,
URLTwo: strings.Trim(url.RequestURI(), "/"),
}
return nil
}
func (e Entry) ProxyLink() (h template.HTML, err error) {
//Retrieve url content
// res, err := http.Get(html.UnescapeString(e.Link))
//Get the actual domain and attempt to replace any img links that are relative
url, err := url.Parse(e.Link)
if err != nil {
glog.Errorf("url.Parse(%s): %s", e.Link, err)
}
proxy_username := "******"
proxy_pass := "******"
link := fmt.Sprintf("https://%s:%[email protected]/nph-proxy.pl/en/20/%s/%s%s", proxy_username, proxy_pass, url.Scheme, url.Host, url.RequestURI())
fmt.Printf("link: %s", link)
ht := fmt.Sprintf("<iframe id='view_iframe' src=\"%s\" style='overflow:auto;height:1000%%;frameborder:0;width:100%%' ></iframe>", link)
return template.HTML(ht), err
/*
res, err := http.Get(html.UnescapeString(link))
if err != nil {
glog.Errorf("http.Get(html.UnescapeString(%s)): %s", link, err)
return h, err
}
*/
/*
client := &http.Client{}
link := fmt.Sprintf("https://proxy.chriskaufmann.com/nph-proxy.pl/en/20/%s/%s%s", url.Scheme, url.Host, url.RequestURI() )
fmt.Printf("link: %s", link)
req, err := http.NewRequest("GET", link, nil)
if err != nil {
glog.Errorf("http.NewRuquest(%s): %s", e.Link, err)
}
req.SetBasicAuth(proxy_username, proxy_pass)
res, err := client.Do(req)
if err != nil {
glog.Errorf("http.Get(%s): %s", e.Link, err)
return h, err
}
content, err := ioutil.ReadAll(res.Body)
if err != nil {
glog.Errorf("ioutil.ReadAll(): %s", err)
return h, err
}
res.Body.Close()
h = template.HTML(content)
return h, err
*/
/*
ht := fmt.Sprintf("<iframe id='view_iframe' src=\"%s\" style='overflow:auto;height:1000%%;frameborder:0;width:100%%' ></iframe>", link)
return template.HTML(ht), err
client := &http.Client{}
req, err := http.NewRequest("GET", e.Link, nil)
if err != nil {
glog.Errorf("http.NewRuquest(%s): %s", e.Link, err)
}
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36")
res, err := client.Do(req)
if err != nil {
glog.Errorf("http.Get(%s): %s", e.Link, err)
return h, err
}
*/
/*
content, err := ioutil.ReadAll(res.Body)
if err != nil {
glog.Errorf("ioutil.ReadAll(): %s", err)
return h, err
}
res.Body.Close()
c := fmt.Sprintf("url: %s\n Content: %s", e.Link, content)
imgregex, err := regexp.Compile("img\\s+src\\s{0,}=\\s{0,}(['\"])/")
if err != nil {
glog.Errorf("regexp.compile(): %s")
}
fmt.Printf("host: %s", url.Host)
c = imgregex.ReplaceAllString(c, "img src=$1")
excludes := []string{"<head(.|\n)*?/head>", "<script(.|\n)*?/script>", "<noscript(.|\n)*?/noscript>"}
for _, e := range excludes {
reg, err := regexp.CompilePOSIX(e)
if err != nil {
glog.Errorf("rgexp.compile: %s", err)
return h, err
}
c = reg.ReplaceAllString(c, " ")
}
fmt.Printf("html: %s", fmt.Sprintf(" %s ", c))
h = template.HTML(c)
return h, err
*/
}
// handlePush performs the processing of SYN_STREAM frames forming a server push.
func (c *Conn) handlePush(frame *frames.SYN_STREAM) {
sid := frame.StreamID
// Check stream creation is allowed.
c.goawayLock.Lock()
goaway := c.goawayReceived || c.goawaySent
c.goawayLock.Unlock()
if goaway || c.Closed() {
return
}
if c.server != nil {
log.Println("Error: Only clients can receive server pushes.")
return
}
if c.check(sid&1 != 0, "Received SYN_STREAM with odd Stream ID %d", sid) {
return
}
c.lastPushStreamIDLock.Lock()
lsid := c.lastPushStreamID
c.lastPushStreamIDLock.Unlock()
if c.check(sid <= lsid, "Received SYN_STREAM with Stream ID %d, less than %d", sid, lsid) {
return
}
if c.criticalCheck(!sid.Valid(), sid, "Received SYN_STREAM with excessive Stream ID %d", sid) {
return
}
// Stream ID is fine.
// Check stream limit would allow the new stream.
if !c.pushStreamLimit.Add() {
c._RST_STREAM(sid, common.RST_STREAM_REFUSED_STREAM)
return
}
ok := frame.Priority.Valid(3)
if c.criticalCheck(!ok, sid, "Received SYN_STREAM with invalid priority %d", frame.Priority) {
return
}
// Parse the request.
header := frame.Header
rawUrl := header.Get(":scheme") + "://" + header.Get(":host") + header.Get(":path")
url, err := url.Parse(rawUrl)
if c.check(err != nil, "Received SYN_STREAM with invalid request URL (%v)", err) {
return
}
vers := header.Get(":version")
major, minor, ok := http.ParseHTTPVersion(vers)
if c.check(!ok, "Invalid HTTP version: "+vers) {
return
}
method := header.Get(":method")
request := &http.Request{
Method: method,
URL: url,
Proto: vers,
ProtoMajor: major,
ProtoMinor: minor,
RemoteAddr: c.remoteAddr,
Header: header,
Host: url.Host,
RequestURI: url.RequestURI(),
TLS: c.tlsState,
}
// Check whether the receiver wants this resource.
if c.PushReceiver != nil && !c.PushReceiver.ReceiveRequest(request) {
c._RST_STREAM(sid, common.RST_STREAM_REFUSED_STREAM)
return
}
if c.PushReceiver != nil {
c.pushRequests[sid] = request
c.lastPushStreamIDLock.Lock()
c.lastPushStreamID = sid
c.lastPushStreamIDLock.Unlock()
c.PushReceiver.ReceiveHeader(request, frame.Header)
}
}
