func validateDomain(base *url.URL, link *url.URL) error { if !link.IsAbs() { return fmt.Errorf("Validated domain error: url is not absolute: %s", link) } if base.Host == "" { return fmt.Errorf("Validated domain error: empty domain: %s", link) } /* if (base.Host != link.Host) { return fmt.Errorf("Validated domain error: different domain: %s", link) }*/ err := isDomainBlacklisted(link) if err != nil { return err } err = isExtenstionBlacklisted(link) if err != nil { return err } return nil }
func parseLinks(urlToGet *url.URL, content string) ([]string, error) { var ( err error links []string = make([]string, 0) matches [][]string findLinks = regexp.MustCompile("<a.*?href=\"(.*?)\"") ) //var // Retrieve all anchor tag URLs from string matches = findLinks.FindAllStringSubmatch(content, -1) for _, val := range matches { var linkUrl *url.URL // Parse the anchr tag URL if linkUrl, err = url.Parse(val[1]); err != nil { return links, err } //if // If the URL is absolute, add it to the slice // If the URL is relative, build an absolute URL if linkUrl.IsAbs() { links = append(links, linkUrl.String()) } else { links = append(links, urlToGet.Scheme+"://"+urlToGet.Host+linkUrl.String()) } //else } //for return links, err } //parseImages
func parseImages(urlToGet *url.URL, content string) ([]string, error) { var ( err error imgs []string matches [][]string findImages = regexp.MustCompile("<img.*?src=\"(.*?)\"") ) //var // Retrieve all image URLs from string matches = findImages.FindAllStringSubmatch(content, -1) for _, val := range matches { var imgUrl *url.URL // Parse the image URL if imgUrl, err = url.Parse(val[1]); err != nil { return imgs, err } //if // If the URL is absolute, add it to the slice // If the URL is relative, build an absolute URL if imgUrl.IsAbs() { imgs = append(imgs, imgUrl.String()) } else { imgs = append(imgs, urlToGet.Scheme+"://"+urlToGet.Host+imgUrl.String()) } //else } //for return imgs, err } //parseImages
// ParseSource - func ParseSource(value string) (*Source, error) { var ( alias string srcURL *url.URL ) parts := strings.SplitN(value, "=", 2) if len(parts) == 1 { f := parts[0] alias = strings.SplitN(value, ".", 2)[0] if path.Base(f) != f { err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory.", value) return nil, err } srcURL = absURL(f) } else if len(parts) == 2 { alias = parts[0] var err error srcURL, err = url.Parse(parts[1]) if err != nil { return nil, err } if !srcURL.IsAbs() { srcURL = absURL(parts[1]) } } s := NewSource(alias, srcURL) return s, nil }
func addFeeds(user content.User, fm *readeef.FeedManager, links []string) (resp responseError) { resp = newResponse() var err error errs := make([]addFeedError, 0, len(links)) for _, link := range links { var u *url.URL if u, err = url.Parse(link); err != nil { resp.err = err errs = append(errs, addFeedError{Link: link, Error: "Error parsing link"}) continue } else if !u.IsAbs() { resp.err = errors.New("Feed has no link") errs = append(errs, addFeedError{Link: link, Error: resp.err.Error()}) continue } else { var f content.Feed if f, err = fm.AddFeedByLink(link); err != nil { resp.err = err errs = append(errs, addFeedError{Link: link, Error: "Error adding feed to the database"}) continue } uf := user.AddFeed(f) if uf.HasErr() { resp.err = f.Err() errs = append(errs, addFeedError{Link: link, Title: f.Data().Title, Error: "Error adding feed to the database"}) continue } tags := strings.SplitN(u.Fragment, ",", -1) if u.Fragment != "" && len(tags) > 0 { repo := uf.Repo() tf := repo.TaggedFeed(user) tf.Data(uf.Data()) t := make([]content.Tag, len(tags)) for i := range tags { t[i] = repo.Tag(user) t[i].Data(data.Tag{Value: data.TagValue(tags[i])}) } tf.Tags(t) if tf.UpdateTags(); tf.HasErr() { resp.err = tf.Err() errs = append(errs, addFeedError{Link: link, Title: f.Data().Title, Error: "Error adding feed to the database"}) continue } } } } resp.val["Errors"] = errs resp.val["Success"] = len(errs) < len(links) return }
func debugUrl(u *url.URL) { logger.Println("Host:", u.Host) logger.Println("Path:", u.Path) logger.Println("Request URI:", u.RequestURI()) logger.Println("Scheme:", u.Scheme) logger.Println("Query:", u.RawQuery) logger.Println("Fragment:", u.Fragment) logger.Println("IsAbs:", u.IsAbs()) }
func addFeed(user content.User, fm *readeef.FeedManager, links []string) (resp responseError) { resp = newResponse() success := false for _, link := range links { var u *url.URL if u, resp.err = url.Parse(link); resp.err != nil { /* TODO: non-fatal error */ return } else if !u.IsAbs() { /* TODO: non-fatal error */ resp.err = errors.New("Feed has no link") return } else { var f content.Feed if f, resp.err = fm.AddFeedByLink(link); resp.err != nil { return } uf := user.AddFeed(f) if uf.HasErr() { resp.err = f.Err() return } tags := strings.SplitN(u.Fragment, ",", -1) if u.Fragment != "" && len(tags) > 0 { repo := uf.Repo() tf := repo.TaggedFeed(user) tf.Data(uf.Data()) t := make([]content.Tag, len(tags)) for i := range tags { t[i] = repo.Tag(user) t[i].Value(data.TagValue(tags[i])) } tf.Tags(t) if tf.UpdateTags(); tf.HasErr() { resp.err = tf.Err() return } } success = true } } resp.val["Success"] = success return }
func endpointIds(uri *url.URL) (deviceId, channelId string, ok bool) { if !uri.IsAbs() { ok = false return } pathPrefix := "/update/" i := strings.Index(uri.Path, pathPrefix) if i < 0 { ok = false return } key := strings.SplitN(uri.Path[i+len(pathPrefix):], ".", 2) if len(key) < 2 { ok = false return } return key[0], key[1], true }
// Combine a URL and Request to make the URL absolute func makeURLAbs(url *url.URL, request *http.Request) { if !url.IsAbs() { url.Host = request.Host if request.TLS != nil || request.Header.Get("X-Forwarded-Proto") == "https" { url.Scheme = "https" } else { url.Scheme = "http" } } }
// Combine a URL and Request to make the URL absolute func makeURLAbs(url *url.URL, request *http.Request) { if !url.IsAbs() { url.Host = request.Host if strings.HasPrefix(request.Proto, "HTTP/") { url.Scheme = "http" } else { url.Scheme = "https" } } }
func IpOutsideOfRanges(testURL url.URL, ranges []IPRange) (bool, error) { if !testURL.IsAbs() { return false, errors.New(fmt.Sprintf("Missing protocol for url: %s", testURL)) } host := strings.Split(testURL.Host, ":")[0] ipAddress := net.ParseIP(host) if ipAddress == nil { ipAddr, err := net.ResolveIPAddr("ip", host) if err != nil { return false, errors.New(fmt.Sprintf("Resolving host failed: %s", err)) } ipAddress = net.ParseIP(ipAddr.String()) } for _, ipRange := range ranges { if bytes.Compare(ipAddress, net.ParseIP(ipRange.Start)) >= 0 && bytes.Compare(ipAddress, net.ParseIP(ipRange.End)) <= 0 { return false, nil } } return true, nil }
// add in any schemas that have any includes func (g *GoWSDL) resolveXSDExternals(schema *XSDSchema, url *url.URL) error { for _, incl := range schema.Includes { log.Println("resolving: ", incl.SchemaLocation) location, err := url.Parse(incl.SchemaLocation) if err != nil { return err } _, schemaName := filepath.Split(location.Path) log.Println("resolving schema name: ", schemaName) if g.resolvedXSDExternals[schemaName] { // memoisation continue } schemaLocation := location.String() if !location.IsAbs() { if !url.IsAbs() { return fmt.Errorf("Unable to resolve external schema %s through WSDL URL %s", schemaLocation, url) } schemaLocation = url.Scheme + "://" + url.Host + schemaLocation } log.Println("Downloading external schema", "location", schemaLocation) data, err := downloadFile(schemaLocation, g.ignoreTLS, g.localRoot) newschema := new(XSDSchema) err = xml.Unmarshal(data, newschema) if err != nil { log.Println(err) return err } if len(newschema.Includes) > 0 && maxRecursion > g.currentRecursionLevel { g.currentRecursionLevel++ log.Printf("Entering recursion %d\n", g.currentRecursionLevel) g.resolveXSDExternals(newschema, url) } g.wsdl.Types.Schemas = append(g.wsdl.Types.Schemas, newschema) if g.resolvedXSDExternals == nil { g.resolvedXSDExternals = make(map[string]bool, maxRecursion) } g.resolvedXSDExternals[schemaName] = true } return nil }
func generateURL(URL *url.URL, host string) *url.URL { if !URL.IsAbs() { return nil } if URL.Scheme != _URL_REQUIRED_SCHEME { return nil } if URL.Host != _URL_REQUIRED_HOST { if URL.Host != _URL_REQUIRED_ALIAS_HOST { return nil } } if path, ok := Paths[URL.Path]; ok { URL.Path = path } else { return nil } vals := URL.Query() if URL.Path == _URL_REQUIRED_QUICK_SEARCH_PATH { vals.Set("quickSearch", "true") } else { vals.Set("quickSearch", "false") } vals.Set("sortBy", _URL_REQUIRED_SORTING_TYPE) vals.Set("sortDirection", _URL_REQUIRED_SORTING_DIRECTION) vals.Set("userId", "null") vals.Set("conf", "true;true;true;true;true;true;true;true;"+ "true;true;true;true;true;true;true;true;true;true;") URL.RawQuery = vals.Encode() return &url.URL{ Scheme: "http", Host: host, Path: "/rss", RawQuery: url.Values{ "url": {URL.String()}, }.Encode(), } }
func (g *GoWsdl) resolveXsdExternals(schema *XsdSchema, url *url.URL) error { for _, incl := range schema.Includes { location, err := url.Parse(incl.SchemaLocation) if err != nil { return err } _, schemaName := filepath.Split(location.Path) if g.resolvedXsdExternals[schemaName] { continue } schemaLocation := location.String() if !location.IsAbs() { if !url.IsAbs() { return errors.New(fmt.Sprintf("Unable to resolve external schema %s through WSDL URL %s", schemaLocation, url)) } schemaLocation = url.Scheme + "://" + url.Host + schemaLocation } Log.Info("Downloading external schema", "location", schemaLocation) data, err := downloadFile(schemaLocation, g.ignoreTls) newschema := &XsdSchema{} err = xml.Unmarshal(data, newschema) if err != nil { return err } if len(newschema.Includes) > 0 && maxRecursion > g.currentRecursionLevel { g.currentRecursionLevel++ //log.Printf("Entering recursion %d\n", g.currentRecursionLevel) g.resolveXsdExternals(newschema, url) } g.wsdl.Types.Schemas = append(g.wsdl.Types.Schemas, newschema) if g.resolvedXsdExternals == nil { g.resolvedXsdExternals = make(map[string]bool, maxRecursion) } g.resolvedXsdExternals[schemaName] = true } return nil }
func urlIsAbs(u *url.URL) lua.Function { return func(l *lua.State) int { l.PushBoolean(u.IsAbs()) return 1 } }
func (con Feed) Handler(c context.Context) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var err error db := readeef.GetDB(c) user := readeef.GetUser(c, r) action := webfw.GetMultiPatternIdentifier(c, r) params := webfw.GetParams(c, r) resp := make(map[string]interface{}) SWITCH: switch action { case "list": var feeds []readeef.Feed feeds, err = db.GetUserTagsFeeds(user) if err != nil { break } respFeeds := []feed{} for _, f := range feeds { respFeeds = append(respFeeds, feed{ Id: f.Id, Title: f.Title, Description: f.Description, Link: f.Link, Image: f.Image, Tags: f.Tags, UpdateError: f.UpdateError, SubscribeError: f.SubscribeError, }) } resp["Feeds"] = respFeeds case "discover": r.ParseForm() link := r.FormValue("url") var u *url.URL /* TODO: non-fatal error */ if u, err = url.Parse(link); err != nil { break /* TODO: non-fatal error */ } else if !u.IsAbs() { u.Scheme = "http" if u.Host == "" { parts := strings.SplitN(u.Path, "/", 2) u.Host = parts[0] if len(parts) > 1 { u.Path = "/" + parts[1] } else { u.Path = "" } } link = u.String() } var feeds []readeef.Feed feeds, err = con.fm.DiscoverFeeds(link) if err != nil { break } var userFeeds []readeef.Feed userFeeds, err = db.GetUserFeeds(user) if err != nil { break } userFeedIdMap := make(map[int64]bool) userFeedLinkMap := make(map[string]bool) for _, f := range userFeeds { userFeedIdMap[f.Id] = true userFeedLinkMap[f.Link] = true u, err := url.Parse(f.Link) if err == nil && strings.HasPrefix(u.Host, "www.") { u.Host = u.Host[4:] userFeedLinkMap[u.String()] = true } } respFeeds := []feed{} for _, f := range feeds { if !userFeedIdMap[f.Id] && !userFeedLinkMap[f.Link] { respFeeds = append(respFeeds, feed{ Id: f.Id, Title: f.Title, Description: f.Description, Link: f.Link, Image: f.Image, }) } } resp["Feeds"] = respFeeds case "opml": buf := util.BufferPool.GetBuffer() defer util.BufferPool.Put(buf) buf.ReadFrom(r.Body) var opml parser.Opml opml, err = parser.ParseOpml(buf.Bytes()) if err != nil { break } var userFeeds []readeef.Feed userFeeds, err = db.GetUserFeeds(user) if err != nil { break } userFeedMap := make(map[int64]bool) for _, f := range userFeeds { userFeedMap[f.Id] = true } var feeds []readeef.Feed for _, opmlFeed := range opml.Feeds { var discovered []readeef.Feed discovered, err = con.fm.DiscoverFeeds(opmlFeed.Url) if err != nil { continue } for _, f := range discovered { if !userFeedMap[f.Id] { if len(opmlFeed.Tags) > 0 { f.Link += "#" + strings.Join(opmlFeed.Tags, ",") } feeds = append(feeds, f) } } } respFeeds := []feed{} for _, f := range feeds { respFeeds = append(respFeeds, feed{ Id: f.Id, Title: f.Title, Description: f.Description, Link: f.Link, Image: f.Image, }) } resp["Feeds"] = respFeeds case "add": r.ParseForm() links := r.Form["url"] success := false for _, link := range links { /* TODO: non-fatal error */ var u *url.URL if u, err = url.Parse(link); err != nil { break SWITCH /* TODO: non-fatal error */ } else if !u.IsAbs() { err = errors.New("Feed has no link") break SWITCH } var f readeef.Feed f, err = con.fm.AddFeedByLink(link) if err != nil { break SWITCH } f, err = db.CreateUserFeed(readeef.GetUser(c, r), f) if err != nil { break SWITCH } tags := strings.SplitN(u.Fragment, ",", -1) if u.Fragment != "" && len(tags) > 0 { err = db.CreateUserFeedTag(f, tags...) } success = true } resp["Success"] = success case "remove": var id int64 id, err = strconv.ParseInt(params["feed-id"], 10, 64) /* TODO: non-fatal error */ if err != nil { break } var feed readeef.Feed feed, err = db.GetUserFeed(id, user) /* TODO: non-fatal error */ if err != nil { break } err = db.DeleteUserFeed(feed) /* TODO: non-fatal error */ if err != nil { break } con.fm.RemoveFeed(feed) resp["Success"] = true case "tags": var id int64 id, err = strconv.ParseInt(params["feed-id"], 10, 64) /* TODO: non-fatal error */ if err != nil { break } var feed readeef.Feed feed, err = db.GetUserFeed(id, user) /* TODO: non-fatal error */ if err != nil { break } if r.Method == "GET" { resp["Tags"] = feed.Tags } else if r.Method == "POST" { var tags []string tags, err = db.GetUserFeedTags(user, feed) if err != nil { break } err = db.DeleteUserFeedTag(feed, tags...) if err != nil { break } decoder := json.NewDecoder(r.Body) tags = []string{} err = decoder.Decode(&tags) if err != nil { break } err = db.CreateUserFeedTag(feed, tags...) if err != nil { break } resp["Success"] = true resp["Id"] = feed.Id } case "read": feedId := params["feed-id"] timestamp := params["timestamp"] var seconds int64 seconds, err = strconv.ParseInt(timestamp, 10, 64) /* TODO: non-fatal error */ if err != nil { break } t := time.Unix(seconds/1000, 0) switch { case feedId == "tag:__all__": err = db.MarkUserArticlesByDateAsRead(user, t, true) case feedId == "__favorite__": // Favorites are assumbed to have been read already case strings.HasPrefix(feedId, "tag:"): tag := feedId[4:] err = db.MarkUserTagArticlesByDateAsRead(user, tag, t, true) default: var id int64 id, err = strconv.ParseInt(feedId, 10, 64) /* TODO: non-fatal error */ if err != nil { break SWITCH } var feed readeef.Feed feed, err = db.GetUserFeed(id, user) /* TODO: non-fatal error */ if err != nil { break SWITCH } err = db.MarkFeedArticlesByDateAsRead(feed, t, true) } if err == nil { resp["Success"] = true } case "articles": var articles []readeef.Article var limit, offset int feedId := params["feed-id"] limit, err = strconv.Atoi(params["limit"]) /* TODO: non-fatal error */ if err != nil { break } offset, err = strconv.Atoi(params["offset"]) /* TODO: non-fatal error */ if err != nil { break } newerFirst := params["newer-first"] == "true" unreadOnly := params["unread-only"] == "true" if limit > 50 { limit = 50 } if feedId == "__favorite__" { if newerFirst { articles, err = db.GetUserFavoriteArticlesDesc(user, limit, offset) } else { articles, err = db.GetUserFavoriteArticles(user, limit, offset) } if err != nil { break } } else if feedId == "tag:__all__" { if newerFirst { if unreadOnly { articles, err = db.GetUnreadUserArticlesDesc(user, limit, offset) } else { articles, err = db.GetUserArticlesDesc(user, limit, offset) } } else { if unreadOnly { articles, err = db.GetUnreadUserArticles(user, limit, offset) } else { articles, err = db.GetUserArticles(user, limit, offset) } } if err != nil { break } } else if strings.HasPrefix(feedId, "tag:") { tag := feedId[4:] if newerFirst { if unreadOnly { articles, err = db.GetUnreadUserTagArticlesDesc(user, tag, limit, offset) } else { articles, err = db.GetUserTagArticlesDesc(user, tag, limit, offset) } } else { if unreadOnly { articles, err = db.GetUnreadUserTagArticles(user, tag, limit, offset) } else { articles, err = db.GetUserTagArticles(user, tag, limit, offset) } } if err != nil { break } } else { var f readeef.Feed var id int64 id, err = strconv.ParseInt(feedId, 10, 64) if err != nil { err = errors.New("Unknown feed id " + feedId) break } f, err = db.GetFeed(id) /* TODO: non-fatal error */ if err != nil { break } f.User = user if newerFirst { if unreadOnly { f, err = db.GetUnreadFeedArticlesDesc(f, limit, offset) } else { f, err = db.GetFeedArticlesDesc(f, limit, offset) } } else { if unreadOnly { f, err = db.GetUnreadFeedArticles(f, limit, offset) } else { f, err = db.GetFeedArticles(f, limit, offset) } } if err != nil { break } articles = f.Articles } resp["Articles"] = articles } var b []byte if err == nil { b, err = json.Marshal(resp) } if err == nil { w.Write(b) } else { webfw.GetLogger(c).Print(err) w.WriteHeader(http.StatusInternalServerError) } } }
// isAbs returns true of the URL is absolute. // This function considers protocol-relative // URLs to be absolute func isAbs(u *url.URL) bool { return u.IsAbs() || strings.HasPrefix(u.String(), "//") }
func IsAbsoluteURL(url *url.URL) bool { return url.IsAbs() }