func (this bing_search) SpellingSuggestion(params BingSearchParameters) (curl.Cache, error) {
params.Format = new_string(SearchResultFormatJson)
q := oauth2.HttpQueryEncode(params)
c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyNoProxy, 0, this, nil)
x := BingSearchServiceRoot + ServiceOperationSpellingSuggestion + "?" + q
return c.Get(x)
}
func image_icon(uri string) (feed.FeedImage, error) {
imgo := new_imagecache_operator()
v, err := imgo.find(uri)
if err == nil {
return v, nil
}
u, err := url.ParseRequestURI(uri)
c := curl.NewCurlerDetail(backend_context.config.ImageFolder, 0, 0, nil, backend_context.ruler)
candis := []string{u.String(), "/", "/favicon.ico"}
for _, candi := range candis {
x, _ := u.Parse(candi)
cache, _ := c.Get(x.String())
if cache.Mime == "text/html" {
cache, _ = icon_from_link_rel(cache.Local)
}
if strings.Split(cache.Mime, "/")[0] == "image" {
v.Mime = cache.Mime
v.Origin = cache.Local
go imgo.save(uri, v)
return v, nil
}
}
return v, new_backenderror(-1, "icon cannot resolve")
}
func (this bcms_queue) FetchAny(msgid, fetch_num *uint) ([]string, error) {
v := make([]string, 0)
var err error
q := bcms_fetch{
bcms_common: bcms_common{
method: "fetch",
timestamp: unix_current(),
access_token: this.access_token,
client_id: this.client_id,
},
msg_id: msgid,
fetch_num: fetch_num,
}
var bcmsr struct {
RequestId int64 `json:"request_id"`
Response BcmsResponse `json:"response_params"`
}
uri := bcms_urlbase + this.queue_name
q.sign = bcms_sign("POST", uri, q, this.client_secret) // use client_secret to sign post-params
vals := oauth2.HttpQueryValues(q)
c := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
err = c.PostFormAsJson(uri, vals, &bcmsr)
if err == nil {
for _, i := range bcmsr.Response.Messages {
v = append(v, i.Message)
}
}
return v, err
}
func icon_from_link_rel(local string) (curl.Cache, error) {
f, err := os.Open(local)
if err != nil {
return curl.Cache{}, err
}
defer f.Close()
doc, err := html.Parse(f)
if err != nil {
return curl.Cache{}, err
}
de := node_query_select(doc, "html")
head := node_query_select(de, "head")
links := node_query_selects(head, "link")
var href string
for _, link := range links {
rel := node_get_attribute(link, "rel")
if rel == "icon" || rel == "shortcut icon" || rel == "apple-touch-icon" {
href = node_get_attribute(link, "href")
break
}
}
if href != "" {
c := curl.NewCurlerDetail(backend_config().ImageFolder, 0, 0, nil, backend_context.ruler)
return c.Get(href)
}
return curl.Cache{}, new_backenderror(-1, "icon cannot resolved in html")
}
func feedentry_fulldoc(uri string) (v feed.FeedContent, err error) {
c := curl.NewCurlerDetail(backend_context.config.DocumentFolder, 0, 0, nil, backend_context.ruler)
cache, err := c.GetUtf8(uri)
if err != nil {
return v, err
}
v.Uri = uri
v.Local = cache.LocalUtf8
if curl.MimeToExt(cache.Mime) != "html" {
return v, new_backenderror(-1, "unsupported mime: "+cache.Mime)
}
doc, err := html_create_from_file(cache.LocalUtf8)
if err != nil {
return v, err
}
article, sum, err := cleaner.NewExtractor(backend_context.config.CleanFolder).MakeHtmlReadable(doc, uri)
if err != nil {
return v, err
}
v.Local, err = html_write_file(article, backend_context.config.DocumentFolder)
redirector := func(turi string) string {
return redirect_thumbnail(url_resolve(uri, turi))
}
imgurl_maker := func(reluri string) string {
u := url_resolve(uri, reluri)
return imageurl_from_video(u)
}
v.Images = append_unique(v.Images, feedmedias_from_docsummary(sum.Images, redirector)...)
v.Images = append_unique(v.Images, feedmedias_from_docsummary(sum.Medias, imgurl_maker)...)
v.Words = uint(sum.WordCount)
v.Links = uint(sum.LinkCount)
v.FlowDoc = new_flowdoc_maker().make(article, v.Images)
return v, err
}
func (this pubsuber) Retrieve(uri string, count int) (string, error) {
p := struct {
mode string `param:"hub.mode"`
topic string `param:"hub.topic"`
count int `param:"count"`
format string `param:"format"`
}{"retrieve", uri, count, "json"}
u := this.service_provider + "?" + oauth2.HttpQueryEncode(p)
return curl.NewCurlerDetail("", this.proxy_policy, 0, nil, nil).GetAsString(u)
}
func (this pubsuber) Unsubscribe(uri string) (int, error) {
p := pubunsub_param{
mode: "unsubscribe",
topic: uri,
callback: hub_callback,
verify: &this.verify,
}
c := curl.NewCurlerDetail("", this.proxy_policy, 0, nil, nil)
resp, err := c.PostForm(this.service_provider, oauth2.HttpQueryValues(p))
return resp, err
}
func (this bcms_proxy) FetchOne() (string, error) {
var x BcmsResponse
client := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
err := client.GetAsJson(this.uri, &x)
if err != nil {
return "", err
}
if x.MessageNum != 1 {
return "", fmt.Errorf("bcmd-message invalid-msgnum: ", x.MessageNum)
}
return x.Messages[0].Message, nil
}
func (this google_feedapi) Find(q, hl string) ([]feed.FeedEntity, error) {
p := struct {
q string `param:"q"`
hl string `param:"hl"` // default en, nil means en
v string `param:"v"`
}{q: q, hl: hl, v: "1.0"}
v := find_result{}
uri := find_url + "?" + oauth2.HttpQueryEncode(p)
c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyAlwayseProxy, 0, this, nil)
err := c.GetAsJson(uri, &v)
if err != nil {
return nil, err
}
x, err := v.to_feedentities()
return x, err
}
func (this google_feedapi) Load(uri, hl string, num int, scoring bool) (feed.FeedEntity, error) {
p := struct {
q string `param:"q"`
hl string `param:"hl"` // default en
num *int `param:"num"` // default 4
scoring string `param:"scoring"` // nil or h
v string `param:"v"`
}{q: uri, hl: hl, num: make_num(num), scoring: make_scoring(scoring), v: "1.0"}
c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyAlwayseProxy, 0, this, nil)
v := load_result{}
url := load_url + "?" + oauth2.HttpQueryEncode(p)
err := c.GetAsJson(url, &v)
if err != nil {
return feed.FeedEntity{}, err
}
s, err := v.to_feedentity()
return s, err
}
func image_description(uri string) (feed.FeedImage, error) {
imgo := new_imagecache_operator()
v, err := imgo.find(uri)
if err == nil {
return v, nil
}
c := curl.NewCurlerDetail(backend_context.config.ImageFolder, 0, 0, nil, backend_context.ruler)
cache, err := c.Get(uri)
v.Mime = cache.Mime
v.Origin = cache.Local
if err != nil {
return v, err
}
v.Thumbnail, v.Mime, v.Width, v.Height, err = curl.NewThumbnail(cache.Local, backend_context.config.ThumbnailFolder, backend_context.config.ThumbnailWidth, 0)
go imgo.save(uri, v)
return v, err
}
func (this processor_qq) process(uri *url.URL) (VideoDescription, error) {
cid := uri.Query().Get("cid")
vid := uri.Query().Get("vid")
if cid != "" && vid != "" {
return this.image_from_cidvid(cid, vid), nil
}
r1s := r1.FindStringSubmatch(uri.Path)
if len(r1s) != 0 && vid != "" {
return this.image_from_cidvid(r1s[1], vid), nil
}
r3s := r3.FindStringSubmatch(uri.Path)
if len(r3s) == 3 {
return this.image_from_cidvid(r3s[1], r3s[2]), nil
}
c := curl.NewCurlerDetail("", 0, 0, nil, nil)
cache, err := c.GetUtf8(uri.String())
if err != nil {
return VideoDescription{}, err
}
return this.image_from_htmlbody(cache.LocalUtf8)
}
// source : feed(atom/rss) url
// count: must large than 0
// page: 0 based page index
// if page is 0, entries may be fetched online
func feedentry_unread(source string, count int, page int) ([]ReadEntry, error, int) {
if count <= 0 {
panic("invalid arg count")
}
var sc int
if page == 0 {
log.Println("curl-get...")
c := curl.NewCurlerDetail(backend_config().FeedEntryFolder, 0, 0, nil, backend_context.ruler)
cache, err := c.GetUtf8(source)
log.Println("curl-get", cache.LocalUtf8)
if err != nil || cache.LocalUtf8 == "" {
return nil, err, cache.StatusCode
}
ext := curl.MimeToExt(cache.Mime)
if ext != "xml" && ext != "atom+xml" && ext != "rss+xml" {
return nil, new_backenderror(cache.StatusCode, "unsupported mime: "+cache.Mime), 0
}
f, err := os.Open(cache.LocalUtf8)
if err != nil {
return nil, err, cache.StatusCode
}
fs, v, err := feed.NewFeedMaker(f, source).MakeFeed()
f.Close()
rs := new_readsource(fs)
if err == nil {
new_feedsource_operator().update(rs)
log.Println("feed-update", fs.Name)
}
rv := readentry_filter(new_readentries(v))
log.Println("feedentries-filter", len(rv))
sc = cache.StatusCode
}
rv, err := new_feedentry_operator().topn_by_feedsource(count*page, count, source)
log.Println("unread-return(uri, page, count)", source, page, count, len(rv), err)
return rv, err, sc
}
func (this bcms_proxy) FetchOneAsJson(v interface{}) error {
var x BcmsResponse
client := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
err := client.GetAsJson(this.uri, &x)
if err != nil {
return err
}
if x.MessageNum != 1 {
return fmt.Errorf("bcms-message: %v unresolved message", x.MessageNum)
}
body := x.Messages[0].Message
switch body[0] {
case '{':
fallthrough
case '[':
err = json.Unmarshal([]byte(body), v)
case '<':
err = xml.Unmarshal([]byte(body), v)
default:
err = fmt.Errorf("bcms-decode informal format:", body)
}
return err
}
func (this bing_search) Composite(params BingSearchCompositeParameters) (curl.Cache, error) {
q := oauth2.HttpQueryEncode(params)
c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyNoProxy, 0, this, nil)
return c.Get(BingSearchServiceRoot + ServiceOperationComposite + "?" + q)
}