Esempio n. 1
0
func (this bing_search) SpellingSuggestion(params BingSearchParameters) (curl.Cache, error) {
	params.Format = new_string(SearchResultFormatJson)
	q := oauth2.HttpQueryEncode(params)
	c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyNoProxy, 0, this, nil)
	x := BingSearchServiceRoot + ServiceOperationSpellingSuggestion + "?" + q
	return c.Get(x)
}
Esempio n. 2
0
func image_icon(uri string) (feed.FeedImage, error) {
	imgo := new_imagecache_operator()
	v, err := imgo.find(uri)

	if err == nil {
		return v, nil
	}
	u, err := url.ParseRequestURI(uri)
	c := curl.NewCurlerDetail(backend_context.config.ImageFolder, 0, 0, nil, backend_context.ruler)
	candis := []string{u.String(), "/", "/favicon.ico"}
	for _, candi := range candis {
		x, _ := u.Parse(candi)
		cache, _ := c.Get(x.String())
		if cache.Mime == "text/html" {
			cache, _ = icon_from_link_rel(cache.Local)
		}
		if strings.Split(cache.Mime, "/")[0] == "image" {
			v.Mime = cache.Mime
			v.Origin = cache.Local
			go imgo.save(uri, v)
			return v, nil
		}
	}
	return v, new_backenderror(-1, "icon cannot resolve")
}
Esempio n. 3
0
func (this bcms_queue) FetchAny(msgid, fetch_num *uint) ([]string, error) {
	v := make([]string, 0)
	var err error
	q := bcms_fetch{
		bcms_common: bcms_common{
			method:       "fetch",
			timestamp:    unix_current(),
			access_token: this.access_token,
			client_id:    this.client_id,
		},
		msg_id:    msgid,
		fetch_num: fetch_num,
	}

	var bcmsr struct {
		RequestId int64        `json:"request_id"`
		Response  BcmsResponse `json:"response_params"`
	}
	uri := bcms_urlbase + this.queue_name
	q.sign = bcms_sign("POST", uri, q, this.client_secret) // use client_secret to sign post-params
	vals := oauth2.HttpQueryValues(q)
	c := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
	err = c.PostFormAsJson(uri, vals, &bcmsr)

	if err == nil {
		for _, i := range bcmsr.Response.Messages {
			v = append(v, i.Message)
		}
	}
	return v, err
}
Esempio n. 4
0
func icon_from_link_rel(local string) (curl.Cache, error) {
	f, err := os.Open(local)
	if err != nil {
		return curl.Cache{}, err
	}
	defer f.Close()
	doc, err := html.Parse(f)
	if err != nil {
		return curl.Cache{}, err
	}
	de := node_query_select(doc, "html")
	head := node_query_select(de, "head")
	links := node_query_selects(head, "link")
	var href string
	for _, link := range links {
		rel := node_get_attribute(link, "rel")
		if rel == "icon" || rel == "shortcut icon" || rel == "apple-touch-icon" {
			href = node_get_attribute(link, "href")
			break
		}
	}
	if href != "" {
		c := curl.NewCurlerDetail(backend_config().ImageFolder, 0, 0, nil, backend_context.ruler)
		return c.Get(href)
	}
	return curl.Cache{}, new_backenderror(-1, "icon cannot resolved in html")
}
Esempio n. 5
0
func feedentry_fulldoc(uri string) (v feed.FeedContent, err error) {
	c := curl.NewCurlerDetail(backend_context.config.DocumentFolder, 0, 0, nil, backend_context.ruler)
	cache, err := c.GetUtf8(uri)
	if err != nil {
		return v, err
	}
	v.Uri = uri
	v.Local = cache.LocalUtf8
	if curl.MimeToExt(cache.Mime) != "html" {
		return v, new_backenderror(-1, "unsupported mime: "+cache.Mime)
	}
	doc, err := html_create_from_file(cache.LocalUtf8)
	if err != nil {
		return v, err
	}
	article, sum, err := cleaner.NewExtractor(backend_context.config.CleanFolder).MakeHtmlReadable(doc, uri)
	if err != nil {
		return v, err
	}
	v.Local, err = html_write_file(article, backend_context.config.DocumentFolder)
	redirector := func(turi string) string {
		return redirect_thumbnail(url_resolve(uri, turi))
	}
	imgurl_maker := func(reluri string) string {
		u := url_resolve(uri, reluri)
		return imageurl_from_video(u)
	}
	v.Images = append_unique(v.Images, feedmedias_from_docsummary(sum.Images, redirector)...)
	v.Images = append_unique(v.Images, feedmedias_from_docsummary(sum.Medias, imgurl_maker)...)
	v.Words = uint(sum.WordCount)
	v.Links = uint(sum.LinkCount)
	v.FlowDoc = new_flowdoc_maker().make(article, v.Images)
	return v, err
}
Esempio n. 6
0
func (this pubsuber) Retrieve(uri string, count int) (string, error) {
	p := struct {
		mode   string `param:"hub.mode"`
		topic  string `param:"hub.topic"`
		count  int    `param:"count"`
		format string `param:"format"`
	}{"retrieve", uri, count, "json"}
	u := this.service_provider + "?" + oauth2.HttpQueryEncode(p)
	return curl.NewCurlerDetail("", this.proxy_policy, 0, nil, nil).GetAsString(u)
}
Esempio n. 7
0
func (this pubsuber) Unsubscribe(uri string) (int, error) {
	p := pubunsub_param{
		mode:     "unsubscribe",
		topic:    uri,
		callback: hub_callback,
		verify:   &this.verify,
	}
	c := curl.NewCurlerDetail("", this.proxy_policy, 0, nil, nil)
	resp, err := c.PostForm(this.service_provider, oauth2.HttpQueryValues(p))
	return resp, err
}
Esempio n. 8
0
func (this bcms_proxy) FetchOne() (string, error) {
	var x BcmsResponse
	client := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
	err := client.GetAsJson(this.uri, &x)
	if err != nil {
		return "", err
	}
	if x.MessageNum != 1 {
		return "", fmt.Errorf("bcmd-message invalid-msgnum: ", x.MessageNum)
	}
	return x.Messages[0].Message, nil
}
Esempio n. 9
0
func (this google_feedapi) Find(q, hl string) ([]feed.FeedEntity, error) {
	p := struct {
		q  string `param:"q"`
		hl string `param:"hl"` // default en, nil means en
		v  string `param:"v"`
	}{q: q, hl: hl, v: "1.0"}
	v := find_result{}
	uri := find_url + "?" + oauth2.HttpQueryEncode(p)
	c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyAlwayseProxy, 0, this, nil)
	err := c.GetAsJson(uri, &v)
	if err != nil {
		return nil, err
	}
	x, err := v.to_feedentities()
	return x, err
}
Esempio n. 10
0
func (this google_feedapi) Load(uri, hl string, num int, scoring bool) (feed.FeedEntity, error) {
	p := struct {
		q       string `param:"q"`
		hl      string `param:"hl"`      // default en
		num     *int   `param:"num"`     // default 4
		scoring string `param:"scoring"` // nil or h
		v       string `param:"v"`
	}{q: uri, hl: hl, num: make_num(num), scoring: make_scoring(scoring), v: "1.0"}
	c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyAlwayseProxy, 0, this, nil)
	v := load_result{}
	url := load_url + "?" + oauth2.HttpQueryEncode(p)
	err := c.GetAsJson(url, &v)

	if err != nil {
		return feed.FeedEntity{}, err
	}
	s, err := v.to_feedentity()
	return s, err
}
Esempio n. 11
0
func image_description(uri string) (feed.FeedImage, error) {
	imgo := new_imagecache_operator()
	v, err := imgo.find(uri)

	if err == nil {
		return v, nil
	}
	c := curl.NewCurlerDetail(backend_context.config.ImageFolder, 0, 0, nil, backend_context.ruler)
	cache, err := c.Get(uri)

	v.Mime = cache.Mime
	v.Origin = cache.Local

	if err != nil {
		return v, err
	}
	v.Thumbnail, v.Mime, v.Width, v.Height, err = curl.NewThumbnail(cache.Local, backend_context.config.ThumbnailFolder, backend_context.config.ThumbnailWidth, 0)
	go imgo.save(uri, v)
	return v, err
}
Esempio n. 12
0
func (this processor_qq) process(uri *url.URL) (VideoDescription, error) {
	cid := uri.Query().Get("cid")
	vid := uri.Query().Get("vid")
	if cid != "" && vid != "" {
		return this.image_from_cidvid(cid, vid), nil
	}
	r1s := r1.FindStringSubmatch(uri.Path)
	if len(r1s) != 0 && vid != "" {
		return this.image_from_cidvid(r1s[1], vid), nil
	}
	r3s := r3.FindStringSubmatch(uri.Path)
	if len(r3s) == 3 {
		return this.image_from_cidvid(r3s[1], r3s[2]), nil
	}
	c := curl.NewCurlerDetail("", 0, 0, nil, nil)
	cache, err := c.GetUtf8(uri.String())
	if err != nil {
		return VideoDescription{}, err
	}
	return this.image_from_htmlbody(cache.LocalUtf8)
}
Esempio n. 13
0
// source : feed(atom/rss) url
// count: must large than 0
// page: 0 based page index
// if page is 0, entries may be fetched online
func feedentry_unread(source string, count int, page int) ([]ReadEntry, error, int) {
	if count <= 0 {
		panic("invalid arg count")
	}
	var sc int
	if page == 0 {
		log.Println("curl-get...")
		c := curl.NewCurlerDetail(backend_config().FeedEntryFolder, 0, 0, nil, backend_context.ruler)
		cache, err := c.GetUtf8(source)
		log.Println("curl-get", cache.LocalUtf8)
		if err != nil || cache.LocalUtf8 == "" {
			return nil, err, cache.StatusCode
		}

		ext := curl.MimeToExt(cache.Mime)
		if ext != "xml" && ext != "atom+xml" && ext != "rss+xml" {
			return nil, new_backenderror(cache.StatusCode, "unsupported mime: "+cache.Mime), 0
		}
		f, err := os.Open(cache.LocalUtf8)
		if err != nil {
			return nil, err, cache.StatusCode
		}

		fs, v, err := feed.NewFeedMaker(f, source).MakeFeed()
		f.Close()
		rs := new_readsource(fs)
		if err == nil {
			new_feedsource_operator().update(rs)
			log.Println("feed-update", fs.Name)
		}
		rv := readentry_filter(new_readentries(v))
		log.Println("feedentries-filter", len(rv))
		sc = cache.StatusCode
	}
	rv, err := new_feedentry_operator().topn_by_feedsource(count*page, count, source)
	log.Println("unread-return(uri, page, count)", source, page, count, len(rv), err)
	return rv, err, sc
}
Esempio n. 14
0
func (this bcms_proxy) FetchOneAsJson(v interface{}) error {
	var x BcmsResponse
	client := curl.NewCurlerDetail("", curl.CurlProxyPolicyNoProxy, 0, nil, nil)
	err := client.GetAsJson(this.uri, &x)
	if err != nil {
		return err
	}
	if x.MessageNum != 1 {
		return fmt.Errorf("bcms-message: %v unresolved message", x.MessageNum)
	}
	body := x.Messages[0].Message
	switch body[0] {
	case '{':
		fallthrough
	case '[':
		err = json.Unmarshal([]byte(body), v)
	case '<':
		err = xml.Unmarshal([]byte(body), v)
	default:
		err = fmt.Errorf("bcms-decode informal format:", body)
	}
	return err
}
Esempio n. 15
0
func (this bing_search) Composite(params BingSearchCompositeParameters) (curl.Cache, error) {
	q := oauth2.HttpQueryEncode(params)
	c := curl.NewCurlerDetail(this.temp_folder, curl.CurlProxyPolicyNoProxy, 0, this, nil)
	return c.Get(BingSearchServiceRoot + ServiceOperationComposite + "?" + q)
}