func (h *HubSpider) Setup(parent *spider.Context) (*spider.Context, error) {
	ctx, err := spiderutils.NewHTTPContext("GET", h.URL, nil)
	if parent != nil {
		ctx.SetParent(parent)
	}
	return ctx, err
}
Beispiel #2
0
func (w *WikipediaHTMLSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
	url := fmt.Sprintf("https://en.wikipedia.org/wiki/%s", w.Title)
	return spiderutils.NewHTTPContext("GET", url, nil)
}
Beispiel #3
0
func (w *WikipediaJSONSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
	params := url.Values{}
	params.Add("titles", w.Title)
	url := fmt.Sprintf("http://en.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&%s", params.Encode())
	return spiderutils.NewHTTPContext("GET", url, nil)
}
Beispiel #4
0
func (s *SimpleSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
	return spiderutils.NewHTTPContext(s.Method, s.URL, s.Body)
}