func (h *HubSpider) Setup(parent *spider.Context) (*spider.Context, error) {
ctx, err := spiderutils.NewHTTPContext("GET", h.URL, nil)
if parent != nil {
ctx.SetParent(parent)
}
return ctx, err
}
func (w *WikipediaHTMLSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
url := fmt.Sprintf("https://en.wikipedia.org/wiki/%s", w.Title)
return spiderutils.NewHTTPContext("GET", url, nil)
}
func (w *WikipediaJSONSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
params := url.Values{}
params.Add("titles", w.Title)
url := fmt.Sprintf("http://en.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&%s", params.Encode())
return spiderutils.NewHTTPContext("GET", url, nil)
}
func (s *SimpleSpider) Setup(ctx *spider.Context) (*spider.Context, error) {
return spiderutils.NewHTTPContext(s.Method, s.URL, s.Body)
}