func (h *HubSpider) Setup(parent *spider.Context) (*spider.Context, error) { ctx, err := spiderutils.NewHTTPContext("GET", h.URL, nil) if parent != nil { ctx.SetParent(parent) } return ctx, err }
func (w *WikipediaHTMLSpider) Setup(ctx *spider.Context) (*spider.Context, error) { url := fmt.Sprintf("https://en.wikipedia.org/wiki/%s", w.Title) return spiderutils.NewHTTPContext("GET", url, nil) }
func (w *WikipediaJSONSpider) Setup(ctx *spider.Context) (*spider.Context, error) { params := url.Values{} params.Add("titles", w.Title) url := fmt.Sprintf("http://en.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&%s", params.Encode()) return spiderutils.NewHTTPContext("GET", url, nil) }
func (s *SimpleSpider) Setup(ctx *spider.Context) (*spider.Context, error) { return spiderutils.NewHTTPContext(s.Method, s.URL, s.Body) }