Example #1
0
func reqToMsg(req *crawler.Request) *requestMsg {
	m := &requestMsg{
		URL:     req.URL.String(),
		Headers: req.Header,
	}
	// if req.Proxy != nil {
	// 	m.Proxy = req.Proxy.String()
	// }
	if ctx := req.Context(); ctx != nil {
		config := ConfigFrom(ctx.C)
		if config != nil {
			m.Mode = config.Mode
			m.Timeout = int(config.Timeout / time.Millisecond)
			m.FetchCode = m.FetchCode
			m.Injection = m.Injection
		}
	}
	// for _, cookie := range req.Cookies {
	// 	m.Cookies = append(m.Cookies, struct {
	// 		Name  string `json:"name"`
	// 		Value string `json:"value"`
	// 	}{Name: cookie.Name, Value: cookie.Value})
	// }
	return m
}
Example #2
0
func Prepare(req *crawler.Request, conf *BrowserConfig) {
	req.Context().C = WithConfig(req.Context().C, conf)
}