Golang Request.GetDownloaderID Examples

Example #1

File: downloader_surfer.go Project: ReinhardHsu/pholcus

func (self *Surfer) Download(sp *spider.Spider, cReq *request.Request) *spider.Context {
	ctx := spider.GetContext(sp, cReq)

	var resp *http.Response
	var err error

	switch cReq.GetDownloaderID() {
	case SURF_ID:
		resp, err = self.surf.Download(cReq)

	case PHANTOM_ID:
		resp, err = self.phantom.Download(cReq)
	}

	if resp.StatusCode >= 400 {
		err = errors.New("响应状态 " + resp.Status)
	}

	if resp.Header.Get("Content-Encoding") == "gzip" {
		var gzipReader *gzip.Reader
		gzipReader, err = gzip.NewReader(resp.Body)
		resp.Body.Close()
		if err == nil {
			resp.Body = ioutil.NopCloser(gzipReader)
		}
	}

	ctx.SetResponse(resp).SetError(err)

	return ctx
}