Example #1
0
func apiGetSiteInfo(g *gin.Context) {
	url, ok := g.GetQuery("url")
	if !ok {
		g.String(500, "missing parameter url")
		return
	}
	_, inclBody := g.GetQuery("body")

	cw := crawlbase.NewCrawler()
	cw.Header.Add("User-Agent", userAgentHeader)
	tags, _ := crawlbase.LoadTagsFromFile("tags.json")
	cw.Validator.AddValidTags(tags)

	page, err := cw.GetPage(url, "GET")

	if err != nil {
		g.String(500, err.Error())
	}

	// first check

	if !inclBody {
		page.RespInfo.Body = ""
	}

	g.JSON(200, page)
}
Example #2
0
func resp(r *http.Request, req *http.Response, dur time.Duration) {
	cw := crawlbase.NewCrawler()
	page := cw.PageFromResponse(r, req, dur)
	c := db.C("requests")
	c.Insert(page)
}