func apiGetSiteInfo(g *gin.Context) {
url, ok := g.GetQuery("url")
if !ok {
g.String(500, "missing parameter url")
return
}
_, inclBody := g.GetQuery("body")
cw := crawlbase.NewCrawler()
cw.Header.Add("User-Agent", userAgentHeader)
tags, _ := crawlbase.LoadTagsFromFile("tags.json")
cw.Validator.AddValidTags(tags)
page, err := cw.GetPage(url, "GET")
if err != nil {
g.String(500, err.Error())
}
// first check
if !inclBody {
page.RespInfo.Body = ""
}
g.JSON(200, page)
}
func resp(r *http.Request, req *http.Response, dur time.Duration) {
cw := crawlbase.NewCrawler()
page := cw.PageFromResponse(r, req, dur)
c := db.C("requests")
c.Insert(page)
}