func apiGetSiteInfo(g *gin.Context) {
url, ok := g.GetQuery("url")
if !ok {
g.String(500, "missing parameter url")
return
}
_, inclBody := g.GetQuery("body")
cw := crawlbase.NewCrawler()
cw.Header.Add("User-Agent", userAgentHeader)
tags, _ := crawlbase.LoadTagsFromFile("tags.json")
cw.Validator.AddValidTags(tags)
page, err := cw.GetPage(url, "GET")
if err != nil {
g.String(500, err.Error())
}
// first check
if !inclBody {
page.RespInfo.Body = ""
}
g.JSON(200, page)
}
func apiAddTag(g *gin.Context) {
if g.Request.Method != "POST" {
g.String(404, "not a get")
return
}
tName, _ := g.GetQuery("TagName")
aName, _ := g.GetQuery("AttributeName")
if tName == "" {
g.String(200, "TagName is empty")
return
}
tags, _ := crawlbase.LoadTagsFromFile("tags.json")
tag, ok := findTag(tags, tName)
if !ok {
tag = &htmlcheck.ValidTag{tName, []string{}, "", false}
tags = append(tags, tag)
}
ok = findString(tag.Attrs, aName)
if !ok {
tag.Attrs = append(tag.Attrs, aName)
}
//t, _ := findTag(tags, tName)
crawlbase.WriteTagsToFile(tags, "tags.json")
g.String(200, "ok")
}