func fetchList(url, prefix string) ([]string, error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("failed to fetch %s - %s", url, res.Status)
	}
	list := []string{}
	selector := cascadia.MustCompile("a")
	webdevdata.ProcessMatchingTagsReader(res.Body, "table tbody tr > td:first-of-type", func(node *html.Node) {
		pkg := ""
		link := selector.MatchFirst(node)
		if link != nil {
			pkg = webdevdata.GetAttr("href", link.Attr)
		} else if node.FirstChild != nil && node.FirstChild.Type == html.TextNode {
			pkg = node.FirstChild.Data
		} else if node.FirstChild != nil && node.FirstChild.Data == "b" {
			return
		}
		if pkg == "" {
			log.Fatal("markup from godoc.org changed")
		}
		p := strings.TrimLeft(pkg, "/")
		if !strings.HasPrefix(p, prefix) {
			return
		}
		list = append(list, p)
	})
	return list, nil
}
Exemple #2
0
func process(file string, selector string, attrList []string, csv *csv.Writer) {
	webdevdata.ProcessMatchingTags(file, selector, func(node *html.Node) {
		content := []string{file, node.Data}
		for _, attr := range attrList {
			if attr != "" {
				content = append(content, webdevdata.GetAttr(attr, node.Attr))
			}
		}
		csv.Write(content)
	})
}
Exemple #3
0
func main() {
	flag.Parse()
	file := flag.Arg(0)

	csv := csv.NewWriter(os.Stdout)
	selector := "meta[name]" // All meta tags with name attribute
	webdevdata.ProcessMatchingTags(file, selector, func(node *html.Node) {
		name := webdevdata.GetAttr("name", node.Attr)
		csv.Write([]string{file, name})
	})

	csv.Flush()
}
Exemple #4
0
func main() {
	flag.Parse()
	file := flag.Arg(0)

	csv := csv.NewWriter(os.Stdout)
	selector := "html[manifest]" // all html tags with manifest attribute
	webdevdata.ProcessMatchingTags(file, selector, func(node *html.Node) {
		manifest := webdevdata.GetAttr("manifest", node.Attr)
		if manifest != "" {
			csv.Write([]string{file, manifest})
		}
	})

	csv.Flush()
}