예제 #1
0
파일: wiki.go 프로젝트: gnagel/spider
func (w *WikipediaHTMLSpider) Spin(ctx *spider.Context) error {
	if _, err := ctx.DoRequest(); err != nil {
		return err
	}

	html, _ := ctx.HTMLParser()
	summary := html.Find("#mw-content-text p").First().Text()

	fmt.Println(summary)
	return nil
}
예제 #2
0
파일: wiki.go 프로젝트: gnagel/spider
func (w *WikipediaJSONSpider) Spin(ctx *spider.Context) error {
	if _, err := ctx.DoRequest(); err != nil {
		return err
	}
	jsonparser, _ := ctx.JSONParser()
	pages, err := jsonparser.GetPath("query", "pages").Map()
	if err != nil {
		return err
	}
	for _, p := range pages {
		fmt.Println(p.(map[string]interface{})["extract"])
	}
	return nil
}
예제 #3
0
func (h *HubSpider) Spin(ctx *spider.Context) error {
	if h.Parent {
		snapshots, err := FindSnapshotByTime(h.DB, time.Now())
		if err == nil && snapshots != nil {
			fmt.Println("Checked today snapshot has already been saved.")
			return nil
		}
	}

	fmt.Println("Fetching ", h.Lang, "at: ", h.URL)
	if _, err := ctx.DoRequest(); err != nil {
		return err
	}
	fmt.Println("Status:", ctx.Response().Status, "for", h.Lang)

	doc, err := ctx.HTMLParser()
	if err != nil {
		return err
	}

	repos, err := ParseHTMLToRepos(doc)
	if err != nil {
		return err
	}

	ctx.Set("lang", h.Lang)
	ctx.Set("repos", repos)

	if !h.Parent {
		return nil
	}

	// Code to find all languages
	// langs := make([]langWithURL, 0)
	//
	// doc.Find(".select-menu .select-menu-list .select-menu-item[role=\"menuitem\"]").Each(func(i int, s *goquery.Selection) {
	// 	a := s.Find("a.select-menu-item-text")
	// 	lang := a.Text()
	// 	url, _ := a.Attr("href")
	// 	langs = append(langs, langWithURL{lang, url})
	// })

	allDoneChan := make(chan struct{})
	errChan := make(chan error)
	// Fetch for each languages
	var wg sync.WaitGroup
	for i, l := range langs {
		ls := NewLanguageSpider(l)
		newCtx, err := ls.Setup(ctx)
		if err != nil {
			return err
		}
		wg.Add(1)
		go func(ls spider.Spider, newCtx *spider.Context, i int) {
			defer wg.Done()
			time.Sleep(time.Duration(i) * 800 * time.Millisecond)
			if err := ls.Spin(newCtx); err != nil {
				errChan <- err
			}
		}(ls, newCtx, i)

	}

	go func() {
		wg.Wait()
		close(allDoneChan)
	}()

	select {
	case err := <-errChan:
		return err
	case <-allDoneChan:
		return h.SaveAll(ctx)
	}
}