Exemple #1
0
func loadSitemapFromFile(r io.Reader) (*domain.Sitemap, error) {
	var site domain.Site
	err := json.NewDecoder(r).Decode(&site)
	if err != nil {
		return nil, err
	}
	sitemap := domain.NewSitemap(&site)
	return sitemap, nil
}
Exemple #2
0
func (builder *SitemapBuilder) Build(startingURL string) *domain.Sitemap {
	var workers sync.WaitGroup

	siteChannel := make(chan *domain.Site, builder.workers*5)
	responseChannel := make(chan *linkResponse, builder.workers)

	defer close(siteChannel)
	defer close(responseChannel)

	url := translateLink("", startingURL)
	top := domain.NewSite(url)
	sitemap := domain.NewSitemap(top)

	go builder.processResponses(sitemap, responseChannel, siteChannel)

	siteChannel <- top

	for i := 0; i < builder.workers; i++ {
		workers.Add(1)
		go func(index int) {
			defer workers.Done()

			last := time.Date(1980, time.January, 1, 1, 1, 1, 1, time.UTC)
		loop:
			for {
				elapsed := time.Since(last)
				remaining := builder.rate - elapsed
				if remaining > 0 {
					select {
					case <-builder.interruptChannel:
						break loop
					case <-time.After(remaining):
					}
				}
				last = time.Now() // reset
				select {
				case <-builder.interruptChannel:
					log.Println("Interrupt:", index)
					break loop
				case site := <-siteChannel:
					builder.procesSite(site, responseChannel)
				case <-time.After(builder.timeout):
					log.Println("Timeout:", index)
					break loop
				}
			}
		}(i)
	}

	workers.Wait()

	return sitemap
}