func loadSitemapFromFile(r io.Reader) (*domain.Sitemap, error) {
var site domain.Site
err := json.NewDecoder(r).Decode(&site)
if err != nil {
return nil, err
}
sitemap := domain.NewSitemap(&site)
return sitemap, nil
}
func (builder *SitemapBuilder) Build(startingURL string) *domain.Sitemap {
var workers sync.WaitGroup
siteChannel := make(chan *domain.Site, builder.workers*5)
responseChannel := make(chan *linkResponse, builder.workers)
defer close(siteChannel)
defer close(responseChannel)
url := translateLink("", startingURL)
top := domain.NewSite(url)
sitemap := domain.NewSitemap(top)
go builder.processResponses(sitemap, responseChannel, siteChannel)
siteChannel <- top
for i := 0; i < builder.workers; i++ {
workers.Add(1)
go func(index int) {
defer workers.Done()
last := time.Date(1980, time.January, 1, 1, 1, 1, 1, time.UTC)
loop:
for {
elapsed := time.Since(last)
remaining := builder.rate - elapsed
if remaining > 0 {
select {
case <-builder.interruptChannel:
break loop
case <-time.After(remaining):
}
}
last = time.Now() // reset
select {
case <-builder.interruptChannel:
log.Println("Interrupt:", index)
break loop
case site := <-siteChannel:
builder.procesSite(site, responseChannel)
case <-time.After(builder.timeout):
log.Println("Timeout:", index)
break loop
}
}
}(i)
}
workers.Wait()
return sitemap
}