func rewriteAndDownloadImages(root string, content string) (*html.Node, error) {
var wg sync.WaitGroup
imageDownloader := newDownloader(root, timeout)
doc, err := boots.Walk(strings.NewReader(content), "img", func(node *html.Node) {
for index, attr := range node.Attr {
if attr.Key == "src" {
uri := attr.Val
altered := fmt.Sprintf("%x.jpg", hashie.Sha1([]byte(uri)))
wg.Add(1)
go func() {
defer wg.Done()
logger.Printf("downloading image: %s", uri)
if err := imageDownloader.downloadToFile(uri, altered); err != nil {
logger.Printf("downloading image failed: %s", err)
}
}()
node.Attr[index].Val = altered
break
}
}
})
wg.Wait()
return doc, err
}
func key(thing string) string {
return hashie.Sha1([]byte(thing), []byte(":blacklisted"))
}
func (j *Job) Hash() string {
return hashie.Sha1([]byte(j.Url), j.Key[:])
}