func rewriteAndDownloadImages(root string, content string) (*html.Node, error) { var wg sync.WaitGroup imageDownloader := newDownloader(root, timeout) doc, err := boots.Walk(strings.NewReader(content), "img", func(node *html.Node) { for index, attr := range node.Attr { if attr.Key == "src" { uri := attr.Val altered := fmt.Sprintf("%x.jpg", hashie.Sha1([]byte(uri))) wg.Add(1) go func() { defer wg.Done() logger.Printf("downloading image: %s", uri) if err := imageDownloader.downloadToFile(uri, altered); err != nil { logger.Printf("downloading image failed: %s", err) } }() node.Attr[index].Val = altered break } } }) wg.Wait() return doc, err }
func key(thing string) string { return hashie.Sha1([]byte(thing), []byte(":blacklisted")) }
func (j *Job) Hash() string { return hashie.Sha1([]byte(j.Url), j.Key[:]) }