Beispiel #1
0
func rewriteAndDownloadImages(root string, content string) (*html.Node, error) {
	var wg sync.WaitGroup
	imageDownloader := newDownloader(root, timeout)
	doc, err := boots.Walk(strings.NewReader(content), "img", func(node *html.Node) {
		for index, attr := range node.Attr {
			if attr.Key == "src" {
				uri := attr.Val
				altered := fmt.Sprintf("%x.jpg", hashie.Sha1([]byte(uri)))
				wg.Add(1)
				go func() {
					defer wg.Done()
					logger.Printf("downloading image: %s", uri)
					if err := imageDownloader.downloadToFile(uri, altered); err != nil {
						logger.Printf("downloading image failed: %s", err)
					}
				}()
				node.Attr[index].Val = altered
				break
			}
		}
	})
	wg.Wait()
	return doc, err
}
Beispiel #2
0
func key(thing string) string {
	return hashie.Sha1([]byte(thing), []byte(":blacklisted"))
}
Beispiel #3
0
func (j *Job) Hash() string {
	return hashie.Sha1([]byte(j.Url), j.Key[:])
}