func rewriteAndDownloadImages(j *job.Job, doc *h5.Node) *h5.Node {
var wg sync.WaitGroup
root := j.Root()
t := transform.NewTransform(doc)
fix := transform.TransformAttrib("src", func(uri string) string {
altered := fmt.Sprintf("%x.jpg", hashie.Sha1([]byte(uri)))
wg.Add(1)
go safely.Ignore(logger, func() {
defer wg.Done()
downloadToFile(uri, fmt.Sprintf("%s/%s", root, altered))
})
return altered
})
t.Apply(fix, "img")
wg.Wait()
return t.Doc()
}
func IsBlacklisted(thing string) bool {
if _, err := cache.Get(hashie.Sha1([]byte(thing))); err == nil {
return true
}
return false
}
func Blacklist(thing string) {
cache.Set(hashie.Sha1([]byte(thing)), "blacklisted", TTL)
}
func (j *Job) Hash() string {
return hashie.Sha1([]byte(j.Url.String()), j.Key)
}