Example #1
0
func rewriteAndDownloadImages(j *job.Job, doc *h5.Node) *h5.Node {
	var wg sync.WaitGroup
	root := j.Root()
	t := transform.NewTransform(doc)
	fix := transform.TransformAttrib("src", func(uri string) string {
		altered := fmt.Sprintf("%x.jpg", hashie.Sha1([]byte(uri)))
		wg.Add(1)
		go safely.Ignore(logger, func() {
			defer wg.Done()
			downloadToFile(uri, fmt.Sprintf("%s/%s", root, altered))
		})
		return altered
	})
	t.Apply(fix, "img")
	wg.Wait()
	return t.Doc()
}
Example #2
0
func IsBlacklisted(thing string) bool {
	if _, err := cache.Get(hashie.Sha1([]byte(thing))); err == nil {
		return true
	}
	return false
}
Example #3
0
func Blacklist(thing string) {
	cache.Set(hashie.Sha1([]byte(thing)), "blacklisted", TTL)
}
Example #4
0
func (j *Job) Hash() string {
	return hashie.Sha1([]byte(j.Url.String()), j.Key)
}