// Build a function to check if the target URL is in scope.
func makeScopeFunc(scope []*url.URL, allowUpgrades bool) func(*url.URL) bool {
allowedScopes := make([]*url.URL, len(scope))
copy(allowedScopes, scope)
if allowUpgrades {
for _, scopeURL := range scope {
if scopeURL.Scheme == "http" {
deref := *scopeURL
clone := &deref // Can't find a way to do this in one statement
clone.Scheme = "https"
allowedScopes = append(allowedScopes, clone)
}
}
}
return func(target *url.URL) bool {
for _, scopeURL := range allowedScopes {
if util.URLIsSubpath(scopeURL, target) {
return true
}
}
return false
}
}
// Apply a filter to a channel of URLs. Runs asynchronously.
func (f *WorkFilter) RunFilter(src <-chan *url.URL) <-chan *url.URL {
c := make(chan *url.URL, f.settings.QueueSize)
go func() {
taskLoop:
for task := range src {
taskURL := task.String()
if _, ok := f.done[taskURL]; ok {
f.reject(task, "already done")
continue
}
f.done[taskURL] = true
for _, exclusion := range f.exclusions {
if util.URLIsSubpath(exclusion, task) {
f.reject(task, "excluded")
continue taskLoop
}
}
c <- task
}
close(c)
}()
return c
}
