Example #1
0
// Build a function to check if the target URL is in scope.
func makeScopeFunc(scope []*url.URL, allowUpgrades bool) func(*url.URL) bool {
	allowedScopes := make([]*url.URL, len(scope))
	copy(allowedScopes, scope)
	if allowUpgrades {
		for _, scopeURL := range scope {
			if scopeURL.Scheme == "http" {
				deref := *scopeURL
				clone := &deref // Can't find a way to do this in one statement
				clone.Scheme = "https"
				allowedScopes = append(allowedScopes, clone)
			}
		}
	}
	return func(target *url.URL) bool {
		for _, scopeURL := range allowedScopes {
			if util.URLIsSubpath(scopeURL, target) {
				return true
			}
		}
		return false
	}
}
Example #2
0
// Apply a filter to a channel of URLs.  Runs asynchronously.
func (f *WorkFilter) RunFilter(src <-chan *url.URL) <-chan *url.URL {
	c := make(chan *url.URL, f.settings.QueueSize)
	go func() {
	taskLoop:
		for task := range src {
			taskURL := task.String()
			if _, ok := f.done[taskURL]; ok {
				f.reject(task, "already done")
				continue
			}
			f.done[taskURL] = true
			for _, exclusion := range f.exclusions {
				if util.URLIsSubpath(exclusion, task) {
					f.reject(task, "excluded")
					continue taskLoop
				}
			}
			c <- task
		}
		close(c)
	}()
	return c
}