// domainsForRateLimiting transforms a list of FQDNs into a list of eTLD+1's
// for the purpose of rate limiting. It also de-duplicates the output
// domains.
func domainsForRateLimiting(names []string) ([]string, error) {
domainsMap := make(map[string]struct{}, len(names))
var domains []string
for _, name := range names {
eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(name)
if err != nil {
// The only possible errors are:
// (1) publicsuffix.PublicSuffix is giving garbage
// values
// (2) the public suffix is the domain itself
//
// Assume (2).
eTLDPlusOne = name
}
if _, ok := domainsMap[eTLDPlusOne]; !ok {
domainsMap[eTLDPlusOne] = struct{}{}
domains = append(domains, eTLDPlusOne)
}
}
return domains, nil
}
func main() {
cacheFile := flag.String("ctFile", "", "")
nameFile := flag.String("nameFile", "", "")
normalResolver := flag.String("normalResolver", "127.0.0.1:53", "")
torResolver := flag.String("torResolver", "127.0.0.1:9053", "")
workers := flag.Int("workers", 1, "")
flag.Parse()
if *cacheFile == "" && *nameFile == "" {
fmt.Println("Either --ctFile or --nameFile is required")
os.Exit(1)
}
t := tester{
workers: *workers,
client: &dns.Client{DialTimeout: dnsTimeout, ReadTimeout: dnsTimeout, Net: "tcp"},
normalResolver: *normalResolver,
torResolver: *torResolver,
}
if *cacheFile != "" {
ctEntries, err := ctCommon.LoadCacheFile(*cacheFile)
if err != nil {
fmt.Printf("Failed to load CT cache file: %s\n", err)
os.Exit(1)
}
names := make(map[string]struct{})
mu := new(sync.Mutex)
ctEntries.Map(func(ent *ct.EntryAndPosition, err error) {
if err != nil || ent.Entry.Type != ct.X509Entry {
return
}
cert, _, err := ctCommon.ParseAndFilter(ent.Entry.X509Cert, nil)
if err != nil {
return
}
for _, name := range cert.DNSNames {
eTLD, err := publicsuffix.EffectiveTLDPlusOne(name)
if err != nil {
continue
}
mu.Lock()
if _, present := names[eTLD]; !present {
names[eTLD] = struct{}{}
}
mu.Unlock()
}
})
numNames := len(names)
t.names = make(chan string, numNames)
t.results = make(chan *result, numNames)
for name := range names {
t.names <- name
}
}
if *nameFile != "" {
data, err := ioutil.ReadFile(*nameFile)
if err != nil {
fmt.Printf("Failed to read name file: %s\n", err)
os.Exit(1)
}
names := strings.Split(string(data), "\n")
numNames := len(names)
t.names = make(chan string, numNames)
t.results = make(chan *result, numNames)
for _, name := range names {
t.names <- name
}
}
close(t.names)
t.run()
t.dump()
}
