Beispiel #1
0
// domainsForRateLimiting transforms a list of FQDNs into a list of eTLD+1's
// for the purpose of rate limiting. It also de-duplicates the output
// domains.
func domainsForRateLimiting(names []string) ([]string, error) {
	domainsMap := make(map[string]struct{}, len(names))
	var domains []string
	for _, name := range names {
		eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(name)
		if err != nil {
			// The only possible errors are:
			// (1) publicsuffix.PublicSuffix is giving garbage
			//     values
			// (2) the public suffix is the domain itself
			//
			// Assume (2).
			eTLDPlusOne = name
		}
		if _, ok := domainsMap[eTLDPlusOne]; !ok {
			domainsMap[eTLDPlusOne] = struct{}{}
			domains = append(domains, eTLDPlusOne)
		}
	}
	return domains, nil
}
func main() {
	cacheFile := flag.String("ctFile", "", "")
	nameFile := flag.String("nameFile", "", "")
	normalResolver := flag.String("normalResolver", "127.0.0.1:53", "")
	torResolver := flag.String("torResolver", "127.0.0.1:9053", "")
	workers := flag.Int("workers", 1, "")
	flag.Parse()

	if *cacheFile == "" && *nameFile == "" {
		fmt.Println("Either --ctFile or --nameFile is required")
		os.Exit(1)
	}

	t := tester{
		workers:        *workers,
		client:         &dns.Client{DialTimeout: dnsTimeout, ReadTimeout: dnsTimeout, Net: "tcp"},
		normalResolver: *normalResolver,
		torResolver:    *torResolver,
	}

	if *cacheFile != "" {
		ctEntries, err := ctCommon.LoadCacheFile(*cacheFile)
		if err != nil {
			fmt.Printf("Failed to load CT cache file: %s\n", err)
			os.Exit(1)
		}

		names := make(map[string]struct{})
		mu := new(sync.Mutex)
		ctEntries.Map(func(ent *ct.EntryAndPosition, err error) {
			if err != nil || ent.Entry.Type != ct.X509Entry {
				return
			}
			cert, _, err := ctCommon.ParseAndFilter(ent.Entry.X509Cert, nil)
			if err != nil {
				return
			}
			for _, name := range cert.DNSNames {
				eTLD, err := publicsuffix.EffectiveTLDPlusOne(name)
				if err != nil {
					continue
				}
				mu.Lock()
				if _, present := names[eTLD]; !present {
					names[eTLD] = struct{}{}
				}
				mu.Unlock()
			}
		})

		numNames := len(names)
		t.names = make(chan string, numNames)
		t.results = make(chan *result, numNames)
		for name := range names {
			t.names <- name
		}
	}
	if *nameFile != "" {
		data, err := ioutil.ReadFile(*nameFile)
		if err != nil {
			fmt.Printf("Failed to read name file: %s\n", err)
			os.Exit(1)
		}
		names := strings.Split(string(data), "\n")
		numNames := len(names)
		t.names = make(chan string, numNames)
		t.results = make(chan *result, numNames)
		for _, name := range names {
			t.names <- name
		}
	}
	close(t.names)

	t.run()
	t.dump()
}