func CheckPublicSuffix(zones map[string]*Zone) { color.Fprintf(os.Stderr, "@{.}Checking against the Public Suffix List for %d zones...\n", len(zones)) mapZones(zones, func(z *Zone) { host, err := idna.ToASCII(pfx + z.Domain) if err != nil { LogWarning(err) return } s, _ := publicsuffix.PublicSuffix(host) s = Normalize(s) switch { // ZoneDB and PSL agree case s == z.Domain: return // PSL wildcard case strings.HasPrefix(s, pfx) && len(z.Subdomains) != 0: return // ZoneDB and PSL disagree default: color.Fprintf(os.Stderr, "@{y}Public Suffix List: @{y!}%s@{y} for @{y!}%s\n", s, z.Domain) } }) }
func WriteMetadata(zones map[string]*Zone) error { var wrote, deleted int for _, z := range zones { z.Normalize() path := filepath.Join(BaseDir, "metadata", z.ASCII()+".json") if !z.HasMetadata() { err := os.Remove(path) if err == nil { deleted++ } continue } f, err := os.Create(path) if err != nil { return err } b, err := json.MarshalIndent(&z, "", "\t") if err != nil { return err } f.Write(b) f.Write([]byte("\n")) f.Close() wrote++ } color.Fprintf(os.Stderr, "@{.}Wrote %d metadata files, deleted %d\n", wrote, deleted) return nil }
func query(name, qtype string) { start := time.Now() qname, err := idna.ToASCII(name) if err != nil { color.Fprintf(os.Stderr, "Invalid IDN domain name: %s\n", name) os.Exit(1) } rrs, err := resolver.ResolveErr(qname, qtype) color.Printf("\n") if len(rrs) > 0 { color.Printf("@{g};; RESULTS:\n") } for _, rr := range rrs { color.Printf("@{g}%s\n", rr.String()) } if err != nil { color.Printf("@{r};; %s\t%s\t%s\n", err, name, qtype) } else if rrs == nil { color.Printf("@{y};; NIL\t%s\t%s\n", name, qtype) } else if len(rrs) > 0 { color.Printf("@{g};; TRUE\t%s\t%s\n", name, qtype) } else { color.Printf("@{r};; FALSE\t%s\t%s\n", name, qtype) } color.Printf("@{.w};; Elapsed: %s\n", time.Since(start).String()) }
func logif(err error) bool { if err != nil { color.Fprintf(os.Stderr, "@{r}%s\n", err) return true } return false }
func main() { flag.Usage = func() { color.Fprintf(os.Stderr, "Usage: %s [arguments] <name> [type]\n\nAvailable arguments:\n", os.Args[0]) flag.PrintDefaults() os.Exit(1) } flag.Parse() qtype := "" args := flag.Args() if len(args) == 0 { flag.Usage() } else if _, isType := dns.StringToType[args[len(args)-1]]; len(args) > 1 && isType { qtype, args = args[len(args)-1], args[:len(args)-1] } if verbose { dnsr.DebugLogger = os.Stderr } var wg sync.WaitGroup start := time.Now() for _, name := range args { wg.Add(1) go func(name string, qtype string) { query(name, qtype) wg.Done() }(name, qtype) } wg.Wait() if len(args) > 1 { color.Printf("\n@{.w};; Total elapsed: %s\n", time.Since(start).String()) } }
func FetchRubyWhoisServers(zones map[string]*Zone, addNew bool) error { res, err := Fetch(rubyWhoisURL) if err != nil { return err } defer res.Body.Close() records := make(map[string]struct { Host string `json:"host"` Adapter string `json:"adapter"` URL string `json:"url"` }) d := json.NewDecoder(res.Body) err = d.Decode(&records) if err != nil { return err } var servers, urls int for d, rec := range records { // Skip empty records if rec.Host == "" && rec.Adapter == "none" && rec.URL == "" { continue } d = Normalize(d) z := zones[d] if z == nil { if !addNew { continue } color.Fprintf(os.Stderr, "@{g}New zone @{g!}%s@{g}\n", d) z = &Zone{Domain: d} zones[d] = z } if rec.Host != "" && rec.Host != z.WhoisServer { err := verifyWhois(rec.Host) if err == nil { z.WhoisServer = Normalize(rec.Host) servers++ } } if rec.URL != "" && rec.URL != z.WhoisURL { z.WhoisURL = rec.URL urls++ } } color.Fprintf(os.Stderr, "@{.}Set %d whois servers, %d URLs from Ruby Whois\n", servers, urls) return nil }
// ReadZonesFile reads and parses the zones.txt file. func ReadZonesFile() (zones map[string]*Zone, errs []error) { path := filepath.Join(BaseDir, "zones.txt") f, err := os.Open(path) if err != nil { LogFatal(err) } defer f.Close() zones = make(map[string]*Zone) var line int s := bufio.NewScanner(f) for s.Scan() { line++ d := s.Text() n := Normalize(d) if n != d { err := fmt.Errorf("invalid domain name %s at %s:%d", d, path, line) errs = append(errs, err) LogError(err) continue } if len(d) == 0 { err := fmt.Errorf("blank line at %s:%d", path, line) errs = append(errs, err) LogWarning(err) continue } if _, ok := zones[d]; ok { err := fmt.Errorf("duplicate zone %s at %s:%d", d, path, line) errs = append(errs, err) LogWarning(err) continue } z := &Zone{Domain: d} zones[z.Domain] = z pd := z.ParentDomain() if pd != "" { p, ok := zones[pd] if !ok { err := fmt.Errorf("missing parent %s for subdomain %s at %s:%d", pd, d, path, line) errs = append(errs, err) LogWarning(err) continue } p.Subdomains = append(p.Subdomains, d) } } if err := s.Err(); err != nil { errs = append(errs, err) LogError(err) } for _, z := range zones { sort.Strings(z.Subdomains) } color.Fprintf(os.Stderr, "@{.}Read %d zones\n", len(zones)) return }
// ReadMetadata reads JSON files for each zone in the metadata directory. func ReadMetadata(zones map[string]*Zone) (errs []error) { dir := filepath.Join(BaseDir, "metadata") paths, _ := filepath.Glob(filepath.Join(dir, "*.json")) var read int for _, path := range paths { // Ensure filename equals ASCII/punycode domain name base := filepath.Base(path) ext := filepath.Ext(base) di := strings.TrimSuffix(base, ext) d, err := idna.ToUnicode(di) if err != nil { errs = append(errs, err) LogError(err) continue } // Ensure the domain exists in zones.txt z, ok := zones[d] if !ok { err = fmt.Errorf("domain not found in zones.txt: %s", base) errs = append(errs, err) LogWarning(err) continue } // Parse the JSON metadata f, err := os.Open(path) if err != nil { err = fmt.Errorf("cannot load %s: %s", base, err) errs = append(errs, err) LogError(err) continue } dec := json.NewDecoder(f) err = dec.Decode(z) f.Close() if err != nil && err != io.EOF { err = fmt.Errorf("unable to parse %s: %s", base, err) errs = append(errs, err) LogError(err) continue } read++ // Ensure domain name matches if z.Domain != d { err = fmt.Errorf("domain %s doesn’t match filename (expected %s): %s", z.Domain, z.ASCII(), base) errs = append(errs, err) LogError(err) continue } } color.Fprintf(os.Stderr, "@{.}Read %d metadata files\n", read) return }
func FetchRootZone(zones map[string]*Zone, addNew bool) error { res, err := Fetch(rootZoneURL) if err != nil { return err } defer res.Body.Close() color.Fprintf(os.Stderr, "@{.}Parsing %s\n", rootZoneURL) for token := range dns.ParseZone(res.Body, "", "") { if token.Error != nil { continue } h := token.RR.Header() if h.Rrtype != dns.TypeNS { continue } d := Normalize(h.Name) if d == "" { continue } // Identify the zone z := zones[d] if z == nil { if !addNew { continue } color.Fprintf(os.Stderr, "@{g}New domain: %s\n", d) z = &Zone{Domain: d} zones[d] = z } // Extract name server if ns, ok := token.RR.(*dns.NS); ok { if verifyNS(ns.Ns) == nil { z.NameServers = append(z.NameServers, Normalize(ns.Ns)) } } } return nil }
func VerifyWhois(zones map[string]*Zone) { color.Fprintf(os.Stderr, "@{.}Verifying whois servers for %d zones...\n", len(zones)) mapZones(zones, func(z *Zone) { if z.WhoisServer != "" { if err := verifyWhois(z.WhoisServer); err != nil { LogWarning(fmt.Errorf("can’t verify whois server %s: %s", z.WhoisServer, err)) z.WhoisServer = "" } } }) }
func FetchNameServers(zones map[string]*Zone) error { color.Fprintf(os.Stderr, "@{.}Fetching name servers for %d zones...\n", len(zones)) var found int32 mapZones(zones, func(z *Zone) { name := z.ASCII() rrs := resolver.Resolve(name, "NS") for _, rr := range rrs { if rr.Type != "NS" || Normalize(rr.Name) != z.Domain { continue } ns := Normalize(rr.Value) if verifyNS(ns) == nil { z.NameServers = append(z.NameServers, ns) atomic.AddInt32(&found, 1) } } }) color.Fprintf(os.Stderr, "@{.}Found %d name servers\n", found) return nil }
func QueryWhoisServers(zones map[string]*Zone) error { color.Fprintf(os.Stderr, "@{.}Querying whois-servers.net for %d zones...\n", len(zones)) var found int32 mapZones(zones, func(z *Zone) { name := z.ASCII() + ".whois-servers.net." rrs := resolver.Resolve(name, "CNAME") for _, rr := range rrs { // whois-servers.net occasionally returns whois.ripe.net (unusable) if rr.Type != "CNAME" || Normalize(rr.Name) != z.Domain || rr.Value == "whois.ripe.net." { continue } if verifyWhois(rr.Value) != nil { continue } z.WhoisServer = Normalize(rr.Value) atomic.AddInt32(&found, 1) return } }) color.Fprintf(os.Stderr, "@{.}Found %d whois servers\n", found) return nil }
func VerifyNameServers(zones map[string]*Zone) { color.Fprintf(os.Stderr, "@{.}Verifying name servers for %d zones...\n", len(zones)) mapZones(zones, func(z *Zone) { var nameServers []string for _, ns := range z.NameServers { if err := verifyNS(ns); err != nil { LogWarning(fmt.Errorf("can’t verify name server %s: %s", ns, err)) } else { nameServers = append(nameServers, ns) } } z.NameServers = nameServers }) }
func WriteZonesFile(zones map[string]*Zone) error { domains := SortedDomains(zones) path := filepath.Join(BaseDir, "zones.txt") f, err := os.Create(path) if err != nil { return err } defer f.Close() _, err = f.WriteString(strings.Join(domains, "\n")) if err != nil { return err } color.Fprintf(os.Stderr, "@{.}Wrote %d zones\n", len(domains)) return nil }
func QueryIANA(zones map[string]*Zone) error { tlds := TLDs(zones) color.Fprintf(os.Stderr, "@{.}Querying whois.iana.org for %d TLDs...\n", len(tlds)) limiter := make(chan struct{}, Concurrency) var wg sync.WaitGroup for _, z := range tlds { limiter <- struct{}{} wg.Add(1) go func(z *Zone) { defer func() { <-limiter wg.Done() }() err := tldWhois(z) if err != nil { LogWarning(err) } }(z) } wg.Wait() return nil }
func handleQueryCommand(client *rpc.Client, args []string, useColors bool, quiet bool) { query := strings.Join(args, " ") reply := []QueryMatch{} err := client.Call("RecentDirServer.Query", query, &reply) if err != nil { fmt.Printf("Failed to query the rd daemon: %v\n", err) os.Exit(1) } if len(reply) == 1 { fmt.Println(reply[0].Dir.Path) } else if len(reply) > 0 { maxMatches := 5 if len(reply) < maxMatches { maxMatches = len(reply) } topMatches := reply[0:maxMatches] for _, match := range topMatches { var highlightedMatch string if useColors { highlightedMatch = highlightMatches(match.Dir.Path, match.MatchOffsets) } else { highlightedMatch = match.Dir.Path } fmt.Printf(" %d: %s\n", match.Id, highlightedMatch) } if len(reply) > maxMatches { fmt.Printf(" ... %d other matches not shown", len(reply)-maxMatches) } } else if !quiet { color.Fprintf(os.Stderr, "No matches for @{r!}%s@{|}.\n", query) } }
func GenerateGo(zones map[string]*Zone) error { tlds := TLDs(zones) domains := SortedDomains(zones) offsets := make(map[string]int, len(domains)) tagSet := NewSet() for i, d := range domains { offsets[d] = i z := zones[d] tagSet.Add(z.Tags...) } tags := tagSet.Values() sort.Strings(tags) tagValues := make(map[string]uint64) for i, t := range tags { tagValues[t] = 1 << uint64(i) } var nameServers []string var codePoints []rune for _, d := range domains { z := zones[d] z.Normalize() // just in case z.POffset = offsets[z.ParentDomain()] if len(z.Subdomains) > 0 { z.SOffset = offsets[z.Subdomains[0]] z.SEnd = z.SOffset + len(z.Subdomains) } z.CPOffset, z.CPEnd = IndexOrAppendRunes(&codePoints, z.CodePoints.Runes()) z.TagBits = tagBits(tagValues, z.Tags) } tagType := "uint32" if len(tags) > 32 { tagType = "uint64" } data := struct { Zones map[string]*Zone TLDs map[string]*Zone Domains []string Offsets map[string]int NameServers []string CodePoints []rune TagType string Tags []string TagValues map[string]uint64 }{ zones, tlds, domains, offsets, nameServers, codePoints, tagType, tags, tagValues, } buf := new(bytes.Buffer) err := goTemplate.Execute(buf, &data) if err != nil { return err } formatted, err := format.Source(buf.Bytes()) if err != nil { return err } fn := filepath.Join(BaseDir, "zones.go") color.Fprintf(os.Stderr, "@{.}Generating Go source code: %s\n", fn) f, err := os.Create(fn) if err != nil { return err } defer f.Close() _, err = f.Write(formatted) if err != nil { return err } return nil }
func main() { var uri, addr, user, pass, proxyAddr, crtPath, caPath, keyPath string var useTLS, batch, verbose bool flag.StringVar(&uri, "url", "", "EPP server URL, e.g. epp://user:[email protected]:700") flag.StringVar(&addr, "addr", "", "EPP server address (HOST:PORT)") flag.StringVar(&user, "u", "", "EPP user name") flag.StringVar(&pass, "p", "", "EPP password") flag.BoolVar(&useTLS, "tls", true, "use TLS") flag.StringVar(&proxyAddr, "proxy", "", "SOCKS5 proxy address (HOST:PORT)") flag.StringVar(&crtPath, "cert", "", "path to SSL certificate") flag.StringVar(&keyPath, "key", "", "path to SSL private key") flag.StringVar(&caPath, "ca", "", "path to SSL certificate authority") flag.BoolVar(&batch, "batch", false, "check all domains in a single EPP command") flag.BoolVar(&verbose, "v", false, "enable verbose debug logging") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <query>\n\nAvailable arguments:\n", os.Args[0]) flag.PrintDefaults() os.Exit(1) } flag.Parse() if len(flag.Args()) == 0 { flag.Usage() } if verbose { epp.DebugLogger = os.Stderr } domains := make([]string, len(flag.Args())) for i, arg := range flag.Args() { domains[i] = arg // FIXME: convert unicode to Punycode? } // Parse URL if uri != "" { addr, user, pass = parseURL(uri) } host, _, err := net.SplitHostPort(addr) if err != nil { host = addr } // Set up TLS cfg := &tls.Config{ InsecureSkipVerify: true, ServerName: host, } // Load certificates if caPath != "" { color.Fprintf(os.Stderr, "Loading CA certificate from %s\n", caPath) ca, err := ioutil.ReadFile(caPath) fatalif(err) cfg.RootCAs = x509.NewCertPool() cfg.RootCAs.AppendCertsFromPEM(ca) } if crtPath != "" && keyPath != "" { color.Fprintf(os.Stderr, "Loading certificate %s and key %s\n", crtPath, keyPath) crt, err := tls.LoadX509KeyPair(crtPath, keyPath) fatalif(err) cfg.Certificates = append(cfg.Certificates, crt) // cfg.BuildNameToCertificate() useTLS = true } // Use TLS? if !useTLS { cfg = nil } // Dial start := time.Now() var conn net.Conn if proxyAddr != "" { color.Fprintf(os.Stderr, "Connecting to %s via proxy %s\n", addr, proxyAddr) dialer, err := proxy.SOCKS5("tcp", proxyAddr, nil, &net.Dialer{}) fatalif(err) conn, err = dialer.Dial("tcp", addr) } else { color.Fprintf(os.Stderr, "Connecting to %s\n", addr) conn, err = net.Dial("tcp", addr) } fatalif(err) // TLS if useTLS { color.Fprintf(os.Stderr, "Establishing TLS connection\n") tc := tls.Client(conn, cfg) err = tc.Handshake() fatalif(err) conn = tc } // EPP color.Fprintf(os.Stderr, "Performing EPP handshake\n") c, err := epp.NewConn(conn) fatalif(err) color.Fprintf(os.Stderr, "Logging in as %s...\n", user) err = c.Login(user, pass, "") fatalif(err) // Check start = time.Now() if batch { dc, err := c.CheckDomain(domains...) logif(err) printDCR(dc) } else { for _, domain := range domains { dc, err := c.CheckDomain(domain) logif(err) printDCR(dc) } } qdur := time.Since(start) color.Fprintf(os.Stderr, "@{.}Query: %s Avg: %s\n", qdur, qdur/time.Duration(len(domains))) }
func fatalif(err error) { if logif(err) { color.Fprintf(os.Stderr, "@{r}EXITING\n") os.Exit(1) } }
func main() { // Default options flag.BoolVar(&build.Verbose, "v", false, "enable verbose logging") flag.StringVar(&build.BaseDir, "dir", "./", "working directory (location of zones.txt and metadata dir)") flag.IntVar(&build.Concurrency, "c", build.Concurrency, "number of concurrent connections") // Filters tlds := flag.Bool("tlds", false, "work on top-level domains only") filterZones := flag.String("zones", "", "select specific working zones (comma-delimited)") filterRegexp := flag.String("x", "", "select working zones on by regular expression") filterTags := flag.String("tags", "", "select working zones by tags (comma-delimited)") // Query operations listZones := flag.Bool("list", false, "list working zones") listTags := flag.Bool("list-tags", false, "list tags in working zones") listLocations := flag.Bool("list-locations", false, "list locations in working zones") // Test operations verifyNS := flag.Bool("verify-ns", false, "verify name servers") verifyWhois := flag.Bool("verify-whois", false, "verify whois servers") checkPS := flag.Bool("ps", false, "check against Public Suffix List") // Mutate operations addTags := flag.String("add-tags", "", "add tags to zones (comma-delimited)") addLocations := flag.String("add-locations", "", "add locations to zones (comma-delimited)") removeTags := flag.String("remove-tags", "", "remove tags from zones (comma-delimited)") removeLocations := flag.String("remove-locations", "", "remove locations from zones (comma-delimited)") updateRoot := flag.Bool("update-root", false, "retrieve updates to the root zone file") updateNS := flag.Bool("update-ns", false, "update name servers") updateRubyWhois := flag.Bool("update-ruby-whois", false, "query Ruby Whois for whois servers") updateWhois := flag.Bool("update-whois", false, "query whois-servers.net for whois servers") updateIANA := flag.Bool("update-iana", false, "query IANA for metadata") updateAll := flag.Bool("update", false, "update all (root zone, whois, IANA data)") // Write operations write := flag.Bool("w", false, "write zones.txt and metadata") generateGo := flag.Bool("generate-go", false, "generate Go source code to specified directory") flag.Usage = func() { color.Fprintf(os.Stderr, "@{!}Usage:@{|} %s [arguments] <command>\n\n", os.Args[0]) color.Fprintf(os.Stderr, "@{!}Available arguments: \n") flag.PrintDefaults() color.Fprintf(os.Stderr, "\n") os.Exit(1) } flag.Parse() startTime := time.Now() defer func() { elapsed := time.Since(startTime) elapsed -= elapsed % 1000000 color.Fprintf(os.Stderr, "@{.}Time elapsed: %s\n", elapsed) }() zones, errs := build.ReadZones() if len(errs) > 0 { build.LogFatal(fmt.Errorf("read failed with %d issue(s)", len(errs))) } workZones := zones if *tlds { workZones = build.TLDs(zones) color.Fprintf(os.Stderr, "@{.}Working on top-level domains\n") } if *filterZones != "" { domains := strings.Split(*filterZones, ",") filtered := make(map[string]*build.Zone, len(domains)) for _, d := range domains { d = build.Normalize(d) if z, ok := workZones[d]; ok { filtered[d] = z } } workZones = filtered } if *filterRegexp != "" { re, err := regexp.Compile(*filterRegexp) if err != nil { build.LogFatal(err) } filtered := make(map[string]*build.Zone, len(workZones)) for d, z := range workZones { if re.MatchString(d) { filtered[d] = z } } workZones = filtered } if *filterTags != "" { tags := strings.Split(*filterTags, ",") filtered := make(map[string]*build.Zone, len(workZones)) for d, z := range workZones { s := build.NewSet(z.Tags...) for _, t := range tags { if _, ok := s[t]; ok { filtered[d] = z break } } } workZones = filtered } color.Fprintf(os.Stderr, "@{.}Working on %d zone(s) out of %d\n", len(workZones), len(zones)) // Add newly found zones? addNew := len(workZones) == len(zones) if *listZones || len(workZones) < len(zones) { domains := build.SortedDomains(workZones) color.Fprintf(os.Stderr, "@{.}Zones: @{c}%s\n", strings.Join(domains, " ")) } if *updateRoot || *updateAll { err := build.FetchRootZone(workZones, addNew) if err != nil { build.LogError(err) } } if *updateRubyWhois || *updateAll { err := build.FetchRubyWhoisServers(workZones, addNew) if err != nil { build.LogError(err) } } // whois-servers.net overrides Ruby Whois if *updateWhois || *updateAll { err := build.QueryWhoisServers(workZones) if err != nil { build.LogError(err) } } // IANA overrides the above if *updateIANA || *updateAll { err := build.QueryIANA(workZones) if err != nil { build.LogError(err) } } if *updateNS || *updateAll { err := build.FetchNameServers(workZones) if err != nil { build.LogError(err) } } if *removeTags != "" { tags := strings.Split(*removeTags, ",") build.RemoveTags(workZones, tags) } if *addTags != "" { tags := strings.Split(*addTags, ",") build.AddTags(workZones, tags) } if *listTags { tags := build.NewSet() for _, z := range workZones { tags.Add(z.Tags...) } color.Fprintf(os.Stderr, "@{.}Tags: @{c}%s\n", strings.Join(tags.Values(), " ")) } if *removeLocations != "" { locations := strings.Split(*removeLocations, ",") build.RemoveLocations(workZones, locations) } if *addLocations != "" { locations := strings.Split(*addLocations, ",") build.AddLocations(workZones, locations) } if *listLocations { locations := build.NewSet() for _, z := range workZones { locations.Add(z.Locations...) } color.Fprintf(os.Stderr, "@{.}Locations: @{c}%s\n", strings.Join(locations.Values(), " ")) } if *verifyNS { build.VerifyNameServers(workZones) } if *verifyWhois { build.VerifyWhois(workZones) } if *checkPS { build.CheckPublicSuffix(workZones) } // Fold newly added zones back in for d, z := range workZones { zones[d] = z } if *write { err := build.WriteZones(zones) if err != nil { build.LogFatal(err) } } if *generateGo { err := build.GenerateGo(zones) if err != nil { build.LogFatal(err) } } }
func LogFatal(err error) { color.Fprintf(os.Stderr, "@{r!}Fatal:@{r} %s\n", err) os.Exit(1) }
func LogWarning(err error) { color.Fprintf(os.Stderr, "@{y!}Warning:@{y} %s\n", err) }
func LogError(err error) { color.Fprintf(os.Stderr, "@{r!}Error:@{r} %s\n", err) }