func writeRegex(tlds []string) error {
allTldsSet := make(map[string]struct{})
add := func(tld string) {
if _, e := allTldsSet[tld]; e {
log.Fatalf("Duplicate TLD: %s", tld)
}
allTldsSet[tld] = struct{}{}
}
for _, tldlist := range [...][]string{tlds, xurls.PseudoTLDs} {
for _, tld := range tldlist {
add(tld)
asciiTld, err := idna.ToASCII(tld)
if err != nil {
return err
}
if asciiTld != tld {
add(asciiTld)
}
}
}
var allTlds []string
for tld := range allTldsSet {
allTlds = append(allTlds, tld)
}
sort.Strings(allTlds)
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
return regexTmpl.Execute(f, map[string]string{
"gtld ": `(?i)(` + strings.Join(allTlds, `|`) + `)(?-i)`,
"otherScheme": `(?i)(` + strings.Join(xurls.SchemesNoAuthority, `|`) + `)(?-i):`,
})
}
func Fuzz(data []byte) int {
for _, v := range data {
if v <= 0x20 || v >= 0x80 {
return 0
}
}
dec, err := idna.ToUnicode(string(data))
if err != nil {
return 0
}
enc, err := idna.ToASCII(dec)
if err != nil {
fmt.Printf("data: %q\n", data)
fmt.Printf("dec : %q\n", dec)
panic(err)
}
dec1, err := idna.ToUnicode(enc)
if err != nil {
fmt.Printf("data: %q\n", data)
fmt.Printf("dec : %q\n", dec)
fmt.Printf("enc : %q\n", enc)
panic(err)
}
if dec != dec1 {
fmt.Printf("data: %q\n", data)
fmt.Printf("dec : %q\n", dec)
fmt.Printf("enc : %q\n", enc)
fmt.Printf("dec1: %q\n", dec1)
panic("not equal")
}
return 1
}
func query(name, qtype string) {
start := time.Now()
qname, err := idna.ToASCII(name)
if err != nil {
color.Fprintf(os.Stderr, "Invalid IDN domain name: %s\n", name)
os.Exit(1)
}
rrs, err := resolver.ResolveErr(qname, qtype)
color.Printf("\n")
if len(rrs) > 0 {
color.Printf("@{g};; RESULTS:\n")
}
for _, rr := range rrs {
color.Printf("@{g}%s\n", rr.String())
}
if err != nil {
color.Printf("@{r};; %s\t%s\t%s\n", err, name, qtype)
} else if rrs == nil {
color.Printf("@{y};; NIL\t%s\t%s\n", name, qtype)
} else if len(rrs) > 0 {
color.Printf("@{g};; TRUE\t%s\t%s\n", name, qtype)
} else {
color.Printf("@{r};; FALSE\t%s\t%s\n", name, qtype)
}
color.Printf("@{.w};; Elapsed: %s\n", time.Since(start).String())
}
// StandardizeURL standardizes the url by making sure it has a schema and converting IDNA domains into ASCII.
func StandardizeURL(url string) string {
link := url
var schema, domain, path string
// Try to get the schema
slice := strings.SplitN(url, "://", 2)
if len(slice) == 2 && len(slice[0]) < 10 { // schema exists
schema = slice[0] + "://"
link = slice[1]
} else {
schema = "http://"
}
// Get the domain
slice = strings.SplitN(link, "/", 2)
if len(slice) == 2 {
domain = slice[0]
path = "/" + slice[1]
} else {
domain = slice[0]
path = "/"
}
domain, _ = idna.ToASCII(domain)
link = schema + domain + path
return link
}
func CheckPublicSuffix(zones map[string]*Zone) {
color.Fprintf(os.Stderr, "@{.}Checking against the Public Suffix List for %d zones...\n", len(zones))
mapZones(zones, func(z *Zone) {
host, err := idna.ToASCII(pfx + z.Domain)
if err != nil {
LogWarning(err)
return
}
s, _ := publicsuffix.PublicSuffix(host)
s = Normalize(s)
switch {
// ZoneDB and PSL agree
case s == z.Domain:
return
// PSL wildcard
case strings.HasPrefix(s, pfx) && len(z.Subdomains) != 0:
return
// ZoneDB and PSL disagree
default:
color.Fprintf(os.Stderr, "@{y}Public Suffix List: @{y!}%s@{y} for @{y!}%s\n", s, z.Domain)
}
})
}
func verifyWhois(host string) error {
host, err := idna.ToASCII(Normalize(host))
if err != nil {
return err
}
err = CanDial("tcp", host+":43")
return err
}
func verifyNS(host string) error {
host, err := idna.ToASCII(Normalize(host))
if err != nil {
return err
}
err = CanDial("udp", host+":53")
return err
}
func IsValid(email string) bool {
at := strings.LastIndex(email, "@")
if at == -1 {
return false
}
local, _ := idna.ToASCII(email[0:at])
domain, _ := idna.ToASCII(email[at+1:])
localLen := len(local)
if localLen < 1 || localLen > 64 {
return false
}
domainLen := len(domain)
if domainLen < 1 || domainLen > 255 {
return false
}
if local[0] == '.' || local[localLen-1] == '.' {
return false
}
if domain[0] == '.' || domain[domainLen-1] == '.' {
return false
}
if dotsCheck.MatchString(local) || dotsCheck.MatchString(domain) {
return false
}
if !domainCheck.MatchString(domain) {
return false
}
local = strings.Replace(local, "\\\\", "", -1)
if !localCheck.MatchString(local) && !quotedCheck.MatchString(local) {
return false
}
return true
}
// NewRuleUnicode is like NewRule, but expects the content to be encoded in Unicode (U-labels).
func NewRuleUnicode(content string) (*Rule, error) {
var err error
content, err = idna.ToASCII(content)
if err != nil {
return nil, err
}
return NewRule(content)
}
// Normalizes the hostname given. If the hostname is not valid, returns "" and
// an error.
func NormalizeHostname(name string) (string, error) {
name = strings.TrimSuffix(strings.ToLower(name), ".")
name, err := idna.ToASCII(name)
if err != nil {
return "", fmt.Errorf("IDN error: %#v: %v", name, err)
}
if !reHostname.MatchString(name) {
return "", fmt.Errorf("invalid hostname: %#v", name)
}
return name, nil
}
func validateHost(host string) (string, error) {
lower := strings.ToLower(host)
if domainRegexp.MatchString(lower) || lower == "localhost" ||
net.ParseIP(lower) != nil {
return lower, nil
}
// The URL will be used by net/http, where IDNA is not supported.
if punycode, err := idna.ToASCII(host); err != nil {
return "", err
} else if domainRegexp.MatchString(punycode) {
return punycode, nil
}
return "", errors.New("not valid domain name or IP address")
}
func normalizeNames(names []string) error {
for i := range names {
n := strings.TrimSuffix(strings.ToLower(names[i]), ".")
n, err := idna.ToASCII(n)
if err != nil {
return fmt.Errorf("IDN error: %v", err)
}
if !validHostname(n) {
return fmt.Errorf("invalid hostname: %q", n)
}
names[i] = n
}
return nil
}
// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
// It takes an URL string as input, as well as the normalization flags.
func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
if parsed, e := url.Parse(u); e != nil {
return "", e
} else {
options := make([]precis.Option, 1, 3)
options[0] = precis.IgnoreCase
if f&FlagLowercaseHost == FlagLowercaseHost {
options = append(options, precis.FoldCase())
}
options = append(options, precis.Norm(norm.NFC))
profile := precis.NewFreeform(options...)
if parsed.Host, e = idna.ToASCII(profile.NewTransformer().String(parsed.Host)); e != nil {
return "", e
}
return NormalizeURL(parsed, f), nil
}
panic("Unreachable code.")
}
func encodeIDNAHost(u *url.URL) {
h := u.Host
p := ""
var err error
if strings.Index(h, ":") > 0 {
h, p, err = net.SplitHostPort(h)
if err != nil {
return
}
}
if h, err = idna.ToASCII(h); err == nil {
if len(p) > 0 {
u.Host = fmt.Sprintf("%s:%s", h, p)
} else {
u.Host = h
}
}
}
// Handler for DNS queries
func query(w http.ResponseWriter, r *http.Request) {
server := r.URL.Query().Get(":server")
domain := dns.Fqdn(r.URL.Query().Get(":domain"))
querytype := r.URL.Query().Get(":querytype")
if domain, err := idna.ToASCII(domain); err == nil { // Valid domain name (ASCII or IDN)
if _, isDomain := dns.IsDomainName(domain); isDomain { // Well-formed domain name
if querytype, ok := dns.StringToType[strings.ToUpper(querytype)]; ok { // Valid DNS query type
resolve(w, r, server, domain, querytype)
} else {
error(w, 400, 404, "Invalid DNS query type")
}
} else {
error(w, 400, 402, "Input string is not a well-formed domain name")
}
} else {
error(w, 400, 401, "Input string could not be parsed")
}
}
// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
// It takes an URL string as input, as well as the normalization flags.
func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
parsed, err := url.Parse(u)
if err != nil {
return "", err
}
if f&FlagLowercaseHost == FlagLowercaseHost {
parsed.Host = strings.ToLower(parsed.Host)
}
// The idna package doesn't fully conform to RFC 5895
// (https://tools.ietf.org/html/rfc5895), so we do it here.
// Taken from Go 1.8 cycle source, courtesy of bradfitz.
// TODO: Remove when (if?) idna package conforms to RFC 5895.
parsed.Host = width.Fold.String(parsed.Host)
parsed.Host = norm.NFC.String(parsed.Host)
if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
return "", err
}
return NormalizeURL(parsed, f), nil
}
func checkHost(host string) error {
if host == "" {
return &url.Error{"host", host, errors.New("empty host")}
}
host = strings.ToLower(host)
if domainRegexp.MatchString(host) || host == "localhost" {
return nil
}
if punycode, err := idna.ToASCII(host); err != nil {
return err
} else if domainRegexp.MatchString(punycode) {
return nil
}
// IPv4 and IPv6.
if ipv4Regexp.MatchString(host) || ipv6Regexp.MatchString(host) {
return nil
}
return &url.Error{"host", host, errors.New("invalid host")}
}
// PunycodeHostPort returns the IDNA Punycode version
// of the provided "host" or "host:port" string.
func PunycodeHostPort(v string) (string, error) {
if isASCII(v) {
return v, nil
}
host, port, err := net.SplitHostPort(v)
if err != nil {
// The input 'v' argument was just a "host" argument,
// without a port. This error should not be returned
// to the caller.
host = v
port = ""
}
host, err = idna.ToASCII(host)
if err != nil {
// Non-UTF-8? Not representable in Punycode, in any
// case.
return "", err
}
if port == "" {
return host, nil
}
return net.JoinHostPort(host, port), nil
}
// Search for URLs in a string
func findURLs(message string) (urls []*url.URL) {
// Source of the regular expression:
// http://daringfireball.net/2010/07/improved_regex_for_matching_urls
re := regexp.MustCompile("(?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\".,<>?«»“”‘’])")
urlCandidates := re.FindAllString(message, maxUrlsCount)
for _, candidate := range urlCandidates {
url, err := url.Parse(candidate)
if err != nil {
break
}
// Scheme is required to query a URL
if url.Scheme == "" {
url.Scheme = "http"
}
// Conversion to ASCII is needed for Unicode hostnames
asciiHost, err := idna.ToASCII(url.Host)
if err == nil {
url.Host = asciiHost
}
urls = append(urls, url)
}
return
}
// Send email
func (m *MailData) Send() error {
// trim space
m.To_email = strings.TrimSpace(m.To_email)
// punycode convert
splitEmail := strings.Split(m.To_email, "@")
if len(splitEmail) != 2 {
return errors.New(fmt.Sprintf("Bad email"))
}
domain, err := idna.ToASCII(splitEmail[1])
if err != nil {
return errors.New(fmt.Sprintf("Domain name failed: %v", err))
}
m.To_email = strings.Split(m.To_email, "@")[0] + "@" + domain
if m.Iface == "" {
// default interface
m.netConn = net.Dialer{}
} else {
if m.Iface[0:8] == "socks://" {
m.Iface = m.Iface[8:]
var err error
m.socksConn, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
if err != nil {
return err
}
} else {
connectAddr := net.ParseIP(m.Iface)
tcpAddr := &net.TCPAddr{
IP: connectAddr,
}
m.netConn = net.Dialer{LocalAddr: tcpAddr}
}
}
start := time.Now()
//record, err := net.LookupMX(c.domain)
record, err := models.DomainGetMX(domain)
if err != nil {
return err
}
lookupTime := time.Since(start)
start = time.Now()
var serverMx string
for i := range record {
smx := net.JoinHostPort(record[i].Host, "25")
if m.socksConn != nil {
m.conn, err = m.socksConn.Dial("tcp", smx)
} else {
m.conn, err = m.netConn.Dial("tcp", smx)
}
if err == nil {
serverMx = record[i].Host
connTime := time.Since(start)
fmt.Printf("Connect time to %s %s. Lookup time %s.\n\r", domain, connTime, lookupTime)
break
}
}
if err != nil {
return err
}
defer m.conn.Close()
// 5 minute by RFC
m.conn.SetDeadline(time.Now().Add(5 * time.Minute))
c, err := smtp.NewClient(m.conn, serverMx)
if err != nil {
return errors.New(fmt.Sprintf("%v (NewClient)", err))
}
if err := c.Hello(m.Host); err != nil {
return errors.New(fmt.Sprintf("%v (Hello)", err))
}
// Set the sender and recipient first
if err := c.Mail(m.From_email); err != nil {
return errors.New(fmt.Sprintf("%v (Mail)", err))
}
if err := c.Rcpt(m.To_email); err != nil {
return errors.New(fmt.Sprintf("%v (Rcpt)", err))
}
w, err := c.Data()
if err != nil {
return errors.New(fmt.Sprintf("%v (Data)", err))
}
_, err = fmt.Fprint(w, m.Data())
if err != nil {
return errors.New(fmt.Sprintf("%v (SendData)", err))
}
err = w.Close()
if err != nil {
return errors.New(fmt.Sprintf("%v (Close)", err))
}
return c.Quit()
}
func main1() error {
flag.Parse()
if nodesBitsTextLength+nodesBitsTextOffset+nodesBitsICANN+nodesBitsChildren > 32 {
return fmt.Errorf("not enough bits to encode the nodes table")
}
if childrenBitsLo+childrenBitsHi+childrenBitsNodeType+childrenBitsWildcard > 32 {
return fmt.Errorf("not enough bits to encode the children table")
}
if *version == "" {
return fmt.Errorf("-version was not specified")
}
var r io.Reader = os.Stdin
if *url != "" {
res, err := http.Get(*url)
if err != nil {
return err
}
if res.StatusCode != http.StatusOK {
return fmt.Errorf("bad GET status for %s: %d", *url, res.Status)
}
r = res.Body
defer res.Body.Close()
}
var root node
icann := false
buf := new(bytes.Buffer)
br := bufio.NewReader(r)
for {
s, err := br.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return err
}
s = strings.TrimSpace(s)
if strings.Contains(s, "BEGIN ICANN DOMAINS") {
icann = true
continue
}
if strings.Contains(s, "END ICANN DOMAINS") {
icann = false
continue
}
if s == "" || strings.HasPrefix(s, "//") {
continue
}
s, err = idna.ToASCII(s)
if err != nil {
return err
}
if !validSuffix.MatchString(s) {
return fmt.Errorf("bad publicsuffix.org list data: %q", s)
}
if *subset {
switch {
case s == "ac.jp" || strings.HasSuffix(s, ".ac.jp"):
case s == "ak.us" || strings.HasSuffix(s, ".ak.us"):
case s == "ao" || strings.HasSuffix(s, ".ao"):
case s == "ar" || strings.HasSuffix(s, ".ar"):
case s == "arpa" || strings.HasSuffix(s, ".arpa"):
case s == "cy" || strings.HasSuffix(s, ".cy"):
case s == "dyndns.org" || strings.HasSuffix(s, ".dyndns.org"):
case s == "jp":
case s == "kobe.jp" || strings.HasSuffix(s, ".kobe.jp"):
case s == "kyoto.jp" || strings.HasSuffix(s, ".kyoto.jp"):
case s == "om" || strings.HasSuffix(s, ".om"):
case s == "uk" || strings.HasSuffix(s, ".uk"):
case s == "uk.com" || strings.HasSuffix(s, ".uk.com"):
case s == "tw" || strings.HasSuffix(s, ".tw"):
case s == "zw" || strings.HasSuffix(s, ".zw"):
case s == "xn--p1ai" || strings.HasSuffix(s, ".xn--p1ai"):
// xn--p1ai is Russian-Cyrillic "рф".
default:
continue
}
}
rules = append(rules, s)
nt, wildcard := nodeTypeNormal, false
switch {
case strings.HasPrefix(s, "*."):
s, nt = s[2:], nodeTypeParentOnly
wildcard = true
case strings.HasPrefix(s, "!"):
s, nt = s[1:], nodeTypeException
}
labels := strings.Split(s, ".")
for n, i := &root, len(labels)-1; i >= 0; i-- {
label := labels[i]
n = n.child(label)
if i == 0 {
if nt != nodeTypeParentOnly && n.nodeType == nodeTypeParentOnly {
n.nodeType = nt
}
n.icann = n.icann && icann
n.wildcard = n.wildcard || wildcard
}
labelsMap[label] = true
}
}
labelsList = make([]string, 0, len(labelsMap))
for label := range labelsMap {
labelsList = append(labelsList, label)
}
sort.Strings(labelsList)
p := printReal
if *test {
p = printTest
}
if err := p(buf, &root); err != nil {
return err
}
b, err := format.Source(buf.Bytes())
if err != nil {
return err
}
_, err = os.Stdout.Write(b)
return err
}
func (m *MailData) SendMail() error {
var smx string
var mx []*net.MX
var conn net.Conn
if m.Iface == "" {
// default interface
m.n = net.Dialer{}
} else {
if m.Iface[0:8] == "socks://" {
m.Iface = m.Iface[8:]
var err error
m.s, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
if err != nil {
return err
}
} else {
connectAddr := net.ParseIP(m.Iface)
tcpAddr := &net.TCPAddr{
IP: connectAddr,
}
m.n = net.Dialer{LocalAddr: tcpAddr}
}
}
//ToDo cache MX servers
// punycode convert
domain, err := idna.ToASCII(strings.Split(m.To, "@")[1])
if err != nil {
return errors.New(fmt.Sprintf("Domain name failed: %v\r\n", err))
}
m.To = strings.Split(m.To, "@")[0] + "@" + domain
mx, err = net.LookupMX(domain)
if err != nil {
return errors.New(fmt.Sprintf("LookupMX failed: %v\r\n", err))
} else {
for i := range mx {
smx := net.JoinHostPort(mx[i].Host, "25")
// Set ip (from MX records) and port mail server
if m.s != nil {
conn, err = m.s.Dial("tcp", smx)
} else {
conn, err = m.n.Dial("tcp", smx)
}
if err == nil {
defer conn.Close()
break
}
}
}
if err != nil {
return err
}
host, _, _ := net.SplitHostPort(smx)
c, err := smtp.NewClient(conn, host)
if err != nil {
return err
}
if err := c.Hello(m.Host); err != nil {
return err
}
// Set the sender and recipient first
if err := c.Mail(m.From); err != nil {
return err
}
if err := c.Rcpt(m.To); err != nil {
return err
}
msg := m.makeMail()
//dkim.New()
w, err := c.Data()
if err != nil {
return err
}
_, err = fmt.Fprintf(w, msg)
if err != nil {
return err
}
err = w.Close()
if err != nil {
return err
}
// Send the QUIT command and close the connection.
return c.Quit()
}
// CreateKeyPair creates a key pair for the given hostname on the fly.
func CreateKeyPair(commonName string) (certFile string, keyFile string, err error) {
mu.Lock()
defer mu.Unlock()
commonName, err = idna.ToASCII(commonName)
if err != nil {
return
}
commonName = strings.ToLower(commonName)
destDir := certDirectory + pathSeparator + commonName + pathSeparator
certFile = destDir + "cert.pem"
keyFile = destDir + "key.pem"
// Attempt to verify certs.
if _, err = tls.LoadX509KeyPair(certFile, keyFile); err == nil {
// Keys already in place
return
}
log.Printf("Creating SSL certificate for %s...", commonName)
notBefore := time.Now().Add(-24 * 30 * time.Hour)
notAfter := notBefore.Add(365 * 24 * time.Hour)
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return
}
template := x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{
Organization: []string{"Hyperfox Fake Certificates, Inc"},
CommonName: commonName,
},
NotBefore: notBefore,
NotAfter: notAfter,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
if ip := net.ParseIP(commonName); ip != nil {
template.IPAddresses = append(template.IPAddresses, ip)
} else {
template.DNSNames = append(template.DNSNames, commonName)
}
rootCA, err := tls.LoadX509KeyPair(rootCACert, rootCAKey)
if err != nil {
return
}
if rootCA.Leaf, err = x509.ParseCertificate(rootCA.Certificate[0]); err != nil {
return
}
var priv *rsa.PrivateKey
if priv, err = rsa.GenerateKey(rand.Reader, rsaBits); err != nil {
return
}
var derBytes []byte
if derBytes, err = x509.CreateCertificate(rand.Reader, &template, rootCA.Leaf, &priv.PublicKey, rootCA.PrivateKey); err != nil {
return
}
if err = os.MkdirAll(destDir, 0755); err != nil {
return
}
certOut, err := os.Create(certFile)
if err != nil {
return
}
pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
certOut.Close()
keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return
}
pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
keyOut.Close()
return
}
func (z *Zone) ASCII() string {
s, _ := idna.ToASCII(z.Domain)
return s
}
// URLParse is an exported function that fills urls/ips maps and return bool
func URLParse(raw []string, urls map[string]bool, domains map[string]bool) bool {
u_parsed := []*url.URL{}
/*
* raw is an array of substrings from the original string that was
* splitted by semicolons. The first one [0] is an array of IPs (one or more,
* divided with pipe), the third one is an array of URLs (zero(!) or more,
* divided by pipe). However, the Split function might return an array of
* one substring if it was unable to find a separator in the original
* string. Therefore, we must firstly ensure that our array consists
* from at least 3 elements.
* [date] [(urls)] [host(s)] [ip(s)]
*/
if len(raw) < 3 {
return false
}
/*
* We assume here that after splitting the original string into substrings,
* we get an array of substrings that explicitly has [0], [1] and [2]
* elements. The previous check exits from the procedure otherwise we would get
* here and catch a panic. Now we check if there is enough a URL-substring length
* to operate on, if not - just return raw[1] which is a domain name of the resource.
*/
if len(raw[1]) < 4 {
goto HaveNonHTTP
}
for _, tmp := range strings.Split(raw[1], ",") {
_url, err := url.Parse(tmp)
if err != nil {
// as "" is a valid URL too (sic!)
// this code does nothing useful
return false //goto host check
}
u_parsed = append(u_parsed, _url)
}
for _, _url := range u_parsed {
/*
It is much better to treat a URL having "/" URI
or too long URI as a domain-only, along with a pure non-http URL
and URI containing symbols like ':' and '*', which are the special
ones in SCE.
*/
rq := _url.RequestURI()
not_ok := ((_url.Scheme != "http") || (rq == "/"))
not_ok = not_ok || (strings.ContainsAny(rq, ":*"))
//TODO: check if our URL's host is already in the non-http database
if not_ok {
goto HaveNonHTTP
}
}
/* If we get here this means all the checks above are ok */
for _, u := range u_parsed {
// some hosts contain port in url, this is not good
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
host = u.Host
}
// some guys in EIAS think that it is totally ok
// to include utf-8 host names in the list without
// encoding them according to the RULES
_t, _ := idna.ToASCII(host)
// those guys are also ok with the domain names
// ending with a dot
_t = strings.TrimSuffix(_t, ".")
_p := u.EscapedPath()
// && (_p[len(_p)-1] == '/')
if len(u.RawQuery) == 0 {
_t = _t + _p + "*"
} else {
_t = _t + u.RequestURI()
}
urls[_t] = true
}
return true
HaveNonHTTP:
/*
We only get here if some of URLs in the array have non-http scheme.
So the domain name should be used instead.
*/
for _, v := range strings.Split(raw[2], ",") {
if len(v) < 1 {
fmt.Println("no domain for", raw)
continue
}
_t, _ := idna.ToASCII(v)
_t = strings.TrimSuffix(_t, ".")
domains[_t] = true
}
return true
/*Never get here*/
return false
}
