Exemple #1
0
func printSuffix() {
	host := *FSuffix
	tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
	fmt.Printf("EffectiveTLDPlusOne: %s\n", tld)
	suffix, _ := publicsuffix.PublicSuffix(host)
	fmt.Printf("PublicSuffix: %s\n", suffix)
}
Exemple #2
0
func (self *Server) Blocked(host string) bool {
	blocked, cached := false, false
	host = HostOnly(host)
	self.mutex.RLock()
	if self.BlockedHosts[host] {
		blocked = true
		cached = true
	}
	self.mutex.RUnlock()

	if !blocked {
		tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
		blocked = self.Cfg.Blocked(tld)
	}

	if !blocked {
		suffix, _ := publicsuffix.PublicSuffix(host)
		blocked = self.Cfg.Blocked(suffix)
	}

	if blocked && !cached {
		self.mutex.Lock()
		self.BlockedHosts[host] = true
		self.mutex.Unlock()
	}
	return blocked
}
Exemple #3
0
// Adds the DNS message data to the supplied MapStr.
func addDnsToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional bool) {
	m["id"] = dns.Id
	m["op_code"] = dnsOpCodeToString(dns.Opcode)

	m["flags"] = common.MapStr{
		"authoritative":       dns.Authoritative,
		"truncated_response":  dns.Truncated,
		"recursion_desired":   dns.RecursionDesired,
		"recursion_available": dns.RecursionAvailable,
		"authentic_data":      dns.AuthenticatedData, // [RFC4035]
		"checking_disabled":   dns.CheckingDisabled,  // [RFC4035]
	}
	m["response_code"] = dnsResponseCodeToString(dns.Rcode)

	if len(dns.Question) > 0 {
		q := dns.Question[0]
		qMapStr := common.MapStr{
			"name":  q.Name,
			"type":  dnsTypeToString(q.Qtype),
			"class": dnsClassToString(q.Qclass),
		}
		m["question"] = qMapStr

		eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(strings.TrimRight(q.Name, "."))
		if err == nil {
			qMapStr["etld_plus_one"] = eTLDPlusOne + "."
		}
	}

	rrOPT := dns.IsEdns0()
	if rrOPT != nil {
		m["opt"] = optToMapStr(rrOPT)
	}

	m["answers_count"] = len(dns.Answer)
	if len(dns.Answer) > 0 {
		m["answers"] = rrsToMapStrs(dns.Answer)
	}

	m["authorities_count"] = len(dns.Ns)
	if authority && len(dns.Ns) > 0 {
		m["authorities"] = rrsToMapStrs(dns.Ns)
	}

	if rrOPT != nil {
		m["additionals_count"] = len(dns.Extra) - 1
	} else {
		m["additionals_count"] = len(dns.Extra)
	}
	if additional && len(dns.Extra) > 0 {
		rrsMapStrs := rrsToMapStrs(dns.Extra)
		// We do not want OPT RR to appear in the 'additional' section,
		// that's why rrsMapStrs could be empty even though len(dns.Extra) > 0
		if len(rrsMapStrs) > 0 {
			m["additionals"] = rrsMapStrs
		}
	}

}
Exemple #4
0
func main() {
	for _, host := range os.Args[1:] {
		fmt.Printf("Host: %s\n", host)
		tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
		fmt.Printf("\tEffectiveTLDPlusOne: %s\n", tld)
		suffix, _ := publicsuffix.PublicSuffix(host)
		fmt.Printf("\tPublicSuffix: %s\n", suffix)
	}
}
Exemple #5
0
Fichier : form.go Projet : cmars/oo
// visitWebPage performs the actual visit request. It attempts to
// determine that form login is supported and then download the form
// schema. It calls v.handler.Handle using the downloaded schema and then
// submits the returned form. Any error produced by v.handler.Handle will
// not have it's cause masked.
func (v webPageVisitor) visitWebPage(u *url.URL) error {
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return errgo.Notef(err, "cannot create request")
	}
	req.Header.Set("Accept", "application/json")
	var lm loginMethods
	if err := v.client.Do(req, nil, &lm); err != nil {
		if v.fallback != nil {
			if err := v.fallback(u); err != nil {
				return errgo.Mask(err, errgo.Any)
			}
			return nil
		}
		return errgo.Notef(err, "cannot get login methods")
	}
	if lm.Form == "" {
		if v.fallback != nil {
			if err := v.fallback(u); err != nil {
				return errgo.Mask(err, errgo.Any)
			}
			return nil
		}
		return errgo.Newf("form login not supported")
	}
	var s SchemaResponse
	if err := v.client.CallURL(lm.Form, &SchemaRequest{}, &s); err != nil {
		return errgo.Notef(err, "cannot get schema")
	}
	if len(s.Schema) == 0 {
		return errgo.Newf("invalid schema: no fields found")
	}
	host, err := publicsuffix.EffectiveTLDPlusOne(u.Host)
	if err != nil {
		host = u.Host
	}
	form, err := v.filler.Fill(form.Form{
		Title:  "Log in to " + host,
		Fields: s.Schema,
	})
	if err != nil {
		return errgo.NoteMask(err, "cannot handle form", errgo.Any)
	}
	lr := LoginRequest{
		Body: LoginBody{
			Form: form,
		},
	}
	if err := v.client.CallURL(lm.Form, &lr, nil); err != nil {
		return errgo.Notef(err, "cannot submit form")
	}
	return nil
}
Exemple #6
0
func check(domain string, tldPlusOne string) bool {
	// check ns record
	if len(tldPlusOne) == 0 {
		tldPlusOne, _ = publicsuffix.EffectiveTLDPlusOne(domain)
	}

	nss, err := net.LookupNS(tldPlusOne)
	if err != nil {
		log.Println("LookupNS failed", tldPlusOne, err)
		addToIgnoreList(tldPlusOne, false)
		addToIgnoreList(domain, false)
		return false
	}

	for _, v := range nss {
		if chinaNS.MatchString(v.Host) {
			addToChinaList(tldPlusOne)
			return true
		}

		// check if ns record is belong to china domain
		ns := strings.TrimSuffix(strings.TrimSpace(v.Host), ".")
		ns, err = publicsuffix.EffectiveTLDPlusOne(domain)
		if err == nil && isChina(ns) {
			addToChinaList(tldPlusOne)
			return true
		}
	}

	// put it in ignores list to avoid double check
	addToIgnoreList(tldPlusOne, false)

	if len(nss) > 0 {
		log.Println("out-china ns server:", nss[0].Host)
	}

	return false
}
Exemple #7
0
func (edb *EntriesDatabase) insertRegisteredDomains(txn *gorp.Transaction, certId uint64, names map[string]struct{}) error {
	domains := make(map[string]struct{})
	for name, _ := range names {
		domain, err := publicsuffix.EffectiveTLDPlusOne(name)
		if err != nil {
			// This is non-critical. We'd rather have the cert with an incomplete
			// eTLD, so mask this error
			if edb.Verbose {
				fmt.Printf("%s\n", err)
			}
			continue
		}
		domains[domain] = struct{}{}
	}
	for domain, _ := range domains {
		etld, _ := publicsuffix.PublicSuffix(domain)
		label := strings.Replace(domain, "."+etld, "", 1)

		var regdomId uint64
		err := txn.SelectOne(&regdomId, "SELECT regdomID FROM registereddomain WHERE domain = ? LIMIT 1", domain)
		if err != nil {
			domainObj := &RegisteredDomain{
				Domain: domain,
				ETLD:   etld,
				Label:  label,
			}
			// Ignore errors on insert
			err := txn.Insert(domainObj)
			if errorIsNotDuplicate(err) {
				return fmt.Errorf("DB error on Registered Domain: %s: %s", domain, err)
			}
			regdomId = domainObj.RegDomID
		}

		certRegDomObj := &CertToRegisteredDomain{
			RegDomID: regdomId,
			CertID:   certId,
		}
		// Ignore errors on insert
		err = txn.Insert(certRegDomObj)
		if errorIsNotDuplicate(err) {
			return fmt.Errorf("DB error on Registered Domain: %s: %s", domain, err)
		}
	}
	return nil
}
Exemple #8
0
func GetCommonName(domain string) (host string, err error) {
	eTLD_1, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		glog.V(1).Infof("GetCommonName(%s) error: %v", domain, err)
		return domain, nil
	}

	prefix := strings.TrimRight(strings.TrimSuffix(domain, eTLD_1), ".")
	switch {
	case prefix == "":
		host = eTLD_1
	case strings.Contains(prefix, "."):
		host = fmt.Sprintf("*.%s.%s", strings.SplitN(prefix, ".", 2)[1], eTLD_1)
	default:
		host = "*." + eTLD_1
	}

	return
}
Exemple #9
0
// Returns an empty string on failure to skip explicit domain.
func (c *Cookie) cookieDomain(host string) string {
	if strings.Contains(host, ":") {
		h, _, err := net.SplitHostPort(host)
		if err != nil {
			c.Logger.Printf("Error parsing host: %s", host)
			return ""
		}
		host = h
	}
	if host == "localhost" {
		return ""
	}
	if net.ParseIP(host) != nil {
		return ""
	}
	registered, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		c.Logger.Printf("Error extracting base domain: %s", err)
		return ""
	}
	return "." + registered
}
func isChina(domain string) bool {
	if val, ok := ignores[domain]; ok {
		return val
	}

	d, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		log.Println(err)
		return false
	}

	if val, ok := ignores[d]; ok {
		return val
	}

	for k, v := range ignores {
		if strings.HasSuffix(domain, k) {
			return v
		}
	}

	return false
}
func isIgnored(domain string) bool {
	if _, ok := ignores[domain]; ok {
		return true
	}

	d, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		log.Println(err)
		return true
	}

	if _, ok := ignores[d]; ok {
		return true
	}

	for k := range ignores {
		if strings.HasSuffix(domain, k) {
			return true
		}
	}

	return false
}
Exemple #12
0
func effectiveDomain(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	host := u.Host
	if host == "" {
		return "", fmt.Errorf("unparsable domain string %#v", str)
	}
	i := strings.Index(host, ":")
	if i >= 0 {
		host = host[:i]
	}

	if host == "localhost" {
		return "localhost", nil
	}
	d, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		return "", err
	}
	return d, nil
}
Exemple #13
0
func checkDomain(domain string) {
	domain = strings.ToLower(domain)

	if !strings.Contains(domain, ".") {
		return
	}

	if !isDomainName(domain) {
		return
	}

	if isIgnored(domain) {
		return
	}

	tldPlusOne, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		log.Println(err)
		tldPlusOne = domain
	}

	check(domain, tldPlusOne)
}
Exemple #14
0
// visitWebPage is the internal version of VisitWebPage that operates
// on a Doer rather than an httpbakery.Client, so that we
// can remain compatible with the historic
// signature of the VisitWebPage function.
func (v Visitor) visitWebPage(doer httprequest.Doer, methodURLs map[string]*url.URL) error {
	schemaURL := methodURLs[InteractionMethod]
	if schemaURL == nil {
		return httpbakery.ErrMethodNotSupported
	}
	logger.Infof("got schemaURL %v", schemaURL)
	httpReqClient := &httprequest.Client{
		Doer: doer,
	}
	var s SchemaResponse
	if err := httpReqClient.CallURL(schemaURL.String(), &SchemaRequest{}, &s); err != nil {
		return errgo.Notef(err, "cannot get schema")
	}
	if len(s.Schema) == 0 {
		return errgo.Newf("invalid schema: no fields found")
	}
	host, err := publicsuffix.EffectiveTLDPlusOne(schemaURL.Host)
	if err != nil {
		host = schemaURL.Host
	}
	form, err := v.Filler.Fill(form.Form{
		Title:  "Log in to " + host,
		Fields: s.Schema,
	})
	if err != nil {
		return errgo.NoteMask(err, "cannot handle form", errgo.Any)
	}
	lr := LoginRequest{
		Body: LoginBody{
			Form: form,
		},
	}
	if err := httpReqClient.CallURL(schemaURL.String(), &lr, nil); err != nil {
		return errgo.Notef(err, "cannot submit form")
	}
	return nil
}