func printSuffix() {
host := *FSuffix
tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
fmt.Printf("EffectiveTLDPlusOne: %s\n", tld)
suffix, _ := publicsuffix.PublicSuffix(host)
fmt.Printf("PublicSuffix: %s\n", suffix)
}
func (self *Server) Blocked(host string) bool {
blocked, cached := false, false
host = HostOnly(host)
self.mutex.RLock()
if self.BlockedHosts[host] {
blocked = true
cached = true
}
self.mutex.RUnlock()
if !blocked {
tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
blocked = self.Cfg.Blocked(tld)
}
if !blocked {
suffix, _ := publicsuffix.PublicSuffix(host)
blocked = self.Cfg.Blocked(suffix)
}
if blocked && !cached {
self.mutex.Lock()
self.BlockedHosts[host] = true
self.mutex.Unlock()
}
return blocked
}
// Adds the DNS message data to the supplied MapStr.
func addDnsToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional bool) {
m["id"] = dns.Id
m["op_code"] = dnsOpCodeToString(dns.Opcode)
m["flags"] = common.MapStr{
"authoritative": dns.Authoritative,
"truncated_response": dns.Truncated,
"recursion_desired": dns.RecursionDesired,
"recursion_available": dns.RecursionAvailable,
"authentic_data": dns.AuthenticatedData, // [RFC4035]
"checking_disabled": dns.CheckingDisabled, // [RFC4035]
}
m["response_code"] = dnsResponseCodeToString(dns.Rcode)
if len(dns.Question) > 0 {
q := dns.Question[0]
qMapStr := common.MapStr{
"name": q.Name,
"type": dnsTypeToString(q.Qtype),
"class": dnsClassToString(q.Qclass),
}
m["question"] = qMapStr
eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(strings.TrimRight(q.Name, "."))
if err == nil {
qMapStr["etld_plus_one"] = eTLDPlusOne + "."
}
}
rrOPT := dns.IsEdns0()
if rrOPT != nil {
m["opt"] = optToMapStr(rrOPT)
}
m["answers_count"] = len(dns.Answer)
if len(dns.Answer) > 0 {
m["answers"] = rrsToMapStrs(dns.Answer)
}
m["authorities_count"] = len(dns.Ns)
if authority && len(dns.Ns) > 0 {
m["authorities"] = rrsToMapStrs(dns.Ns)
}
if rrOPT != nil {
m["additionals_count"] = len(dns.Extra) - 1
} else {
m["additionals_count"] = len(dns.Extra)
}
if additional && len(dns.Extra) > 0 {
rrsMapStrs := rrsToMapStrs(dns.Extra)
// We do not want OPT RR to appear in the 'additional' section,
// that's why rrsMapStrs could be empty even though len(dns.Extra) > 0
if len(rrsMapStrs) > 0 {
m["additionals"] = rrsMapStrs
}
}
}
func main() {
for _, host := range os.Args[1:] {
fmt.Printf("Host: %s\n", host)
tld, _ := publicsuffix.EffectiveTLDPlusOne(host)
fmt.Printf("\tEffectiveTLDPlusOne: %s\n", tld)
suffix, _ := publicsuffix.PublicSuffix(host)
fmt.Printf("\tPublicSuffix: %s\n", suffix)
}
}
// visitWebPage performs the actual visit request. It attempts to
// determine that form login is supported and then download the form
// schema. It calls v.handler.Handle using the downloaded schema and then
// submits the returned form. Any error produced by v.handler.Handle will
// not have it's cause masked.
func (v webPageVisitor) visitWebPage(u *url.URL) error {
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return errgo.Notef(err, "cannot create request")
}
req.Header.Set("Accept", "application/json")
var lm loginMethods
if err := v.client.Do(req, nil, &lm); err != nil {
if v.fallback != nil {
if err := v.fallback(u); err != nil {
return errgo.Mask(err, errgo.Any)
}
return nil
}
return errgo.Notef(err, "cannot get login methods")
}
if lm.Form == "" {
if v.fallback != nil {
if err := v.fallback(u); err != nil {
return errgo.Mask(err, errgo.Any)
}
return nil
}
return errgo.Newf("form login not supported")
}
var s SchemaResponse
if err := v.client.CallURL(lm.Form, &SchemaRequest{}, &s); err != nil {
return errgo.Notef(err, "cannot get schema")
}
if len(s.Schema) == 0 {
return errgo.Newf("invalid schema: no fields found")
}
host, err := publicsuffix.EffectiveTLDPlusOne(u.Host)
if err != nil {
host = u.Host
}
form, err := v.filler.Fill(form.Form{
Title: "Log in to " + host,
Fields: s.Schema,
})
if err != nil {
return errgo.NoteMask(err, "cannot handle form", errgo.Any)
}
lr := LoginRequest{
Body: LoginBody{
Form: form,
},
}
if err := v.client.CallURL(lm.Form, &lr, nil); err != nil {
return errgo.Notef(err, "cannot submit form")
}
return nil
}
func check(domain string, tldPlusOne string) bool {
// check ns record
if len(tldPlusOne) == 0 {
tldPlusOne, _ = publicsuffix.EffectiveTLDPlusOne(domain)
}
nss, err := net.LookupNS(tldPlusOne)
if err != nil {
log.Println("LookupNS failed", tldPlusOne, err)
addToIgnoreList(tldPlusOne, false)
addToIgnoreList(domain, false)
return false
}
for _, v := range nss {
if chinaNS.MatchString(v.Host) {
addToChinaList(tldPlusOne)
return true
}
// check if ns record is belong to china domain
ns := strings.TrimSuffix(strings.TrimSpace(v.Host), ".")
ns, err = publicsuffix.EffectiveTLDPlusOne(domain)
if err == nil && isChina(ns) {
addToChinaList(tldPlusOne)
return true
}
}
// put it in ignores list to avoid double check
addToIgnoreList(tldPlusOne, false)
if len(nss) > 0 {
log.Println("out-china ns server:", nss[0].Host)
}
return false
}
func (edb *EntriesDatabase) insertRegisteredDomains(txn *gorp.Transaction, certId uint64, names map[string]struct{}) error {
domains := make(map[string]struct{})
for name, _ := range names {
domain, err := publicsuffix.EffectiveTLDPlusOne(name)
if err != nil {
// This is non-critical. We'd rather have the cert with an incomplete
// eTLD, so mask this error
if edb.Verbose {
fmt.Printf("%s\n", err)
}
continue
}
domains[domain] = struct{}{}
}
for domain, _ := range domains {
etld, _ := publicsuffix.PublicSuffix(domain)
label := strings.Replace(domain, "."+etld, "", 1)
var regdomId uint64
err := txn.SelectOne(®domId, "SELECT regdomID FROM registereddomain WHERE domain = ? LIMIT 1", domain)
if err != nil {
domainObj := &RegisteredDomain{
Domain: domain,
ETLD: etld,
Label: label,
}
// Ignore errors on insert
err := txn.Insert(domainObj)
if errorIsNotDuplicate(err) {
return fmt.Errorf("DB error on Registered Domain: %s: %s", domain, err)
}
regdomId = domainObj.RegDomID
}
certRegDomObj := &CertToRegisteredDomain{
RegDomID: regdomId,
CertID: certId,
}
// Ignore errors on insert
err = txn.Insert(certRegDomObj)
if errorIsNotDuplicate(err) {
return fmt.Errorf("DB error on Registered Domain: %s: %s", domain, err)
}
}
return nil
}
func GetCommonName(domain string) (host string, err error) {
eTLD_1, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
glog.V(1).Infof("GetCommonName(%s) error: %v", domain, err)
return domain, nil
}
prefix := strings.TrimRight(strings.TrimSuffix(domain, eTLD_1), ".")
switch {
case prefix == "":
host = eTLD_1
case strings.Contains(prefix, "."):
host = fmt.Sprintf("*.%s.%s", strings.SplitN(prefix, ".", 2)[1], eTLD_1)
default:
host = "*." + eTLD_1
}
return
}
// Returns an empty string on failure to skip explicit domain.
func (c *Cookie) cookieDomain(host string) string {
if strings.Contains(host, ":") {
h, _, err := net.SplitHostPort(host)
if err != nil {
c.Logger.Printf("Error parsing host: %s", host)
return ""
}
host = h
}
if host == "localhost" {
return ""
}
if net.ParseIP(host) != nil {
return ""
}
registered, err := publicsuffix.EffectiveTLDPlusOne(host)
if err != nil {
c.Logger.Printf("Error extracting base domain: %s", err)
return ""
}
return "." + registered
}
func isChina(domain string) bool {
if val, ok := ignores[domain]; ok {
return val
}
d, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
log.Println(err)
return false
}
if val, ok := ignores[d]; ok {
return val
}
for k, v := range ignores {
if strings.HasSuffix(domain, k) {
return v
}
}
return false
}
func isIgnored(domain string) bool {
if _, ok := ignores[domain]; ok {
return true
}
d, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
log.Println(err)
return true
}
if _, ok := ignores[d]; ok {
return true
}
for k := range ignores {
if strings.HasSuffix(domain, k) {
return true
}
}
return false
}
func effectiveDomain(str string) (string, error) {
u, err := url.Parse(str)
if err != nil {
return "", err
}
host := u.Host
if host == "" {
return "", fmt.Errorf("unparsable domain string %#v", str)
}
i := strings.Index(host, ":")
if i >= 0 {
host = host[:i]
}
if host == "localhost" {
return "localhost", nil
}
d, err := publicsuffix.EffectiveTLDPlusOne(host)
if err != nil {
return "", err
}
return d, nil
}
func checkDomain(domain string) {
domain = strings.ToLower(domain)
if !strings.Contains(domain, ".") {
return
}
if !isDomainName(domain) {
return
}
if isIgnored(domain) {
return
}
tldPlusOne, err := publicsuffix.EffectiveTLDPlusOne(domain)
if err != nil {
log.Println(err)
tldPlusOne = domain
}
check(domain, tldPlusOne)
}
// visitWebPage is the internal version of VisitWebPage that operates
// on a Doer rather than an httpbakery.Client, so that we
// can remain compatible with the historic
// signature of the VisitWebPage function.
func (v Visitor) visitWebPage(doer httprequest.Doer, methodURLs map[string]*url.URL) error {
schemaURL := methodURLs[InteractionMethod]
if schemaURL == nil {
return httpbakery.ErrMethodNotSupported
}
logger.Infof("got schemaURL %v", schemaURL)
httpReqClient := &httprequest.Client{
Doer: doer,
}
var s SchemaResponse
if err := httpReqClient.CallURL(schemaURL.String(), &SchemaRequest{}, &s); err != nil {
return errgo.Notef(err, "cannot get schema")
}
if len(s.Schema) == 0 {
return errgo.Newf("invalid schema: no fields found")
}
host, err := publicsuffix.EffectiveTLDPlusOne(schemaURL.Host)
if err != nil {
host = schemaURL.Host
}
form, err := v.Filler.Fill(form.Form{
Title: "Log in to " + host,
Fields: s.Schema,
})
if err != nil {
return errgo.NoteMask(err, "cannot handle form", errgo.Any)
}
lr := LoginRequest{
Body: LoginBody{
Form: form,
},
}
if err := httpReqClient.CallURL(schemaURL.String(), &lr, nil); err != nil {
return errgo.Notef(err, "cannot submit form")
}
return nil
}