func testDetectDomainAndTLD() { s, t, err := tlds.DetectDomainAndTLD("ns1.google.co.uk") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.ng") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.com") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.foo") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) }
func AddQuery(q string) Record { res := make(chan whois.Record) // could offload this in to a seperate anon function to avoid bottleneck s, t, err := tlds.DetectDomainAndTLD(q) if err != nil { log.Error("Whois dipatcher: Unable to detect TLD and domain: %s (%s)", err, q) return res } d, err := domains.GetByNameAndTLD(s, t).One() if err != nil { log.Error("Whois dispatcher: unable to get domain: %s (%s)", err, s) d = domains.New(s, t) err = d.Insert() if err != nil { log.Error("Whois dispatcher: unable to insert domain: %s (%s)", err, d.String()) return res } } wr := worker.Request{ Domain: d, Record: res, } Work <- wr return res }
func (p *Parser) Exec(d domains.Domain) (whois.Record, error) { out, err := exec.Command("pwhois", "-j", d.String()).Output() if err != nil { log.Error("Whois error for " + d.String()) return whois.Record{}, err } w, err := whois.New(d, out) if err != nil { log.Error("Parse Whois for %s: %s", d.String(), err) return w, err } return w, w.Insert() }
func (c Crawler) Start() { go func() { for { select { case <-c.quit: log.Info("Quit crawler.") return default: domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List() if err != nil { log.Error("Crawler. Unable to get domains: %s", err) // shutdown c.Stop() return } for _, d := range domainList { whoisDispatcher.AddDomain(d) digDispatcher.AddDomain(d) time.Sleep(c.delay) } } c.offset += WINDOW } }() }
func (p *Parser) Exec(d domains.Domain) ([]records.Record, error) { out, err := exec.Command("dig", "all", d.String()).Output() if err != nil { return []records.Record{}, err } date := time.Now() origin := d.TLD.Name + "." lines := strings.Split(strings.ToLower(string(out)), "\n") results := make([]records.Record, 0) for _, line := range lines { if len(line) == 0 || line[0] == ';' { continue } rr, err := records.New(line, origin, d.TLD, 86400, date, 0) if err != nil { log.Error("%s: %s", line, err) continue } err = rr.Insert() if err != nil { //log.Error("%s: %s", line, err) continue } results = append(results, rr) } return results, nil }
func (p *Parser) handleLine(line string) { rr, err := records.New(line, p.origin, p.TLD, p.ttl, p.Date, p.ID) if err != nil { log.Warn("handleLine:getRecord: %s", err) log.Warn("handleLine:line: %s", line) return } rr.Parser = p.Parser err = p.domainInsert.Add(&rr.Domain) if err != nil { log.Error("handleLine: Unable to bulk insert Domain: %s", err) return } err = p.recordInsert.Add(&rr) if err != nil { log.Error("handleLine:recordInsert.Add: %s", err) return } p.recordTypes[rr.Type.Name] = rr.Type.ID }
func main() { w, err := watcher.New() if err != nil { log.Error("Unable to start Watcher: %s", err) return } w.Start() log.Info("Watcher started.") q := make(chan bool) <-q }
func main() { go func() { w, err := watcher.New() if err != nil { log.Error("Unable to start watcher: %s", err) return } w.Start() }() startREST() }
func (r Request) Do(w Worker) { res, err := w.Parser.Exec(r.Domain) if err != nil { log.Error("worker%d: unable to parse domain: %s", w.ID, err) return } select { case r.Record <- res: break default: break } }
func (r Request) Do(w Worker) { res, err := w.Parser.Exec(r.Domain) if err != nil { log.Error("worker%d: unable to parse domain: %s", w.ID, err) return } //log.Info("Results for %s (%s): %d", r.Domain, r.Domain.UUID.String(), len(res)) select { case r.Result <- res: break default: break } }
func (w Watcher) handler(i intervals.Interval) { list, err := db.GetByInterval(i).List() if err != nil { log.Error("Error parsing interval (%d): %s", i.ID, err) return } for _, watch := range list { w.DigParser.Exec(watch.Domain) before, err := whois.GetByDomain(watch.Domain).Count() if err != nil { log.Error("%s", err) } w.WhoisParser.Exec(watch.Domain) after, err := whois.GetByDomain(watch.Domain).Count() if err != nil { log.Error("%s", err) } if after > before && before > 0 { log.Info("%s whois changed", watch.Domain) message := notifications.Message{ Added: time.Now(), Message: "Domain Whois update.", Domain: watch.Domain, } for _, user := range watch.Users { log.Info("%s", user.Email) noter, _ := notifications.SetupUserNotification(user) noter.AddMessage(message) } } err = watch.Save() if err != nil { log.Error("%s", err) } } }
func (p *Parser) setupFile(filename string, gunzip bool) error { _p, err := db.New(filename) if err != nil { log.Error("Unable to setup Zonefile: %s", err) return err } p.Filename = _p.Filename p.TLD = _p.TLD p.Date = _p.Date p.origin = p.TLD.Name + "." err = p.Insert() if err != nil { log.Error("Unable to save Zonefile Parser: %s", err) return err } file, err := os.Open(filename) if err != nil { log.Error("Unable to open Zonefile: %s", err) return err } p.setupFileDefer = func() { file.Close() } var reader io.Reader = file if gunzip { reader, err = gzip.NewReader(file) if err != nil { log.Error("Unable to setup Zonefile gzip reader: %s", err) return err } } p.scanner = bufio.NewScanner(reader) return nil }
func parseZonefiles() { config := jconfig.LoadConfig("config.json") dir := config.GetString("zonefile_dir") p := zonefile.New() files := []string{ //"20141113-net.zone.gz", "20140621-biz.zone.gz", "20140622-biz.zone.gz", "20141210-biz.zone.gz", } for _, f := range files { err := p.SetupGunzipFile(dir + f) if err != nil { log.Error("Unable to setup %s: %s", f, err) return } p.Parse() } }
func (p *Parser) Parse() error { defer p.Close() defer util.Un(util.Trace("zonefile parser")) ri, err := records.NewBulkInsert() if err != nil { return err } p.recordInsert = &ri bi, err := domains.NewBulkInsert() if err != nil { return err } p.recordTypes = make(map[string]int32) p.domainInsert = &bi log.Info("Parsing Zonefile: %s", p.String()) err = p.Update("Load file in to temporary tables.") if err != nil { log.Error("%s", err) return err } var previous string p.lineCount = 0 func() { defer util.Un(util.Trace("zonefile parser scan file")) for p.scanner.Scan() { p.lineCount++ p.line = strings.ToLower(p.scanner.Text()) line := p.line commentIdx := strings.Index(line, ";") if commentIdx > 0 { //comment := line[commentIdx:] line = line[:commentIdx] } if len(line) == 0 { continue } switch line[0] { case ';': p.handleComment(line) case '@': // need to wdd more ways of detecting SOA, could have a switch // that only goes to handleLine when it has parsed $origin var p.handleSOA(line) case '$': p.handleVariable(line) case ' ': case ' ': p.handleZonedLine(line, previous) default: if !p.originCheck { p.handleSOA(line) } else { p.handleLine(line) } } previous = line } err = p.Update("File read in to temporary tables.") if err != nil { log.Error("Unable to read in temporary tables: %s", err) return } log.Info("Parse %s complete. Proceed with sql operations.", p.String()) }() // insert our domains and commit our tx to avoid err = p.Update("Close insert domains statement.") if err != nil { return err } p.domainInsert.Insert() err = p.Update("Close insert records statement.") if err != nil { return err } p.recordInsert.Insert() // TODO: drop index to record__%d_%d // create index's err = p.Update("Create temp domain table index.") if err != nil { return err } err = p.domainInsert.Index(fmt.Sprintf("CREATE INDEX _domain__%d_uuid_idx ON %%s (uuid)", p.TLD.ID)) if err != nil { return err } err = p.Update("Create temp record table index.") err = p.recordInsert.Index("CREATE INDEX uuid_idx ON %s (uuid)") if err != nil { return err } err = p.Update("Merge temp domain with domain.") err = p.domainInsert.Merge( fmt.Sprintf(` INSERT INTO domain__%d SELECT DISTINCT * FROM %%s d2 WHERE NOT EXISTS ( SELECT NULL FROM domain__%d d WHERE d.uuid = d2.uuid )`, p.TLD.ID, p.TLD.ID, ), ) if err != nil { return err } for rtName, rtID := range p.recordTypes { err = p.recordInsert.Exec(fmt.Sprintf( "DROP INDEX IF EXISTS record__%d_%d_idx", rtID, p.TLD.ID, )) if err != nil { return err } err = p.Update("Merge temp record with record (type: %s).", rtName) if err != nil { return err } err = p.recordInsert.Merge( fmt.Sprintf(` INSERT INTO record__%d_%d SELECT DISTINCT ON (uuid) *, %d FROM %%s r2 WHERE NOT EXISTS ( SELECT NULL FROM record__%d_%d r WHERE r.uuid = r2.uuid AND r.record_type = %d ) AND r2.record_type = %d`, rtID, p.TLD.ID, p.ID, rtID, p.TLD.ID, rtID, rtID, ), ) if err != nil { return err } err = p.recordInsert.Exec(fmt.Sprintf( "CREATE INDEX record__%d_%d_idx ON record__%d_%d USING HASH (domain)", rtID, p.TLD.ID, rtID, p.TLD.ID, )) if err != nil { return err } } err = p.Update("Commit domain insert.") if err != nil { return err } p.domainInsert.Finish() // TODO: add index to record__%d_%d err = p.Update("Commit record insert.") if err != nil { return err } p.recordInsert.Finish() log.Info("Parse %s complete", p.String()) return nil }
/* Search is used for retrieving whois records for a domain. It accepts the following GET params: + uuid - uuid of record + duuid - uuid of domain + domain - string of domain */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var domain domains.Domain var where []string var args []interface{} var orderBy string i := 1 for k, _ := range params { switch k { case "orderBy": orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k)) if v, ok := params["order"]; ok { order := strings.ToLower(v[0]) if order == "desc" { orderBy += "DESC " } else if order == "asc" { orderBy += "ASC " } } case "uuid": where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, params.Get(k)) i++ case "duuid", "domain": where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, params.Get(k)) v := params.Get(k) params.Del("duuid") params.Del("domain") params.Set("domain", v) i++ case "name": name, tld, err := tlds.DetectDomainAndTLD(params.Get(k)) if err != nil { util.Error(err, w) return } domain, err = domains.GetByNameAndTLD(name, tld).One() if err != nil { domain = domains.New(name, tld) err = domain.Insert() if err != nil { util.Error(err, w) return } } where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, domain.UUID.String()) i++ case "email": where = append(where, fmt.Sprintf("emails ? $%d", i)) args = append(args, params.Get(k)) i++ case "raw": where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i)) args = append(args, "%"+params.Get(k)+"%") i++ } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } if len(orderBy) > 0 { query += orderBy + " " } query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) recordList, err := db.GetList(query, args...) if err != nil { panic(err) } // check for non sql errors // if we have no results dispatch a worker to get one if len(recordList) == 0 { if len(domain.Name) == 0 { // no domain lets grab one using what we assume is a duuid if duuid := params.Get("domain"); duuid != "" { log.Info("duuid: " + duuid) domain, err = domains.GetByUUID(duuid).One() if err != nil { util.Error(err, w) return } } } if len(domain.Name) == 0 { log.Error("Unable to detect domain for whois lookup, params: %+v", params) util.Error(errors.New("Unable to find domain for whois lookup."), w) return } result := <-dispatcher.AddDomain(domain) recordList = append(recordList, result) } util.ToJSON(recordList, err, w) }