/* Search is used for retrieving watchers. It accepts the following GET params: + duuid - uuid of domain + domain - string of domain + name - string of name */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var where []string var args []interface{} i := 1 for k, _ := range params { switch k { case "user": user, err := auth.GetUser(r) if err != nil { util.Error(err, w) return } where = append(where, fmt.Sprintf("users@> '[%d]'", user.ID)) i++ case "name": where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, params.Get(k)) i++ case "duuid", "domain": where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, params.Get(k)) i++ } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) watcherList, err := db.GetList(query, args...) util.ToJSON(watcherList, err, w) }
func testDetectDomainAndTLD() { s, t, err := tlds.DetectDomainAndTLD("ns1.google.co.uk") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.ng") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.com") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) s, t, err = tlds.DetectDomainAndTLD("ns1.google.foo") if err != nil { log.Error("%s", err) return } log.Info("%+v", t) log.Info("%+v", s) }
func (c Crawler) Start() { go func() { for { select { case <-c.quit: log.Info("Quit crawler.") return default: domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List() if err != nil { log.Error("Crawler. Unable to get domains: %s", err) // shutdown c.Stop() return } for _, d := range domainList { whoisDispatcher.AddDomain(d) digDispatcher.AddDomain(d) time.Sleep(c.delay) } } c.offset += WINDOW } }() }
/* Search is used for retrieving notifications. It accepts the following GET params: */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var where []string var args []interface{} var orderBy = "" i := 1 // inject user user, err := auth.GetUser(r) if err != nil { util.Error(err, w) return } where = append(where, fmt.Sprintf("user_id = $%d", i)) args = append(args, user.ID) i++ for k, _ := range params { switch k { case "duuid": where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, params.Get(k)) i++ case "orderBy": orderBy = fmt.Sprintf("ORDER BY $%d ", i) args = append(args, params.Get(k)) if v, ok := params["order"]; ok { order := strings.ToLower(v[0]) if order == "desc" { orderBy += "DESC " } else if order == "asc" { orderBy += "ASC " } } } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } if len(orderBy) > 0 { query += orderBy + " " } query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) notificationList, err := db.GetList(query, args...) util.ToJSON(notificationList, err, w) }
func init() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill) go func() { <-c log.Info("Shutting down") os.Exit(1) }() }
func Create(w http.ResponseWriter, r *http.Request) { var post struct { Domain string `json:"domain"` Name string `json:"name"` Interval string `json:"interval"` } decoder := json.NewDecoder(r.Body) err := decoder.Decode(&post) if err != nil { util.Error(err, w) return } var domain domains.Domain if len(post.Name) > 0 { name, tld, err := tlds.DetectDomainAndTLD(post.Name) if err != nil { util.Error(err, w) return } domain, err = domains.GetByNameAndTLD(name, tld).One() if err != nil { domain = domains.New(name, tld) err = domain.Insert() if err != nil { util.Error(err, w) return } } } else { domain, err = domains.GetByUUID(post.Domain).One() if err != nil { util.Error(err, w) return } } watcher, err := db.GetByDomain(domain).One() if err != nil { watcher, err = db.New(domain, post.Interval) if err != nil { util.Error(err, w) return } } watcher.SetLowerInterval(post.Interval) user, err := auth.GetUser(r) if err != nil { util.Error(err, w) return } watcher.AddUser(*user) err = watcher.Save() log.Info("%+v", watcher) dispatcher.AddDomain(domain) util.ToJSON(watcher, err, w) }
func main() { w, err := watcher.New() if err != nil { log.Error("Unable to start Watcher: %s", err) return } w.Start() log.Info("Watcher started.") q := make(chan bool) <-q }
func (w Watcher) handler(i intervals.Interval) { list, err := db.GetByInterval(i).List() if err != nil { log.Error("Error parsing interval (%d): %s", i.ID, err) return } for _, watch := range list { w.DigParser.Exec(watch.Domain) before, err := whois.GetByDomain(watch.Domain).Count() if err != nil { log.Error("%s", err) } w.WhoisParser.Exec(watch.Domain) after, err := whois.GetByDomain(watch.Domain).Count() if err != nil { log.Error("%s", err) } if after > before && before > 0 { log.Info("%s whois changed", watch.Domain) message := notifications.Message{ Added: time.Now(), Message: "Domain Whois update.", Domain: watch.Domain, } for _, user := range watch.Users { log.Info("%s", user.Email) noter, _ := notifications.SetupUserNotification(user) noter.AddMessage(message) } } err = watch.Save() if err != nil { log.Error("%s", err) } } }
/* Search is used for retrieving records for a domain. It accepts the following GET params: + duuid - uuid of domain + domain - string of domain + name - string of name */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var where []string var args []interface{} i := 1 for k, _ := range params { switch k { case "name": where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, params.Get(k)) i++ case "duuid", "domain": where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, params.Get(k)) v := params.Get(k) params.Del("duuid") params.Del("domain") params.Set("domain", v) i++ } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) recordList, err := db.GetList(query, args...) if err != nil { util.Error(err, w) return } // check for none sql errors // if we have no results dispatch a worker to get one if len(recordList) == 0 { // no domain lets grab one using what we assume is a duuid log.Info("no records") if duuid := params.Get("domain"); duuid != "" { log.Info("duuid: " + duuid) domain, err := domains.GetByUUID(duuid).One() if err != nil { log.Info("herererere") util.Error(err, w) return } results := <-dispatcher.AddDomain(domain) log.Info("%v", results) for _, result := range results { recordList = append(recordList, result) } } } util.ToJSON(cleanParserFromRecords(recordList), err, w) }
func (p *Parser) Parse() error { defer p.Close() defer util.Un(util.Trace("zonefile parser")) ri, err := records.NewBulkInsert() if err != nil { return err } p.recordInsert = &ri bi, err := domains.NewBulkInsert() if err != nil { return err } p.recordTypes = make(map[string]int32) p.domainInsert = &bi log.Info("Parsing Zonefile: %s", p.String()) err = p.Update("Load file in to temporary tables.") if err != nil { log.Error("%s", err) return err } var previous string p.lineCount = 0 func() { defer util.Un(util.Trace("zonefile parser scan file")) for p.scanner.Scan() { p.lineCount++ p.line = strings.ToLower(p.scanner.Text()) line := p.line commentIdx := strings.Index(line, ";") if commentIdx > 0 { //comment := line[commentIdx:] line = line[:commentIdx] } if len(line) == 0 { continue } switch line[0] { case ';': p.handleComment(line) case '@': // need to wdd more ways of detecting SOA, could have a switch // that only goes to handleLine when it has parsed $origin var p.handleSOA(line) case '$': p.handleVariable(line) case ' ': case ' ': p.handleZonedLine(line, previous) default: if !p.originCheck { p.handleSOA(line) } else { p.handleLine(line) } } previous = line } err = p.Update("File read in to temporary tables.") if err != nil { log.Error("Unable to read in temporary tables: %s", err) return } log.Info("Parse %s complete. Proceed with sql operations.", p.String()) }() // insert our domains and commit our tx to avoid err = p.Update("Close insert domains statement.") if err != nil { return err } p.domainInsert.Insert() err = p.Update("Close insert records statement.") if err != nil { return err } p.recordInsert.Insert() // TODO: drop index to record__%d_%d // create index's err = p.Update("Create temp domain table index.") if err != nil { return err } err = p.domainInsert.Index(fmt.Sprintf("CREATE INDEX _domain__%d_uuid_idx ON %%s (uuid)", p.TLD.ID)) if err != nil { return err } err = p.Update("Create temp record table index.") err = p.recordInsert.Index("CREATE INDEX uuid_idx ON %s (uuid)") if err != nil { return err } err = p.Update("Merge temp domain with domain.") err = p.domainInsert.Merge( fmt.Sprintf(` INSERT INTO domain__%d SELECT DISTINCT * FROM %%s d2 WHERE NOT EXISTS ( SELECT NULL FROM domain__%d d WHERE d.uuid = d2.uuid )`, p.TLD.ID, p.TLD.ID, ), ) if err != nil { return err } for rtName, rtID := range p.recordTypes { err = p.recordInsert.Exec(fmt.Sprintf( "DROP INDEX IF EXISTS record__%d_%d_idx", rtID, p.TLD.ID, )) if err != nil { return err } err = p.Update("Merge temp record with record (type: %s).", rtName) if err != nil { return err } err = p.recordInsert.Merge( fmt.Sprintf(` INSERT INTO record__%d_%d SELECT DISTINCT ON (uuid) *, %d FROM %%s r2 WHERE NOT EXISTS ( SELECT NULL FROM record__%d_%d r WHERE r.uuid = r2.uuid AND r.record_type = %d ) AND r2.record_type = %d`, rtID, p.TLD.ID, p.ID, rtID, p.TLD.ID, rtID, rtID, ), ) if err != nil { return err } err = p.recordInsert.Exec(fmt.Sprintf( "CREATE INDEX record__%d_%d_idx ON record__%d_%d USING HASH (domain)", rtID, p.TLD.ID, rtID, p.TLD.ID, )) if err != nil { return err } } err = p.Update("Commit domain insert.") if err != nil { return err } p.domainInsert.Finish() // TODO: add index to record__%d_%d err = p.Update("Commit record insert.") if err != nil { return err } p.recordInsert.Finish() log.Info("Parse %s complete", p.String()) return nil }
/* Search is used for retrieving domain records. It accepts the following GET params: + uuid - uuid of domain + name - string representation of domain (can include wildcards) + email - domains attached to an email */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var where []string var args []interface{} i := 1 for k, _ := range params { switch k { case "uuid": where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, params.Get(k)) case "email": // special case which overrides results domainList, err := db.GetByJoinWhoisEmails(params.Get(k)).List() util.ToJSON(domainList, err, w) return case "organization": domainList, err := db.GetByJoinWhoisOrganizations(params.Get(k)).List() util.ToJSON(domainList, err, w) return case "person_name": domainList, err := db.GetByJoinWhoisNames(params.Get(k)).List() util.ToJSON(domainList, err, w) return case "phone": domainList, err := db.GetByJoinWhoisPhones(params.Get(k)).List() util.ToJSON(domainList, err, w) return case "postcode": domainList, err := db.GetByJoinWhoisPostcodes(params.Get(k)).List() util.ToJSON(domainList, err, w) return case "name": name := params.Get(k) if strings.ContainsAny(name, ".") { // attempt to detect tld tld, err := tlds.Detect(name) if err == nil { where = append(where, fmt.Sprintf("tld = $%d", i)) args = append(args, tld.ID) // strip tld from name name = strings.TrimSuffix(name, "."+tld.Name) } } if strings.ContainsAny(name, "* || % || ") { // contains wildcards name = strings.Replace(name, "*", "%", -1) name = strings.Replace(name, " ", "%", -1) } where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, name) } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) domainList, err := db.GetList(query, args...) util.ToJSON(domainList, err, w) }
/* Search is used for retrieving whois records for a domain. It accepts the following GET params: + uuid - uuid of record + duuid - uuid of domain + domain - string of domain */ func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) { query := db.SELECT var domain domains.Domain var where []string var args []interface{} var orderBy string i := 1 for k, _ := range params { switch k { case "orderBy": orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k)) if v, ok := params["order"]; ok { order := strings.ToLower(v[0]) if order == "desc" { orderBy += "DESC " } else if order == "asc" { orderBy += "ASC " } } case "uuid": where = append(where, fmt.Sprintf(k+" = $%d", i)) args = append(args, params.Get(k)) i++ case "duuid", "domain": where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, params.Get(k)) v := params.Get(k) params.Del("duuid") params.Del("domain") params.Set("domain", v) i++ case "name": name, tld, err := tlds.DetectDomainAndTLD(params.Get(k)) if err != nil { util.Error(err, w) return } domain, err = domains.GetByNameAndTLD(name, tld).One() if err != nil { domain = domains.New(name, tld) err = domain.Insert() if err != nil { util.Error(err, w) return } } where = append(where, fmt.Sprintf("domain = $%d", i)) args = append(args, domain.UUID.String()) i++ case "email": where = append(where, fmt.Sprintf("emails ? $%d", i)) args = append(args, params.Get(k)) i++ case "raw": where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i)) args = append(args, "%"+params.Get(k)+"%") i++ } } if len(where) > 0 { query += "WHERE " + strings.Join(where, " AND ") + " " } if len(orderBy) > 0 { query += orderBy + " " } query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2) args = append(args, limit, offset) log.Info("Query: " + query) log.Info("Args: %+v", args) recordList, err := db.GetList(query, args...) if err != nil { panic(err) } // check for non sql errors // if we have no results dispatch a worker to get one if len(recordList) == 0 { if len(domain.Name) == 0 { // no domain lets grab one using what we assume is a duuid if duuid := params.Get("domain"); duuid != "" { log.Info("duuid: " + duuid) domain, err = domains.GetByUUID(duuid).One() if err != nil { util.Error(err, w) return } } } if len(domain.Name) == 0 { log.Error("Unable to detect domain for whois lookup, params: %+v", params) util.Error(errors.New("Unable to find domain for whois lookup."), w) return } result := <-dispatcher.AddDomain(domain) recordList = append(recordList, result) } util.ToJSON(recordList, err, w) }