/*
Search is used for retrieving watchers. It accepts the following
GET params:
+ duuid - uuid of domain
+ domain - string of domain
+ name - string of name
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var where []string
var args []interface{}
i := 1
for k, _ := range params {
switch k {
case "user":
user, err := auth.GetUser(r)
if err != nil {
util.Error(err, w)
return
}
where = append(where, fmt.Sprintf("users@> '[%d]'", user.ID))
i++
case "name":
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, params.Get(k))
i++
case "duuid", "domain":
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, params.Get(k))
i++
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
watcherList, err := db.GetList(query, args...)
util.ToJSON(watcherList, err, w)
}
func testDetectDomainAndTLD() {
s, t, err := tlds.DetectDomainAndTLD("ns1.google.co.uk")
if err != nil {
log.Error("%s", err)
return
}
log.Info("%+v", t)
log.Info("%+v", s)
s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.ng")
if err != nil {
log.Error("%s", err)
return
}
log.Info("%+v", t)
log.Info("%+v", s)
s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.com")
if err != nil {
log.Error("%s", err)
return
}
log.Info("%+v", t)
log.Info("%+v", s)
s, t, err = tlds.DetectDomainAndTLD("ns1.google.foo")
if err != nil {
log.Error("%s", err)
return
}
log.Info("%+v", t)
log.Info("%+v", s)
}
func (c Crawler) Start() {
go func() {
for {
select {
case <-c.quit:
log.Info("Quit crawler.")
return
default:
domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List()
if err != nil {
log.Error("Crawler. Unable to get domains: %s", err)
// shutdown
c.Stop()
return
}
for _, d := range domainList {
whoisDispatcher.AddDomain(d)
digDispatcher.AddDomain(d)
time.Sleep(c.delay)
}
}
c.offset += WINDOW
}
}()
}
/*
Search is used for retrieving notifications. It accepts the following
GET params:
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var where []string
var args []interface{}
var orderBy = ""
i := 1
// inject user
user, err := auth.GetUser(r)
if err != nil {
util.Error(err, w)
return
}
where = append(where, fmt.Sprintf("user_id = $%d", i))
args = append(args, user.ID)
i++
for k, _ := range params {
switch k {
case "duuid":
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, params.Get(k))
i++
case "orderBy":
orderBy = fmt.Sprintf("ORDER BY $%d ", i)
args = append(args, params.Get(k))
if v, ok := params["order"]; ok {
order := strings.ToLower(v[0])
if order == "desc" {
orderBy += "DESC "
} else if order == "asc" {
orderBy += "ASC "
}
}
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
if len(orderBy) > 0 {
query += orderBy + " "
}
query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
notificationList, err := db.GetList(query, args...)
util.ToJSON(notificationList, err, w)
}
func init() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
go func() {
<-c
log.Info("Shutting down")
os.Exit(1)
}()
}
func Create(w http.ResponseWriter, r *http.Request) {
var post struct {
Domain string `json:"domain"`
Name string `json:"name"`
Interval string `json:"interval"`
}
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&post)
if err != nil {
util.Error(err, w)
return
}
var domain domains.Domain
if len(post.Name) > 0 {
name, tld, err := tlds.DetectDomainAndTLD(post.Name)
if err != nil {
util.Error(err, w)
return
}
domain, err = domains.GetByNameAndTLD(name, tld).One()
if err != nil {
domain = domains.New(name, tld)
err = domain.Insert()
if err != nil {
util.Error(err, w)
return
}
}
} else {
domain, err = domains.GetByUUID(post.Domain).One()
if err != nil {
util.Error(err, w)
return
}
}
watcher, err := db.GetByDomain(domain).One()
if err != nil {
watcher, err = db.New(domain, post.Interval)
if err != nil {
util.Error(err, w)
return
}
}
watcher.SetLowerInterval(post.Interval)
user, err := auth.GetUser(r)
if err != nil {
util.Error(err, w)
return
}
watcher.AddUser(*user)
err = watcher.Save()
log.Info("%+v", watcher)
dispatcher.AddDomain(domain)
util.ToJSON(watcher, err, w)
}
func main() {
w, err := watcher.New()
if err != nil {
log.Error("Unable to start Watcher: %s", err)
return
}
w.Start()
log.Info("Watcher started.")
q := make(chan bool)
<-q
}
func (w Watcher) handler(i intervals.Interval) {
list, err := db.GetByInterval(i).List()
if err != nil {
log.Error("Error parsing interval (%d): %s", i.ID, err)
return
}
for _, watch := range list {
w.DigParser.Exec(watch.Domain)
before, err := whois.GetByDomain(watch.Domain).Count()
if err != nil {
log.Error("%s", err)
}
w.WhoisParser.Exec(watch.Domain)
after, err := whois.GetByDomain(watch.Domain).Count()
if err != nil {
log.Error("%s", err)
}
if after > before && before > 0 {
log.Info("%s whois changed", watch.Domain)
message := notifications.Message{
Added: time.Now(),
Message: "Domain Whois update.",
Domain: watch.Domain,
}
for _, user := range watch.Users {
log.Info("%s", user.Email)
noter, _ := notifications.SetupUserNotification(user)
noter.AddMessage(message)
}
}
err = watch.Save()
if err != nil {
log.Error("%s", err)
}
}
}
/*
Search is used for retrieving records for a domain. It accepts the following
GET params:
+ duuid - uuid of domain
+ domain - string of domain
+ name - string of name
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var where []string
var args []interface{}
i := 1
for k, _ := range params {
switch k {
case "name":
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, params.Get(k))
i++
case "duuid", "domain":
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, params.Get(k))
v := params.Get(k)
params.Del("duuid")
params.Del("domain")
params.Set("domain", v)
i++
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
recordList, err := db.GetList(query, args...)
if err != nil {
util.Error(err, w)
return
}
// check for none sql errors
// if we have no results dispatch a worker to get one
if len(recordList) == 0 {
// no domain lets grab one using what we assume is a duuid
log.Info("no records")
if duuid := params.Get("domain"); duuid != "" {
log.Info("duuid: " + duuid)
domain, err := domains.GetByUUID(duuid).One()
if err != nil {
log.Info("herererere")
util.Error(err, w)
return
}
results := <-dispatcher.AddDomain(domain)
log.Info("%v", results)
for _, result := range results {
recordList = append(recordList, result)
}
}
}
util.ToJSON(cleanParserFromRecords(recordList), err, w)
}
func (p *Parser) Parse() error {
defer p.Close()
defer util.Un(util.Trace("zonefile parser"))
ri, err := records.NewBulkInsert()
if err != nil {
return err
}
p.recordInsert = &ri
bi, err := domains.NewBulkInsert()
if err != nil {
return err
}
p.recordTypes = make(map[string]int32)
p.domainInsert = &bi
log.Info("Parsing Zonefile: %s", p.String())
err = p.Update("Load file in to temporary tables.")
if err != nil {
log.Error("%s", err)
return err
}
var previous string
p.lineCount = 0
func() {
defer util.Un(util.Trace("zonefile parser scan file"))
for p.scanner.Scan() {
p.lineCount++
p.line = strings.ToLower(p.scanner.Text())
line := p.line
commentIdx := strings.Index(line, ";")
if commentIdx > 0 {
//comment := line[commentIdx:]
line = line[:commentIdx]
}
if len(line) == 0 {
continue
}
switch line[0] {
case ';':
p.handleComment(line)
case '@':
// need to wdd more ways of detecting SOA, could have a switch
// that only goes to handleLine when it has parsed $origin var
p.handleSOA(line)
case '$':
p.handleVariable(line)
case ' ':
case ' ':
p.handleZonedLine(line, previous)
default:
if !p.originCheck {
p.handleSOA(line)
} else {
p.handleLine(line)
}
}
previous = line
}
err = p.Update("File read in to temporary tables.")
if err != nil {
log.Error("Unable to read in temporary tables: %s", err)
return
}
log.Info("Parse %s complete. Proceed with sql operations.", p.String())
}()
// insert our domains and commit our tx to avoid
err = p.Update("Close insert domains statement.")
if err != nil {
return err
}
p.domainInsert.Insert()
err = p.Update("Close insert records statement.")
if err != nil {
return err
}
p.recordInsert.Insert()
// TODO: drop index to record__%d_%d
// create index's
err = p.Update("Create temp domain table index.")
if err != nil {
return err
}
err = p.domainInsert.Index(fmt.Sprintf("CREATE INDEX _domain__%d_uuid_idx ON %%s (uuid)", p.TLD.ID))
if err != nil {
return err
}
err = p.Update("Create temp record table index.")
err = p.recordInsert.Index("CREATE INDEX uuid_idx ON %s (uuid)")
if err != nil {
return err
}
err = p.Update("Merge temp domain with domain.")
err = p.domainInsert.Merge(
fmt.Sprintf(`
INSERT INTO domain__%d
SELECT DISTINCT * FROM %%s d2
WHERE NOT EXISTS (
SELECT NULL FROM domain__%d d WHERE
d.uuid = d2.uuid
)`,
p.TLD.ID,
p.TLD.ID,
),
)
if err != nil {
return err
}
for rtName, rtID := range p.recordTypes {
err = p.recordInsert.Exec(fmt.Sprintf(
"DROP INDEX IF EXISTS record__%d_%d_idx",
rtID,
p.TLD.ID,
))
if err != nil {
return err
}
err = p.Update("Merge temp record with record (type: %s).", rtName)
if err != nil {
return err
}
err = p.recordInsert.Merge(
fmt.Sprintf(`
INSERT INTO record__%d_%d
SELECT DISTINCT ON (uuid) *, %d FROM %%s r2
WHERE NOT EXISTS (
SELECT NULL FROM record__%d_%d r WHERE
r.uuid = r2.uuid AND r.record_type = %d
) AND r2.record_type = %d`,
rtID,
p.TLD.ID,
p.ID,
rtID,
p.TLD.ID,
rtID,
rtID,
),
)
if err != nil {
return err
}
err = p.recordInsert.Exec(fmt.Sprintf(
"CREATE INDEX record__%d_%d_idx ON record__%d_%d USING HASH (domain)",
rtID,
p.TLD.ID,
rtID,
p.TLD.ID,
))
if err != nil {
return err
}
}
err = p.Update("Commit domain insert.")
if err != nil {
return err
}
p.domainInsert.Finish()
// TODO: add index to record__%d_%d
err = p.Update("Commit record insert.")
if err != nil {
return err
}
p.recordInsert.Finish()
log.Info("Parse %s complete", p.String())
return nil
}
/*
Search is used for retrieving domain records. It accepts the following GET params:
+ uuid - uuid of domain
+ name - string representation of domain (can include wildcards)
+ email - domains attached to an email
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var where []string
var args []interface{}
i := 1
for k, _ := range params {
switch k {
case "uuid":
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, params.Get(k))
case "email":
// special case which overrides results
domainList, err := db.GetByJoinWhoisEmails(params.Get(k)).List()
util.ToJSON(domainList, err, w)
return
case "organization":
domainList, err := db.GetByJoinWhoisOrganizations(params.Get(k)).List()
util.ToJSON(domainList, err, w)
return
case "person_name":
domainList, err := db.GetByJoinWhoisNames(params.Get(k)).List()
util.ToJSON(domainList, err, w)
return
case "phone":
domainList, err := db.GetByJoinWhoisPhones(params.Get(k)).List()
util.ToJSON(domainList, err, w)
return
case "postcode":
domainList, err := db.GetByJoinWhoisPostcodes(params.Get(k)).List()
util.ToJSON(domainList, err, w)
return
case "name":
name := params.Get(k)
if strings.ContainsAny(name, ".") {
// attempt to detect tld
tld, err := tlds.Detect(name)
if err == nil {
where = append(where, fmt.Sprintf("tld = $%d", i))
args = append(args, tld.ID)
// strip tld from name
name = strings.TrimSuffix(name, "."+tld.Name)
}
}
if strings.ContainsAny(name, "* || % || ") {
// contains wildcards
name = strings.Replace(name, "*", "%", -1)
name = strings.Replace(name, " ", "%", -1)
}
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, name)
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
domainList, err := db.GetList(query, args...)
util.ToJSON(domainList, err, w)
}
/*
Search is used for retrieving whois records for a domain. It accepts the following
GET params:
+ uuid - uuid of record
+ duuid - uuid of domain
+ domain - string of domain
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var domain domains.Domain
var where []string
var args []interface{}
var orderBy string
i := 1
for k, _ := range params {
switch k {
case "orderBy":
orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k))
if v, ok := params["order"]; ok {
order := strings.ToLower(v[0])
if order == "desc" {
orderBy += "DESC "
} else if order == "asc" {
orderBy += "ASC "
}
}
case "uuid":
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, params.Get(k))
i++
case "duuid", "domain":
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, params.Get(k))
v := params.Get(k)
params.Del("duuid")
params.Del("domain")
params.Set("domain", v)
i++
case "name":
name, tld, err := tlds.DetectDomainAndTLD(params.Get(k))
if err != nil {
util.Error(err, w)
return
}
domain, err = domains.GetByNameAndTLD(name, tld).One()
if err != nil {
domain = domains.New(name, tld)
err = domain.Insert()
if err != nil {
util.Error(err, w)
return
}
}
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, domain.UUID.String())
i++
case "email":
where = append(where, fmt.Sprintf("emails ? $%d", i))
args = append(args, params.Get(k))
i++
case "raw":
where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i))
args = append(args, "%"+params.Get(k)+"%")
i++
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
if len(orderBy) > 0 {
query += orderBy + " "
}
query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
recordList, err := db.GetList(query, args...)
if err != nil {
panic(err)
}
// check for non sql errors
// if we have no results dispatch a worker to get one
if len(recordList) == 0 {
if len(domain.Name) == 0 {
// no domain lets grab one using what we assume is a duuid
if duuid := params.Get("domain"); duuid != "" {
log.Info("duuid: " + duuid)
domain, err = domains.GetByUUID(duuid).One()
if err != nil {
util.Error(err, w)
return
}
}
}
if len(domain.Name) == 0 {
log.Error("Unable to detect domain for whois lookup, params: %+v", params)
util.Error(errors.New("Unable to find domain for whois lookup."), w)
return
}
result := <-dispatcher.AddDomain(domain)
recordList = append(recordList, result)
}
util.ToJSON(recordList, err, w)
}