func (c Crawler) Start() {
go func() {
for {
select {
case <-c.quit:
log.Info("Quit crawler.")
return
default:
domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List()
if err != nil {
log.Error("Crawler. Unable to get domains: %s", err)
// shutdown
c.Stop()
return
}
for _, d := range domainList {
whoisDispatcher.AddDomain(d)
digDispatcher.AddDomain(d)
time.Sleep(c.delay)
}
}
c.offset += WINDOW
}
}()
}
func Create(w http.ResponseWriter, r *http.Request) {
var post struct {
Domain string `json:"domain"`
Name string `json:"name"`
Interval string `json:"interval"`
}
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(&post)
if err != nil {
util.Error(err, w)
return
}
var domain domains.Domain
if len(post.Name) > 0 {
name, tld, err := tlds.DetectDomainAndTLD(post.Name)
if err != nil {
util.Error(err, w)
return
}
domain, err = domains.GetByNameAndTLD(name, tld).One()
if err != nil {
domain = domains.New(name, tld)
err = domain.Insert()
if err != nil {
util.Error(err, w)
return
}
}
} else {
domain, err = domains.GetByUUID(post.Domain).One()
if err != nil {
util.Error(err, w)
return
}
}
watcher, err := db.GetByDomain(domain).One()
if err != nil {
watcher, err = db.New(domain, post.Interval)
if err != nil {
util.Error(err, w)
return
}
}
watcher.SetLowerInterval(post.Interval)
user, err := auth.GetUser(r)
if err != nil {
util.Error(err, w)
return
}
watcher.AddUser(*user)
err = watcher.Save()
log.Info("%+v", watcher)
dispatcher.AddDomain(domain)
util.ToJSON(watcher, err, w)
}
/*
Search is used for retrieving whois records for a domain. It accepts the following
GET params:
+ uuid - uuid of record
+ duuid - uuid of domain
+ domain - string of domain
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
query := db.SELECT
var domain domains.Domain
var where []string
var args []interface{}
var orderBy string
i := 1
for k, _ := range params {
switch k {
case "orderBy":
orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k))
if v, ok := params["order"]; ok {
order := strings.ToLower(v[0])
if order == "desc" {
orderBy += "DESC "
} else if order == "asc" {
orderBy += "ASC "
}
}
case "uuid":
where = append(where, fmt.Sprintf(k+" = $%d", i))
args = append(args, params.Get(k))
i++
case "duuid", "domain":
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, params.Get(k))
v := params.Get(k)
params.Del("duuid")
params.Del("domain")
params.Set("domain", v)
i++
case "name":
name, tld, err := tlds.DetectDomainAndTLD(params.Get(k))
if err != nil {
util.Error(err, w)
return
}
domain, err = domains.GetByNameAndTLD(name, tld).One()
if err != nil {
domain = domains.New(name, tld)
err = domain.Insert()
if err != nil {
util.Error(err, w)
return
}
}
where = append(where, fmt.Sprintf("domain = $%d", i))
args = append(args, domain.UUID.String())
i++
case "email":
where = append(where, fmt.Sprintf("emails ? $%d", i))
args = append(args, params.Get(k))
i++
case "raw":
where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i))
args = append(args, "%"+params.Get(k)+"%")
i++
}
}
if len(where) > 0 {
query += "WHERE " + strings.Join(where, " AND ") + " "
}
if len(orderBy) > 0 {
query += orderBy + " "
}
query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
log.Info("Query: " + query)
log.Info("Args: %+v", args)
recordList, err := db.GetList(query, args...)
if err != nil {
panic(err)
}
// check for non sql errors
// if we have no results dispatch a worker to get one
if len(recordList) == 0 {
if len(domain.Name) == 0 {
// no domain lets grab one using what we assume is a duuid
if duuid := params.Get("domain"); duuid != "" {
log.Info("duuid: " + duuid)
domain, err = domains.GetByUUID(duuid).One()
if err != nil {
util.Error(err, w)
return
}
}
}
if len(domain.Name) == 0 {
log.Error("Unable to detect domain for whois lookup, params: %+v", params)
util.Error(errors.New("Unable to find domain for whois lookup."), w)
return
}
result := <-dispatcher.AddDomain(domain)
recordList = append(recordList, result)
}
util.ToJSON(recordList, err, w)
}