Пример #1
0
func (c Crawler) Start() {
	go func() {
		for {
			select {
			case <-c.quit:
				log.Info("Quit crawler.")
				return
			default:
				domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List()
				if err != nil {
					log.Error("Crawler. Unable to get domains: %s", err)
					// shutdown
					c.Stop()
					return
				}
				for _, d := range domainList {
					whoisDispatcher.AddDomain(d)
					digDispatcher.AddDomain(d)
					time.Sleep(c.delay)
				}
			}
			c.offset += WINDOW
		}
	}()
}
Пример #2
0
func Create(w http.ResponseWriter, r *http.Request) {
	var post struct {
		Domain   string `json:"domain"`
		Name     string `json:"name"`
		Interval string `json:"interval"`
	}
	decoder := json.NewDecoder(r.Body)
	err := decoder.Decode(&post)
	if err != nil {
		util.Error(err, w)
		return
	}
	var domain domains.Domain
	if len(post.Name) > 0 {
		name, tld, err := tlds.DetectDomainAndTLD(post.Name)
		if err != nil {
			util.Error(err, w)
			return
		}
		domain, err = domains.GetByNameAndTLD(name, tld).One()
		if err != nil {
			domain = domains.New(name, tld)
			err = domain.Insert()
			if err != nil {
				util.Error(err, w)
				return
			}
		}
	} else {
		domain, err = domains.GetByUUID(post.Domain).One()
		if err != nil {
			util.Error(err, w)
			return
		}
	}
	watcher, err := db.GetByDomain(domain).One()
	if err != nil {
		watcher, err = db.New(domain, post.Interval)
		if err != nil {
			util.Error(err, w)
			return
		}
	}
	watcher.SetLowerInterval(post.Interval)
	user, err := auth.GetUser(r)
	if err != nil {
		util.Error(err, w)
		return
	}
	watcher.AddUser(*user)
	err = watcher.Save()
	log.Info("%+v", watcher)
	dispatcher.AddDomain(domain)
	util.ToJSON(watcher, err, w)
}
Пример #3
0
/*
	Search is used for retrieving whois records for a domain. It accepts the following
	GET params:

	+ uuid		- uuid of record
	+ duuid 	- uuid of domain
	+ domain	- string of domain

*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var domain domains.Domain
	var where []string
	var args []interface{}
	var orderBy string
	i := 1
	for k, _ := range params {
		switch k {
		case "orderBy":
			orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k))
			if v, ok := params["order"]; ok {
				order := strings.ToLower(v[0])
				if order == "desc" {
					orderBy += "DESC "
				} else if order == "asc" {
					orderBy += "ASC "
				}
			}
		case "uuid":
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, params.Get(k))
			i++
		case "duuid", "domain":
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, params.Get(k))
			v := params.Get(k)
			params.Del("duuid")
			params.Del("domain")
			params.Set("domain", v)
			i++
		case "name":
			name, tld, err := tlds.DetectDomainAndTLD(params.Get(k))
			if err != nil {
				util.Error(err, w)
				return
			}
			domain, err = domains.GetByNameAndTLD(name, tld).One()
			if err != nil {
				domain = domains.New(name, tld)
				err = domain.Insert()
				if err != nil {
					util.Error(err, w)
					return
				}
			}
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, domain.UUID.String())
			i++
		case "email":
			where = append(where, fmt.Sprintf("emails ? $%d", i))
			args = append(args, params.Get(k))
			i++
		case "raw":
			where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i))
			args = append(args, "%"+params.Get(k)+"%")
			i++
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	if len(orderBy) > 0 {
		query += orderBy + " "
	}
	query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	recordList, err := db.GetList(query, args...)
	if err != nil {
		panic(err)
	}
	// check for non sql errors
	// if we have no results dispatch a worker to get one
	if len(recordList) == 0 {
		if len(domain.Name) == 0 {
			// no domain lets grab one using what we assume is a duuid
			if duuid := params.Get("domain"); duuid != "" {
				log.Info("duuid: " + duuid)
				domain, err = domains.GetByUUID(duuid).One()
				if err != nil {
					util.Error(err, w)
					return
				}
			}
		}
		if len(domain.Name) == 0 {
			log.Error("Unable to detect domain for whois lookup, params: %+v", params)
			util.Error(errors.New("Unable to find domain for whois lookup."), w)
			return
		}
		result := <-dispatcher.AddDomain(domain)
		recordList = append(recordList, result)
	}
	util.ToJSON(recordList, err, w)
}