Exemple #1
0
/*
	Search is used for retrieving watchers. It accepts the following
	GET params:

	+ duuid 	- uuid of domain
	+ domain	- string of domain
	+ name		- string of name

*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var where []string
	var args []interface{}
	i := 1
	for k, _ := range params {
		switch k {
		case "user":
			user, err := auth.GetUser(r)
			if err != nil {
				util.Error(err, w)
				return
			}
			where = append(where, fmt.Sprintf("users@> '[%d]'", user.ID))
			i++
		case "name":
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, params.Get(k))
			i++
		case "duuid", "domain":
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, params.Get(k))
			i++
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	watcherList, err := db.GetList(query, args...)
	util.ToJSON(watcherList, err, w)
}
Exemple #2
0
func testDetectDomainAndTLD() {
	s, t, err := tlds.DetectDomainAndTLD("ns1.google.co.uk")
	if err != nil {
		log.Error("%s", err)
		return
	}
	log.Info("%+v", t)
	log.Info("%+v", s)
	s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.ng")
	if err != nil {
		log.Error("%s", err)
		return
	}
	log.Info("%+v", t)
	log.Info("%+v", s)
	s, t, err = tlds.DetectDomainAndTLD("ns1.google.co.com")
	if err != nil {
		log.Error("%s", err)
		return
	}
	log.Info("%+v", t)
	log.Info("%+v", s)
	s, t, err = tlds.DetectDomainAndTLD("ns1.google.foo")
	if err != nil {
		log.Error("%s", err)
		return
	}
	log.Info("%+v", t)
	log.Info("%+v", s)

}
Exemple #3
0
func (c Crawler) Start() {
	go func() {
		for {
			select {
			case <-c.quit:
				log.Info("Quit crawler.")
				return
			default:
				domainList, err := domains.GetAllLimitOffset(WINDOW, c.offset).List()
				if err != nil {
					log.Error("Crawler. Unable to get domains: %s", err)
					// shutdown
					c.Stop()
					return
				}
				for _, d := range domainList {
					whoisDispatcher.AddDomain(d)
					digDispatcher.AddDomain(d)
					time.Sleep(c.delay)
				}
			}
			c.offset += WINDOW
		}
	}()
}
Exemple #4
0
/*
	Search is used for retrieving notifications. It accepts the following
	GET params:


*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var where []string
	var args []interface{}
	var orderBy = ""
	i := 1
	// inject user
	user, err := auth.GetUser(r)
	if err != nil {
		util.Error(err, w)
		return
	}
	where = append(where, fmt.Sprintf("user_id = $%d", i))
	args = append(args, user.ID)
	i++
	for k, _ := range params {
		switch k {
		case "duuid":
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, params.Get(k))
			i++
		case "orderBy":
			orderBy = fmt.Sprintf("ORDER BY $%d ", i)
			args = append(args, params.Get(k))
			if v, ok := params["order"]; ok {
				order := strings.ToLower(v[0])
				if order == "desc" {
					orderBy += "DESC "
				} else if order == "asc" {
					orderBy += "ASC "
				}
			}
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	if len(orderBy) > 0 {
		query += orderBy + " "
	}
	query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	notificationList, err := db.GetList(query, args...)
	util.ToJSON(notificationList, err, w)
}
Exemple #5
0
func init() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	go func() {
		<-c
		log.Info("Shutting down")
		os.Exit(1)
	}()
}
Exemple #6
0
func Create(w http.ResponseWriter, r *http.Request) {
	var post struct {
		Domain   string `json:"domain"`
		Name     string `json:"name"`
		Interval string `json:"interval"`
	}
	decoder := json.NewDecoder(r.Body)
	err := decoder.Decode(&post)
	if err != nil {
		util.Error(err, w)
		return
	}
	var domain domains.Domain
	if len(post.Name) > 0 {
		name, tld, err := tlds.DetectDomainAndTLD(post.Name)
		if err != nil {
			util.Error(err, w)
			return
		}
		domain, err = domains.GetByNameAndTLD(name, tld).One()
		if err != nil {
			domain = domains.New(name, tld)
			err = domain.Insert()
			if err != nil {
				util.Error(err, w)
				return
			}
		}
	} else {
		domain, err = domains.GetByUUID(post.Domain).One()
		if err != nil {
			util.Error(err, w)
			return
		}
	}
	watcher, err := db.GetByDomain(domain).One()
	if err != nil {
		watcher, err = db.New(domain, post.Interval)
		if err != nil {
			util.Error(err, w)
			return
		}
	}
	watcher.SetLowerInterval(post.Interval)
	user, err := auth.GetUser(r)
	if err != nil {
		util.Error(err, w)
		return
	}
	watcher.AddUser(*user)
	err = watcher.Save()
	log.Info("%+v", watcher)
	dispatcher.AddDomain(domain)
	util.ToJSON(watcher, err, w)
}
Exemple #7
0
func main() {
	w, err := watcher.New()
	if err != nil {
		log.Error("Unable to start Watcher: %s", err)
		return
	}
	w.Start()
	log.Info("Watcher started.")
	q := make(chan bool)
	<-q
}
Exemple #8
0
func (w Watcher) handler(i intervals.Interval) {
	list, err := db.GetByInterval(i).List()
	if err != nil {
		log.Error("Error parsing interval (%d): %s", i.ID, err)
		return
	}
	for _, watch := range list {
		w.DigParser.Exec(watch.Domain)
		before, err := whois.GetByDomain(watch.Domain).Count()
		if err != nil {
			log.Error("%s", err)
		}
		w.WhoisParser.Exec(watch.Domain)
		after, err := whois.GetByDomain(watch.Domain).Count()
		if err != nil {
			log.Error("%s", err)
		}
		if after > before && before > 0 {
			log.Info("%s whois changed", watch.Domain)
			message := notifications.Message{
				Added:   time.Now(),
				Message: "Domain Whois update.",
				Domain:  watch.Domain,
			}
			for _, user := range watch.Users {
				log.Info("%s", user.Email)
				noter, _ := notifications.SetupUserNotification(user)
				noter.AddMessage(message)
			}
		}
		err = watch.Save()
		if err != nil {
			log.Error("%s", err)
		}

	}
}
Exemple #9
0
/*
	Search is used for retrieving records for a domain. It accepts the following
	GET params:

	+ duuid 	- uuid of domain
	+ domain	- string of domain
	+ name		- string of name

*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var where []string
	var args []interface{}
	i := 1
	for k, _ := range params {
		switch k {
		case "name":
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, params.Get(k))
			i++
		case "duuid", "domain":
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, params.Get(k))
			v := params.Get(k)
			params.Del("duuid")
			params.Del("domain")
			params.Set("domain", v)
			i++
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	recordList, err := db.GetList(query, args...)
	if err != nil {
		util.Error(err, w)
		return
	}
	// check for none sql errors
	// if we have no results dispatch a worker to get one
	if len(recordList) == 0 {
		// no domain lets grab one using what we assume is a duuid
		log.Info("no records")
		if duuid := params.Get("domain"); duuid != "" {
			log.Info("duuid: " + duuid)
			domain, err := domains.GetByUUID(duuid).One()
			if err != nil {
				log.Info("herererere")
				util.Error(err, w)
				return
			}
			results := <-dispatcher.AddDomain(domain)
			log.Info("%v", results)
			for _, result := range results {
				recordList = append(recordList, result)
			}
		}
	}
	util.ToJSON(cleanParserFromRecords(recordList), err, w)
}
Exemple #10
0
func (p *Parser) Parse() error {
	defer p.Close()
	defer util.Un(util.Trace("zonefile parser"))
	ri, err := records.NewBulkInsert()
	if err != nil {
		return err
	}
	p.recordInsert = &ri
	bi, err := domains.NewBulkInsert()
	if err != nil {
		return err
	}
	p.recordTypes = make(map[string]int32)
	p.domainInsert = &bi
	log.Info("Parsing Zonefile: %s", p.String())
	err = p.Update("Load file in to temporary tables.")
	if err != nil {
		log.Error("%s", err)
		return err
	}
	var previous string
	p.lineCount = 0
	func() {
		defer util.Un(util.Trace("zonefile parser scan file"))
		for p.scanner.Scan() {
			p.lineCount++
			p.line = strings.ToLower(p.scanner.Text())
			line := p.line
			commentIdx := strings.Index(line, ";")
			if commentIdx > 0 {
				//comment := line[commentIdx:]
				line = line[:commentIdx]
			}
			if len(line) == 0 {
				continue
			}
			switch line[0] {
			case ';':
				p.handleComment(line)
			case '@':
				// need to wdd more ways of detecting SOA, could have a switch
				// that only goes to handleLine when it has parsed $origin var
				p.handleSOA(line)
			case '$':
				p.handleVariable(line)
			case ' ':
			case '	':
				p.handleZonedLine(line, previous)
			default:
				if !p.originCheck {
					p.handleSOA(line)
				} else {
					p.handleLine(line)
				}
			}
			previous = line
		}
		err = p.Update("File read in to temporary tables.")
		if err != nil {
			log.Error("Unable to read in temporary tables: %s", err)
			return
		}
		log.Info("Parse %s complete. Proceed with sql operations.", p.String())
	}()
	// insert our domains and commit our tx to avoid
	err = p.Update("Close insert domains statement.")
	if err != nil {
		return err
	}
	p.domainInsert.Insert()
	err = p.Update("Close insert records statement.")
	if err != nil {
		return err
	}
	p.recordInsert.Insert()
	// TODO: drop index to record__%d_%d
	// create index's
	err = p.Update("Create temp domain table index.")
	if err != nil {
		return err
	}

	err = p.domainInsert.Index(fmt.Sprintf("CREATE INDEX _domain__%d_uuid_idx ON %%s (uuid)", p.TLD.ID))
	if err != nil {
		return err
	}
	err = p.Update("Create temp record table index.")
	err = p.recordInsert.Index("CREATE INDEX uuid_idx ON %s (uuid)")
	if err != nil {
		return err
	}
	err = p.Update("Merge temp domain with domain.")
	err = p.domainInsert.Merge(
		fmt.Sprintf(`
			INSERT INTO domain__%d
			SELECT DISTINCT * FROM %%s d2
				WHERE NOT EXISTS (
					SELECT NULL FROM domain__%d d WHERE
						d.uuid = d2.uuid
				)`,
			p.TLD.ID,
			p.TLD.ID,
		),
	)
	if err != nil {
		return err
	}
	for rtName, rtID := range p.recordTypes {
		err = p.recordInsert.Exec(fmt.Sprintf(
			"DROP INDEX IF EXISTS record__%d_%d_idx",
			rtID,
			p.TLD.ID,
		))
		if err != nil {
			return err
		}
		err = p.Update("Merge temp record with record (type: %s).", rtName)
		if err != nil {
			return err
		}
		err = p.recordInsert.Merge(
			fmt.Sprintf(`
				INSERT INTO record__%d_%d
				SELECT DISTINCT ON (uuid) *, %d FROM %%s r2
					WHERE NOT EXISTS (
						SELECT NULL FROM record__%d_%d r WHERE
							r.uuid = r2.uuid AND r.record_type = %d
					) AND r2.record_type = %d`,
				rtID,
				p.TLD.ID,
				p.ID,
				rtID,
				p.TLD.ID,
				rtID,
				rtID,
			),
		)
		if err != nil {
			return err
		}
		err = p.recordInsert.Exec(fmt.Sprintf(
			"CREATE INDEX record__%d_%d_idx ON record__%d_%d USING HASH (domain)",
			rtID,
			p.TLD.ID,
			rtID,
			p.TLD.ID,
		))
		if err != nil {
			return err
		}
	}
	err = p.Update("Commit domain insert.")
	if err != nil {
		return err
	}
	p.domainInsert.Finish()
	// TODO: add index to record__%d_%d
	err = p.Update("Commit record insert.")
	if err != nil {
		return err
	}
	p.recordInsert.Finish()
	log.Info("Parse %s complete", p.String())
	return nil
}
Exemple #11
0
/*
	Search is used for retrieving domain records. It accepts the following GET params:

	+ uuid		- uuid of domain
	+ name		- string representation of domain (can include wildcards)
	+ email		- domains attached to an email
*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var where []string
	var args []interface{}
	i := 1
	for k, _ := range params {
		switch k {
		case "uuid":
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, params.Get(k))
		case "email":
			// special case which overrides results
			domainList, err := db.GetByJoinWhoisEmails(params.Get(k)).List()
			util.ToJSON(domainList, err, w)
			return
		case "organization":
			domainList, err := db.GetByJoinWhoisOrganizations(params.Get(k)).List()
			util.ToJSON(domainList, err, w)
			return
		case "person_name":
			domainList, err := db.GetByJoinWhoisNames(params.Get(k)).List()
			util.ToJSON(domainList, err, w)
			return
		case "phone":
			domainList, err := db.GetByJoinWhoisPhones(params.Get(k)).List()
			util.ToJSON(domainList, err, w)
			return
		case "postcode":
			domainList, err := db.GetByJoinWhoisPostcodes(params.Get(k)).List()
			util.ToJSON(domainList, err, w)
			return
		case "name":
			name := params.Get(k)
			if strings.ContainsAny(name, ".") {
				// attempt to detect tld
				tld, err := tlds.Detect(name)
				if err == nil {
					where = append(where, fmt.Sprintf("tld = $%d", i))
					args = append(args, tld.ID)
					// strip tld from name
					name = strings.TrimSuffix(name, "."+tld.Name)
				}
			}
			if strings.ContainsAny(name, "* || % ||  ") {
				// contains wildcards
				name = strings.Replace(name, "*", "%", -1)
				name = strings.Replace(name, " ", "%", -1)
			}
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, name)
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	query += fmt.Sprintf("LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	domainList, err := db.GetList(query, args...)
	util.ToJSON(domainList, err, w)
}
Exemple #12
0
/*
	Search is used for retrieving whois records for a domain. It accepts the following
	GET params:

	+ uuid		- uuid of record
	+ duuid 	- uuid of domain
	+ domain	- string of domain

*/
func Search(w http.ResponseWriter, r *http.Request, params url.Values, limit, offset int) {
	query := db.SELECT
	var domain domains.Domain
	var where []string
	var args []interface{}
	var orderBy string
	i := 1
	for k, _ := range params {
		switch k {
		case "orderBy":
			orderBy = fmt.Sprintf("ORDER BY %s ", params.Get(k))
			if v, ok := params["order"]; ok {
				order := strings.ToLower(v[0])
				if order == "desc" {
					orderBy += "DESC "
				} else if order == "asc" {
					orderBy += "ASC "
				}
			}
		case "uuid":
			where = append(where, fmt.Sprintf(k+" = $%d", i))
			args = append(args, params.Get(k))
			i++
		case "duuid", "domain":
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, params.Get(k))
			v := params.Get(k)
			params.Del("duuid")
			params.Del("domain")
			params.Set("domain", v)
			i++
		case "name":
			name, tld, err := tlds.DetectDomainAndTLD(params.Get(k))
			if err != nil {
				util.Error(err, w)
				return
			}
			domain, err = domains.GetByNameAndTLD(name, tld).One()
			if err != nil {
				domain = domains.New(name, tld)
				err = domain.Insert()
				if err != nil {
					util.Error(err, w)
					return
				}
			}
			where = append(where, fmt.Sprintf("domain = $%d", i))
			args = append(args, domain.UUID.String())
			i++
		case "email":
			where = append(where, fmt.Sprintf("emails ? $%d", i))
			args = append(args, params.Get(k))
			i++
		case "raw":
			where = append(where, fmt.Sprintf("raw_whois ->>0 ILIKE $%d", i))
			args = append(args, "%"+params.Get(k)+"%")
			i++
		}
	}
	if len(where) > 0 {
		query += "WHERE " + strings.Join(where, " AND ") + " "
	}
	if len(orderBy) > 0 {
		query += orderBy + " "
	}
	query += fmt.Sprintf("ORDER BY added DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
	args = append(args, limit, offset)
	log.Info("Query: " + query)
	log.Info("Args: %+v", args)
	recordList, err := db.GetList(query, args...)
	if err != nil {
		panic(err)
	}
	// check for non sql errors
	// if we have no results dispatch a worker to get one
	if len(recordList) == 0 {
		if len(domain.Name) == 0 {
			// no domain lets grab one using what we assume is a duuid
			if duuid := params.Get("domain"); duuid != "" {
				log.Info("duuid: " + duuid)
				domain, err = domains.GetByUUID(duuid).One()
				if err != nil {
					util.Error(err, w)
					return
				}
			}
		}
		if len(domain.Name) == 0 {
			log.Error("Unable to detect domain for whois lookup, params: %+v", params)
			util.Error(errors.New("Unable to find domain for whois lookup."), w)
			return
		}
		result := <-dispatcher.AddDomain(domain)
		recordList = append(recordList, result)
	}
	util.ToJSON(recordList, err, w)
}