// Function to mock a domain
func generateAndSaveDomain(fqdn string, domainDAO dao.DomainDAO, language string) {
lastOKAt := time.Now().Add(time.Duration(-config.ShelterConfig.Notification.NameserverErrorAlertDays*24) * time.Hour)
owner, _ := mail.ParseAddress("*****@*****.**")
domain := model.Domain{
FQDN: fqdn,
Nameservers: []model.Nameserver{
{
Host: fmt.Sprintf("ns1.%s", fqdn),
IPv4: net.ParseIP("127.0.0.1"),
LastStatus: model.NameserverStatusServerFailure,
LastOKAt: lastOKAt,
},
},
Owners: []model.Owner{
{
Email: owner,
Language: language,
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln(fmt.Sprintf("Fail to save domain %s", domain.FQDN), err)
}
}
// Generates a report with the amount of time of a scan
func scanReport(domainDAO dao.DomainDAO, scanDAO dao.ScanDAO, scanConfig ScanTestConfigFile) {
report := " # | Total | DPS | Memory (MB)\n" +
"-----------------------------------------------------\n"
// Report variables
scale := []int{10, 50, 100, 500, 1000, 5000,
10000, 50000, 100000, 500000, 1000000, 5000000}
dnskey, privateKey, err := utils.GenerateKey()
if err != nil {
utils.Fatalln("Error generating DNSKEY", err)
}
reportHandler := ReportHandler{
DNSKEY: dnskey,
PrivateKey: privateKey,
}
server.Handler = reportHandler
dns.DefaultServeMux = nil
for _, numberOfItems := range scale {
utils.Println(fmt.Sprintf("Generating report - scale %d", numberOfItems))
for i := 0; i < numberOfItems; i++ {
if i%1000 == 0 {
utils.PrintProgress("building scenario", (i*100)/numberOfItems)
}
fqdn := fmt.Sprintf("domain%d.br.", i)
generateAndSaveDomain(fqdn, domainDAO, dnskey)
}
utils.PrintProgress("building scenario", 100)
totalDuration, domainsPerSecond := calculateScanDurations(numberOfItems, scanDAO)
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
report += fmt.Sprintf("% -8d | %16s | %4d | %14.2f\n",
numberOfItems,
time.Duration(int64(totalDuration)).String(),
domainsPerSecond,
float64(memStats.Alloc)/float64(MB),
)
if err := domainDAO.RemoveAll(); err != nil {
// When the result set is too big to remove, we got a timeout error from the
// connection, but it's ok
//utils.Fatalln("Error removing domains generated for report", err)
}
}
utils.WriteReport(scanConfig.Report.File, report)
}
// Function created to remove all entries from the database to ensure that the tests
// enviroments are always equal
func ClearDatabase(database *mgo.Database) {
domainDAO := dao.DomainDAO{
Database: database,
}
domainDAO.RemoveAll()
scanDAO := dao.ScanDAO{
Database: database,
}
scanDAO.RemoveAll()
}
func main() {
flag.Parse()
var config DomainDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
domainLifeCycle(domainDAO)
domainsLifeCycle(domainDAO)
domainUniqueFQDN(domainDAO)
domainConcurrency(domainDAO)
domainsPagination(domainDAO)
domainsNotification(domainDAO)
domainsExpand(domainDAO)
domainFilter(domainDAO)
// Domain DAO performance report is optional and only generated when the report file
// path parameter is given
if report {
domainDAOPerformanceReport(config.Report.ReportFile, domainDAO)
}
utils.Println("SUCCESS!")
}
// Function to mock a domain
func generateSignAndSaveDomain(fqdn string, domainDAO dao.DomainDAO) (
model.Domain, *dns.DNSKEY, *dns.RRSIG, time.Time, time.Time,
) {
domain, dnskey, rrsig, lastCheckAt, lastOKAt := generateAndSignDomain(fqdn)
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain", err)
}
return domain, dnskey, rrsig, lastCheckAt, lastOKAt
}
// Function to mock a domain
func generateAndSaveDomain(fqdn string, domainDAO dao.DomainDAO, dnskey *dns.DNSKEY) {
ds := dnskey.ToDS(uint8(model.DSDigestTypeSHA1))
domain := model.Domain{
FQDN: fqdn,
Nameservers: []model.Nameserver{
{
Host: fmt.Sprintf("ns1.%s", fqdn),
IPv4: net.ParseIP("127.0.0.1"),
},
},
DSSet: []model.DS{
{
Keytag: dnskey.KeyTag(),
Algorithm: utils.ConvertKeyAlgorithm(dnskey.Algorithm),
DigestType: model.DSDigestTypeSHA1,
Digest: ds.Digest,
},
},
}
owner, _ := mail.ParseAddress("*****@*****.**")
domain.Owners = []model.Owner{
{
Email: owner,
Language: "pt-BR",
},
}
lastCheckAt := time.Now().Add(-72 * time.Hour)
lastOKAt := lastCheckAt.Add(-24 * time.Hour)
// Set all nameservers with error and the last check equal of the error check interval,
// this will force the domain to be checked
for index, _ := range domain.Nameservers {
domain.Nameservers[index].LastCheckAt = lastCheckAt
domain.Nameservers[index].LastOKAt = lastOKAt
domain.Nameservers[index].LastStatus = model.NameserverStatusServerFailure
}
// Set all DS records with error and the last check equal of the error check interval,
// this will force the domain to be checked
for index, _ := range domain.DSSet {
domain.DSSet[index].LastCheckAt = lastCheckAt
domain.DSSet[index].LastOKAt = lastOKAt
domain.DSSet[index].LastStatus = model.DSStatusTimeout
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln(fmt.Sprintf("Fail to save domain %s", domain.FQDN), err)
}
}
func (h *DomainHandler) Delete(w http.ResponseWriter, r *http.Request) {
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if err := domainDAO.Remove(&h.domain); err != nil {
log.Println("Error while removing domain object. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (i *Domain) Before(w http.ResponseWriter, r *http.Request) {
domainDAO := dao.DomainDAO{
Database: i.domainHandler.GetDatabase(),
}
domain, err := domainDAO.FindByFQDN(i.domainHandler.GetFQDN())
// For PUT method if the domain does not exist yet thats alright because we will create
// it
if r.Method != "PUT" && err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
i.domainHandler.SetDomain(domain)
}
func main() {
flag.Parse()
var restConfig RESTHandlerDomainVerificationTestConfigFile
err := utils.ReadConfigFile(configFilePath, &restConfig)
config.ShelterConfig = restConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
restConfig.Database.URIs,
restConfig.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
utils.StartDNSServer(restConfig.DNSServerPort, restConfig.Scan.UDPMaxSize)
scanDomain()
scanPersistedDomain(domainDAO)
queryDomain()
utils.Println("SUCCESS!")
}
func domainWithNoErrors(config ScanInjectorTestConfigFile, domainDAO dao.DomainDAO) {
domain := newDomain()
// Set all nameservers as configured correctly and the last check as now, this domain is
// unlikely to be selected
for index, _ := range domain.Nameservers {
domain.Nameservers[index].LastCheckAt = time.Now()
domain.Nameservers[index].LastStatus = model.NameserverStatusOK
}
// Set all DS records as configured correctly and the last check as now, this domain is
// unlikely to be selected
for index, _ := range domain.DSSet {
domain.DSSet[index].LastCheckAt = time.Now()
domain.DSSet[index].LastStatus = model.DSStatusOK
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain for scan scenario", err)
}
model.StartNewScan()
if domains := runScan(config, domainDAO); len(domains) > 0 {
utils.Fatalln(fmt.Sprintf("Selected a domain configured correctly for the scan. "+
"Expected 0 got %d", len(domains)), nil)
}
currentScan := model.GetCurrentScan()
if currentScan.Status != model.ScanStatusRunning {
utils.Fatalln("Not changing the scan info status for domain with no errors", nil)
}
if currentScan.DomainsToBeScanned > 0 {
utils.Fatalln("Not counting the domains to be scanned for domain with no errors", nil)
}
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Error removing domain", err)
}
}
func domainWithDNSErrors(config ScanInjectorTestConfigFile, domainDAO dao.DomainDAO) {
domain := newDomain()
// Set all nameservers with error and the last check equal of the error check interval,
// this will force the domain to be checked
for index, _ := range domain.Nameservers {
maxErrorHours := config.Scan.VerificationIntervals.MaxErrorDays * 24
lessThreeDays, _ := time.ParseDuration("-" + strconv.Itoa(maxErrorHours) + "h")
domain.Nameservers[index].LastCheckAt = time.Now().Add(lessThreeDays)
domain.Nameservers[index].LastStatus = model.NameserverStatusServerFailure
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain for scan scenario", err)
}
model.StartNewScan()
if domains := runScan(config, domainDAO); len(domains) != 1 {
utils.Fatalln(fmt.Sprintf("Couldn't load a domain with DNS errors for scan. "+
"Expected 1 got %d", len(domains)), nil)
}
currentScan := model.GetCurrentScan()
if currentScan.Status != model.ScanStatusRunning {
utils.Fatalln("Not changing the scan info status with DNS errors", nil)
}
if currentScan.DomainsToBeScanned != 1 {
utils.Fatalln("Not counting the domains to be scanned with DNS errors", nil)
}
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Error removing domain", err)
}
}
func domainFilter(domainDAO dao.DomainDAO) {
numberOfItems := 20
for i := 0; i < numberOfItems; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("example%d.com.br", i),
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
pagination := dao.DomainDAOPagination{
PageSize: 10,
Page: 5,
OrderBy: []dao.DomainDAOSort{
{
Field: dao.DomainDAOOrderByFieldFQDN,
Direction: dao.DAOOrderByDirectionAscending,
},
},
}
domains, err := domainDAO.FindAll(&pagination, true, "example1\\.com.*")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
if len(domains) != 1 {
utils.Fatalln(fmt.Sprintf("Wrong number of domains when there's filter. "+
"Expected '1' and got '%d'", len(domains)), nil)
}
if domains[0].FQDN != "example1.com.br" {
utils.Fatalln("Wrong domain returned", nil)
}
for i := 0; i < numberOfItems; i++ {
fqdn := fmt.Sprintf("example%d.com.br", i)
if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
utils.Fatalln("Error removing domain from database", err)
}
}
}
// FQDN must be unique in the database, today we limit this using an unique index key
func domainUniqueFQDN(domainDAO dao.DomainDAO) {
domain1 := newDomain()
// Create domain
if err := domainDAO.Save(&domain1); err != nil {
utils.Fatalln("Couldn't save domain in database", err)
}
domain2 := newDomain()
// Create another domain with the same FQDN
if err := domainDAO.Save(&domain2); err == nil {
utils.Fatalln("Allowing more than one object with the same FQDN", nil)
}
// Remove domain
if err := domainDAO.RemoveByFQDN(domain1.FQDN); err != nil {
utils.Fatalln("Error while trying to remove a domain", err)
}
}
// Check if the revision field avoid data concurrency. Is better to fail than to store the
// wrong state
func domainConcurrency(domainDAO dao.DomainDAO) {
domain := newDomain()
// Create domain
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Couldn't save domain in database", err)
}
domain1, err := domainDAO.FindByFQDN(domain.FQDN)
if err != nil {
utils.Fatalln("Couldn't find created domain in database", err)
}
domain2, err := domainDAO.FindByFQDN(domain.FQDN)
if err != nil {
utils.Fatalln("Couldn't find created domain in database", err)
}
if err := domainDAO.Save(&domain1); err != nil {
utils.Fatalln("Couldn't save domain in database", err)
}
if err := domainDAO.Save(&domain2); err == nil {
utils.Fatalln("Not controlling domain concurrency", nil)
}
// Remove domain
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Error while trying to remove a domain", err)
}
}
// Test all phases from a domain lyfe cycle, but now working with a group of domains
func domainsLifeCycle(domainDAO dao.DomainDAO) {
domains := newDomains()
// Create domains
domainResults := domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Search and compare created domains
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find created domain %s in database", domain.FQDN), err)
} else if !utils.CompareDomain(*domain, domainRetrieved) {
utils.Fatalln(fmt.Sprintf("Domain %s created is being persisted wrongly", domain.FQDN), nil)
}
}
// Update domains
for _, domain := range domains {
domain.Owners = []model.Owner{}
}
domainResults = domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't update domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Search and compare updated domains
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find updated domain %s in database", domain.FQDN), err)
} else if !utils.CompareDomain(*domain, domainRetrieved) {
utils.Fatalln(fmt.Sprintf("Domain %s updated in being persisted wrongly", domain.FQDN), nil)
}
}
// Check if find all really return all domains
allDomainsChannel, err := domainDAO.FindAllAsync()
if err != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
}
var allDomains []model.Domain
for {
domainRetrieved := <-allDomainsChannel
if domainRetrieved.Error != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
} else if domainRetrieved.Domain == nil {
break
}
allDomains = append(allDomains, *domainRetrieved.Domain)
}
if len(allDomains) != len(domains) {
utils.Fatalln(fmt.Sprintf("FindAll method is not returning all domains we expected %d but got %d",
len(domains), len(allDomains)), nil)
}
// Detected a problem in FindAsync method on 2014-01-17 where we were returning the same
// object many times because we were reusing the same pointer. For that reason we are
// going to add a test to check if the items returned are the same set of the inserted
// ones
for _, domain := range domains {
found := false
for _, domainRetrieved := range allDomains {
if domainRetrieved.Id.Hex() == domain.Id.Hex() {
found = true
break
}
}
if !found {
utils.Fatalln("FindAll method is not returning all objects "+
"that were inserted, apparently there are duplicated objects in the result set", nil)
}
}
// Remove domains
domainResults = domainDAO.RemoveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Error while trying to remove domain %s from database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Check removals
if _, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
utils.Fatalln(fmt.Sprintf("Domain %s was not removed from database", domain.FQDN), nil)
}
}
// Let's add and remove the domains again to test the remove all method
domainResults = domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if err := domainDAO.RemoveAll(); err != nil {
utils.Fatalln("Couldn't remove all domains", err)
}
allDomainsChannel, err = domainDAO.FindAllAsync()
if err != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
}
allDomains = []model.Domain{}
for {
domainRetrieved := <-allDomainsChannel
if domainRetrieved.Error != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
} else if domainRetrieved.Domain == nil {
break
}
allDomains = append(allDomains, *domainRetrieved.Domain)
}
if len(allDomains) > 0 {
utils.Fatalln("RemoveAll method is not removing the domains from the database", nil)
}
}
// Test all phases of the domain life cycle
func domainLifeCycle(domainDAO dao.DomainDAO) {
domain := newDomain()
// Create domain
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Couldn't save domain in database", err)
}
// Search and compare created domain
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Couldn't find created domain in database", err)
} else if !utils.CompareDomain(domain, domainRetrieved) {
utils.Fatalln("Domain created is being persisted wrongly", nil)
}
// Update domain
domain.Owners = []model.Owner{}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Couldn't save domain in database", err)
}
// Search and compare updated domain
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Couldn't find updated domain in database", err)
} else if !utils.CompareDomain(domain, domainRetrieved) {
utils.Fatalln("Domain updated is being persisted wrongly", nil)
}
// Remove domain
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln("Error while trying to remove a domain", err)
}
// Check removal
if _, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
utils.Fatalln("Domain was not removed from database", nil)
}
}
// Notify is responsable for selecting the domains that should be notified in the system.
// It will send alert e-mails for each owner of a domain
func Notify() {
defer func() {
// Something went really wrong while notifying the owners. Log the error stacktrace
// and move out
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Printf("Panic detected while notifying the owners. Details: %v\n%s", r, buf)
}
}()
log.Info("Start notification job")
defer func() {
log.Info("End notification job")
}()
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
log.Println("Error while initializing database. Details:", err)
return
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
config.ShelterConfig.Notification.NameserverErrorAlertDays,
config.ShelterConfig.Notification.NameserverTimeoutAlertDays,
config.ShelterConfig.Notification.DSErrorAlertDays,
config.ShelterConfig.Notification.DSTimeoutAlertDays,
// TODO: Should we move this configuration parameter to a place were both modules can
// access it. This sounds better for configuration deployment
config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
)
if err != nil {
log.Println("Error retrieving domains to notify. Details:", err)
return
}
// Dispatch the asynchronous part of the method
for {
// Get domain from the database (one-by-one)
domainResult := <-domainChannel
// Detect errors while retrieving a specific domain. We are not going to stop all the
// process when only one domain got an error
if domainResult.Error != nil {
log.Println("Error retrieving domain to notify. Details:", domainResult.Error)
continue
}
// Problem detected while retrieving a domain or we don't have domains anymore
if domainResult.Error != nil || domainResult.Domain == nil {
break
}
if err := notifyDomain(domainResult.Domain); err != nil {
log.Println("Error notifying a domain. Details:", err)
}
}
}
// The HEAD method is identical to GET except that the server MUST NOT return a message-
// body in the response. But now the responsability for don't adding the body is from the
// mux while writing the response
func (h *DomainsHandler) retrieveDomains(w http.ResponseWriter, r *http.Request) {
var pagination dao.DomainDAOPagination
expand := false
filter := ""
for key, values := range r.URL.Query() {
key = strings.TrimSpace(key)
key = strings.ToLower(key)
// A key can have multiple values in a query string, we are going to always consider
// the last one (overwrite strategy)
for _, value := range values {
value = strings.TrimSpace(value)
value = strings.ToLower(value)
switch key {
case "orderby":
// OrderBy parameter will store the fields that the user want to be the keys of the sort
// algorithm in the result set and the direction that each sort field will have. The format
// that will be used is:
//
// <field1>:<direction1>@<field2>:<direction2>@...@<fieldN>:<directionN>
orderByParts := strings.Split(value, "@")
for _, orderByPart := range orderByParts {
orderByPart = strings.TrimSpace(orderByPart)
orderByAndDirection := strings.Split(orderByPart, ":")
var field, direction string
if len(orderByAndDirection) == 1 {
field, direction = orderByAndDirection[0], "asc"
} else if len(orderByAndDirection) == 2 {
field, direction = orderByAndDirection[0], orderByAndDirection[1]
} else {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByField, err := dao.DomainDAOOrderByFieldFromString(field)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByDirection, err := dao.DAOOrderByDirectionFromString(direction)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
pagination.OrderBy = append(pagination.OrderBy, dao.DomainDAOSort{
Field: orderByField,
Direction: orderByDirection,
})
}
case "pagesize":
var err error
pagination.PageSize, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page-size", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "page":
var err error
pagination.Page, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "expand":
expand = true
case "filter":
filter = value
}
}
}
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
domains, err := domainDAO.FindAll(&pagination, expand, filter)
if err != nil {
log.Println("Error while filtering domains objects. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
domainsResponse := protocol.ToDomainsResponse(domains, pagination, expand, filter)
h.Response = &domainsResponse
// Last-Modified is going to be the most recent date of the list
for _, domain := range domains {
if domain.LastModifiedAt.After(h.lastModifiedAt) {
h.lastModifiedAt = domain.LastModifiedAt
}
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.lastModifiedAt.Format(time.RFC1123))
w.WriteHeader(http.StatusOK)
}
// Put is responsable for checking a domain object on-the-fly without persisting in
// database, useful for pre-registration validations in the registry
func (h *DomainVerificationHandler) Put(w http.ResponseWriter, r *http.Request) {
// We need to set the FQDN in the domain request object because it is sent only in the
// URI and not in the domain request body to avoid information redudancy
h.Request.FQDN = h.GetFQDN()
var domain model.Domain
var err error
if domain, err = protocol.Merge(domain, h.Request); err != nil {
messageId := ""
switch err {
case protocol.ErrInvalidDNSKEY:
messageId = "invalid-dnskey"
case protocol.ErrInvalidDSAlgorithm:
messageId = "invalid-ds-algorithm"
case protocol.ErrInvalidDSDigestType:
messageId = "invalid-ds-digest-type"
case protocol.ErrInvalidIP:
messageId = "invalid-ip"
case protocol.ErrInvalidLanguage:
messageId = "invalid-language"
}
if len(messageId) == 0 {
log.Println("Error while merging domain objects for domain verification "+
"operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
} else {
if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
return
}
scan.ScanDomain(&domain)
// As we alredy did the scan, if the domain is registered in the system, we update it for this
// results. This also gives a more intuitive design for when the user wants to force a check a
// specific domain in the Shelter system
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if dbDomain, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
update := true
// Check if we have the same nameservers, and if so update the last status
if len(dbDomain.Nameservers) == len(domain.Nameservers) {
for i := range dbDomain.Nameservers {
dbNameserver := dbDomain.Nameservers[i]
nameserver := domain.Nameservers[i]
if dbNameserver.Host == nameserver.Host &&
dbNameserver.IPv4.Equal(nameserver.IPv4) &&
dbNameserver.IPv6.Equal(nameserver.IPv6) {
dbDomain.Nameservers[i].ChangeStatus(nameserver.LastStatus)
} else {
update = false
break
}
}
} else {
update = false
}
// Check if we have the same DS set, and if so update the last status
if len(dbDomain.DSSet) == len(domain.DSSet) {
for i := range dbDomain.DSSet {
dbDS := dbDomain.DSSet[i]
ds := domain.DSSet[i]
if dbDS.Keytag == ds.Keytag &&
dbDS.Algorithm == ds.Algorithm &&
dbDS.DigestType == ds.DigestType &&
dbDS.Digest == ds.Digest {
dbDomain.DSSet[i].ChangeStatus(ds.LastStatus)
} else {
update = false
break
}
}
} else {
update = false
}
if update {
// We don't care about errors resulted here, because the main idea of this service is to scan
// a domaion, not persist the results
domainDAO.Save(&dbDomain)
}
}
w.WriteHeader(http.StatusOK)
domainResponse := protocol.ToDomainResponse(domain, false)
h.Response = &domainResponse
}
// Method that starts the injector job, retrieving the data from the database and adding
// the same data into a channel for a querier start sending DNS requests. There are two
// parameters, one to control the scan go routines and sinalize to the main thread the
// end, and other to define a channel to report errors while loading the data. This method
// is asynchronous and will finish sending a poison pill (error or nil domain) to indicate
// to the querier that there are no more domains
func (i *Injector) Start(scanGroup *sync.WaitGroup, errorsChannel chan error) chan *model.Domain {
// Create the output channel where we are going to add the domains retrieved from the
// database for the querier
domainsToQueryChannel := make(chan *model.Domain, i.DomainsBufferSize)
// Add one more to the group of scan go routines
scanGroup.Add(1)
go func() {
// Initialize Domain DAO using injected database connection
domainDAO := dao.DomainDAO{
Database: i.Database,
}
// Load all domains from database to begin the scan
domainChannel, err := domainDAO.FindAllAsync()
// Low level error was detected. No domain was processed yet, but we still need to
// shutdown the querier and by consequence the collector, so we send back the error
// and add the poison pill
if err != nil {
errorsChannel <- err
domainsToQueryChannel <- nil
// Tells the scan information structure that the injector is done
model.FinishLoadingDomainsForScan()
scanGroup.Done()
return
}
// Dispatch the asynchronous part of the method
for {
// Get domain from the database (one-by-one)
domainResult := <-domainChannel
// Send back the error to the caller thread. We don't log the error here directly
// into the log interface because sometimes we want to do something when an error
// occurs, like in a test enviroment
if domainResult.Error != nil {
errorsChannel <- domainResult.Error
}
// Problem detected while retrieving a domain or we don't have domains anymore, send
// the poison pill to alert the querier
if domainResult.Error != nil || domainResult.Domain == nil {
domainsToQueryChannel <- nil
// Tells the scan information structure that the injector is done
model.FinishLoadingDomainsForScan()
scanGroup.Done()
return
}
// The logic that decides if a domain is going to be a part of this scan or not is
// inside the domain object for better unit testing
if domainResult.Domain.ShouldBeScanned(i.MaxOKVerificationDays,
i.MaxErrorVerificationDays, i.MaxExpirationAlertDays) {
// Send to the querier
domainsToQueryChannel <- domainResult.Domain
// Count domain for the scan information to estimate the scan progress
model.LoadedDomainForScan()
}
}
}()
return domainsToQueryChannel
}
func (h *DomainHandler) Put(w http.ResponseWriter, r *http.Request) {
// We need to set the FQDN in the domain request object because it is sent only in the
// URI and not in the domain request body to avoid information redudancy
h.Request.FQDN = h.GetFQDN()
var err error
if h.domain, err = protocol.Merge(h.domain, h.Request); err != nil {
messageId := ""
switch err {
case model.ErrInvalidFQDN:
messageId = "invalid-fqdn"
case protocol.ErrInvalidDNSKEY:
messageId = "invalid-dnskey"
case protocol.ErrInvalidDSAlgorithm:
messageId = "invalid-ds-algorithm"
case protocol.ErrInvalidDSDigestType:
messageId = "invalid-ds-digest-type"
case protocol.ErrInvalidIP:
messageId = "invalid-ip"
case protocol.ErrInvalidLanguage:
messageId = "invalid-language"
}
if len(messageId) == 0 {
log.Println("Error while merging domain objects for create or "+
"update operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
} else {
if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
return
}
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if err := domainDAO.Save(&h.domain); err != nil {
if strings.Index(err.Error(), "duplicate key error index") != -1 {
if err := h.MessageResponse("conflict", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusConflict)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
} else {
log.Println("Error while saving domain object for create or "+
"update operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.GetLastModifiedAt().Format(time.RFC1123))
if h.domain.Revision == 1 {
w.Header().Add("Location", "/domain/"+h.domain.FQDN)
w.WriteHeader(http.StatusCreated)
} else {
w.WriteHeader(http.StatusNoContent)
}
}
func domainWithNoErrors(domainDAO dao.DomainDAO) {
domain, dnskey, rrsig, lastCheckAt, lastOKAt := generateSignAndSaveDomain("br.", domainDAO)
dns.HandleFunc("br.", func(w dns.ResponseWriter, dnsRequestMessage *dns.Msg) {
defer w.Close()
if dnsRequestMessage.Question[0].Qtype == dns.TypeSOA {
dnsResponseMessage := &dns.Msg{
MsgHdr: dns.MsgHdr{
Authoritative: true,
},
Question: dnsRequestMessage.Question,
Answer: []dns.RR{
&dns.SOA{
Hdr: dns.RR_Header{
Name: "br.",
Rrtype: dns.TypeSOA,
Class: dns.ClassINET,
Ttl: 86400,
},
Ns: "ns1.br.",
Mbox: "rafael.justo.net.br.",
Serial: 2013112600,
Refresh: 86400,
Retry: 86400,
Expire: 86400,
Minttl: 900,
},
},
}
dnsResponseMessage.SetReply(dnsRequestMessage)
w.WriteMsg(dnsResponseMessage)
} else if dnsRequestMessage.Question[0].Qtype == dns.TypeDNSKEY {
dnsResponseMessage := &dns.Msg{
MsgHdr: dns.MsgHdr{
Authoritative: true,
},
Question: dnsRequestMessage.Question,
Answer: []dns.RR{
dnskey,
rrsig,
},
}
dnsResponseMessage.SetReply(dnsRequestMessage)
w.WriteMsg(dnsResponseMessage)
}
})
scan.ScanDomains()
domain, err := domainDAO.FindByFQDN(domain.FQDN)
if err != nil {
utils.Fatalln("Didn't find scanned domain", err)
}
for _, nameserver := range domain.Nameservers {
if nameserver.LastStatus != model.NameserverStatusOK {
utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured nameserver '%s'. Found status: %s",
nameserver.Host, model.NameserverStatusToString(nameserver.LastStatus)), nil)
}
if nameserver.LastCheckAt.Before(lastCheckAt) ||
nameserver.LastCheckAt.Equal(lastCheckAt) {
utils.Fatalln(fmt.Sprintf("Last check date was not updated in nameserver '%s'",
nameserver.Host), nil)
}
if nameserver.LastOKAt.Before(lastOKAt) || nameserver.LastOKAt.Equal(lastOKAt) {
utils.Fatalln(fmt.Sprintf("Last OK date was not updated in nameserver '%s'",
nameserver.Host), nil)
}
}
for _, ds := range domain.DSSet {
if ds.LastStatus != model.DSStatusOK {
utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured DS %d. "+
"Found status: %s", ds.Keytag, model.DSStatusToString(ds.LastStatus)), nil)
}
if ds.LastCheckAt.Before(lastCheckAt) || ds.LastCheckAt.Equal(lastCheckAt) {
utils.Fatalln(fmt.Sprintf("Last check date was not updated in DS %d",
ds.Keytag), nil)
}
if ds.LastOKAt.Before(lastOKAt) || ds.LastOKAt.Equal(lastOKAt) {
utils.Fatalln(fmt.Sprintf("Last OK date was not updated in DS %d",
ds.Keytag), nil)
}
}
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Error removing domain %s", domain.FQDN), err)
}
}
func simpleNotification(domainDAO dao.DomainDAO, templateName string,
messageChannel chan *mail.Message, errorChannel chan error) {
generateAndSaveDomain("example.com.br.", domainDAO, templateName)
notification.TemplateExtension = ""
if err := notification.LoadTemplates(); err != nil {
utils.Fatalln("Error loading templates", err)
}
notification.Notify()
timeout := make(chan bool, 1)
go func() {
time.Sleep(5 * time.Second)
timeout <- true
}()
select {
case message := <-messageChannel:
if message.Header.Get("From") != "*****@*****.**" {
utils.Fatalln(fmt.Sprintf("E-mail from header is different. Expected "+
"[email protected] but found %s", message.Header.Get("From")), nil)
}
if message.Header.Get("To") != "*****@*****.**" {
utils.Fatalln("E-mail to header is different", nil)
}
if message.Header.Get("Subject") != "=?UTF-8?B?TWlzY29uZmlndXJhdGlvbiBvbiBkb21haW4gZXhhbXBsZS5jb20uYnIu?=" {
utils.Fatalln("E-mail subject header is different", nil)
}
body, err := ioutil.ReadAll(message.Body)
if err != nil {
utils.Fatalln("Error reading e-mail body", err)
}
expectedBody := "Dear Sir/Madam,\r\n" +
"\r\n" +
"During our periodically domain verification, a configuration problem was detected with the\r\n" +
"domain example.com.br..\r\n" +
"\r\n" +
" * Nameserver ns1.example.com.br. got an internal error while receiving the DNS request.\r\n" +
" Please check the DNS server log to detect and solve the problem.\r\n" +
"\r\n" +
"Best regards,\r\n" +
"LACTLD\r\n" +
".\r\n"
if string(body) != expectedBody {
utils.Fatalln(fmt.Sprintf("E-mail body is different from what we expected. "+
"Expected [%s], but found [%s]", expectedBody, body), nil)
}
case err := <-errorChannel:
utils.Fatalln("Error receiving message", err)
case <-timeout:
utils.Fatalln("No mail sent", nil)
}
if err := domainDAO.RemoveByFQDN("example.com.br."); err != nil {
utils.Fatalln("Error removing domain", err)
}
}
func domainsPagination(domainDAO dao.DomainDAO) {
numberOfItems := 1000
for i := 0; i < numberOfItems; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("example%d.com.br", i),
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
pagination := dao.DomainDAOPagination{
PageSize: 10,
Page: 5,
OrderBy: []dao.DomainDAOSort{
{
Field: dao.DomainDAOOrderByFieldFQDN,
Direction: dao.DAOOrderByDirectionAscending,
},
},
}
domains, err := domainDAO.FindAll(&pagination, true, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
if pagination.NumberOfItems != numberOfItems {
utils.Errorln("Number of items not calculated correctly", nil)
}
if pagination.NumberOfPages != numberOfItems/pagination.PageSize {
utils.Errorln("Number of pages not calculated correctly", nil)
}
if len(domains) != pagination.PageSize {
utils.Errorln("Number of domains not following page size", nil)
}
pagination = dao.DomainDAOPagination{
PageSize: 10000,
Page: 1,
OrderBy: []dao.DomainDAOSort{
{
Field: dao.DomainDAOOrderByFieldFQDN,
Direction: dao.DAOOrderByDirectionAscending,
},
},
}
domains, err = domainDAO.FindAll(&pagination, true, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
if pagination.NumberOfPages != 1 {
utils.Fatalln("Calculating wrong number of pages when there's only one page", nil)
}
for i := 0; i < numberOfItems; i++ {
fqdn := fmt.Sprintf("example%d.com.br", i)
if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
utils.Fatalln("Error removing domain from database", err)
}
}
}
// Verify if the method that choose the domains that needs to be verified is correct
func domainsNotification(domainDAO dao.DomainDAO) {
numberOfItemsToBeVerified := 1000
numberOfItemsToDontBeVerified := 1000
nameserverErrorAlertDays := 7
nameserverTimeoutAlertDays := 30
dsErrorAlertDays := 1
dsTimeoutAlertDays := 7
maxExpirationAlertDays := 5
data := []struct {
name string
numberOfItems int
nameserverTimeoutLastOKAt time.Time
nameserverErrorLastOKAt time.Time
dsTimeoutLastOkAt time.Time
dsErrorLastOkAt time.Time
dsExpiresAt time.Time
}{
{
name: "shouldbenotified",
numberOfItems: numberOfItemsToBeVerified,
nameserverTimeoutLastOKAt: time.Now().Add(time.Duration(-nameserverTimeoutAlertDays*24) * time.Hour),
nameserverErrorLastOKAt: time.Now().Add(time.Duration(-nameserverErrorAlertDays*24) * time.Hour),
dsTimeoutLastOkAt: time.Now().Add(time.Duration(-dsTimeoutAlertDays*24) * time.Hour),
dsErrorLastOkAt: time.Now().Add(time.Duration(-dsErrorAlertDays*24) * time.Hour),
dsExpiresAt: time.Now().Add(time.Duration((maxExpirationAlertDays)*24) * time.Hour),
},
{
name: "shouldnotbenotified",
numberOfItems: numberOfItemsToDontBeVerified,
nameserverTimeoutLastOKAt: time.Now().Add(time.Duration((-nameserverTimeoutAlertDays+1)*24) * time.Hour),
nameserverErrorLastOKAt: time.Now().Add(time.Duration((-nameserverErrorAlertDays+1)*24) * time.Hour),
dsTimeoutLastOkAt: time.Now().Add(time.Duration((-dsTimeoutAlertDays+1)*24) * time.Hour),
dsErrorLastOkAt: time.Now().Add(time.Duration((-dsErrorAlertDays+1)*24) * time.Hour),
dsExpiresAt: time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
},
}
for _, item := range data {
for i := 0; i < item.numberOfItems/5; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
Nameservers: []model.Nameserver{
{
LastStatus: model.NameserverStatusTimeout,
LastOKAt: item.nameserverTimeoutLastOKAt,
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
for i := item.numberOfItems / 5; i < item.numberOfItems/5*2; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
Nameservers: []model.Nameserver{
{
LastStatus: model.NameserverStatusNoAuthority,
LastOKAt: item.nameserverErrorLastOKAt,
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
for i := item.numberOfItems / 5 * 2; i < item.numberOfItems/5*3; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
DSSet: []model.DS{
{
LastStatus: model.DSStatusTimeout,
LastOKAt: item.dsTimeoutLastOkAt,
ExpiresAt: time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
for i := item.numberOfItems / 5 * 3; i < item.numberOfItems/5*4; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
DSSet: []model.DS{
{
LastStatus: model.DSStatusExpiredSignature,
LastOKAt: item.dsErrorLastOkAt,
ExpiresAt: time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
for i := item.numberOfItems / 5 * 4; i < item.numberOfItems; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
DSSet: []model.DS{
{
LastStatus: model.DSStatusOK,
LastOKAt: time.Now(),
ExpiresAt: item.dsExpiresAt,
},
},
}
if err := domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving domain in database", err)
}
}
}
domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
nameserverErrorAlertDays,
nameserverTimeoutAlertDays,
dsErrorAlertDays,
dsTimeoutAlertDays,
maxExpirationAlertDays,
)
if err != nil {
utils.Fatalln("Error retrieving domains to be notified", err)
}
var domains []*model.Domain
for {
domainResult := <-domainChannel
if domainResult.Error != nil {
utils.Fatalln("Error retrieving domain to be notified", domainResult.Error)
}
if domainResult.Error != nil || domainResult.Domain == nil {
break
}
domains = append(domains, domainResult.Domain)
}
if len(domains) != numberOfItemsToBeVerified {
utils.Fatalln(fmt.Sprintf("Did not select all the domains ready for notification. "+
"Expected %d and got %d", numberOfItemsToBeVerified, len(domains)), nil)
}
for _, item := range data {
for i := 0; i < item.numberOfItems; i++ {
fqdn := fmt.Sprintf("%s%d.com.br", item.name, i)
if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
utils.Fatalln("Error removing domain from database", err)
}
}
}
}
func domainWithErrors(config ScanCollectorTestConfigFile, database *mgo.Database) {
domainsToSave := make(chan *model.Domain, config.Scan.DomainsBufferSize)
domainsToSave <- &model.Domain{
FQDN: "br.",
Nameservers: []model.Nameserver{
{
Host: "ns1.br",
IPv4: net.ParseIP("127.0.0.1"),
LastStatus: model.NameserverStatusTimeout,
},
},
DSSet: []model.DS{
{
Keytag: 1234,
Algorithm: model.DSAlgorithmRSASHA1NSEC3,
DigestType: model.DSDigestTypeSHA1,
Digest: "EAA0978F38879DB70A53F9FF1ACF21D046A98B5C",
LastStatus: model.DSStatusExpiredSignature,
},
},
}
domainsToSave <- nil
model.StartNewScan()
runScan(config, database, domainsToSave)
domainDAO := dao.DomainDAO{
Database: database,
}
domain, err := domainDAO.FindByFQDN("br.")
if err != nil {
utils.Fatalln("Error loading domain with problems", err)
}
if len(domain.Nameservers) == 0 {
utils.Fatalln("Error saving nameservers", nil)
}
if domain.Nameservers[0].LastStatus != model.NameserverStatusTimeout {
utils.Fatalln("Error setting status in the nameserver", nil)
}
if len(domain.DSSet) == 0 {
utils.Fatalln("Error saving the DS set", nil)
}
if domain.DSSet[0].LastStatus != model.DSStatusExpiredSignature {
utils.Fatalln("Error setting status in the DS", nil)
}
if err := domainDAO.RemoveByFQDN("br."); err != nil {
utils.Fatalln("Error removing test domain", err)
}
currentScan := model.GetCurrentScan()
if currentScan.DomainsScanned != 1 || currentScan.DomainsWithDNSSECScanned != 1 {
utils.Fatalln("Not counting domains for scan progress when there're errors", nil)
}
if currentScan.NameserverStatistics[model.NameserverStatusToString(model.NameserverStatusTimeout)] != 1 ||
currentScan.DSStatistics[model.DSStatusToString(model.DSStatusExpiredSignature)] != 1 {
utils.Fatalln("Not counting statistics properly when there're errors", nil)
}
}
func domainsExpand(domainDAO dao.DomainDAO) {
newDomains := newDomains()
domainsResult := domainDAO.SaveMany(newDomains)
for _, domainResult := range domainsResult {
if domainResult.Error != nil {
utils.Fatalln("Error creating domains", domainResult.Error)
}
}
pagination := dao.DomainDAOPagination{}
domains, err := domainDAO.FindAll(&pagination, false, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
for _, domain := range domains {
if len(domain.Owners) > 0 {
utils.Fatalln("Not compressing owners in results", nil)
}
for _, nameserver := range domain.Nameservers {
if len(nameserver.Host) > 0 ||
nameserver.IPv4 != nil ||
nameserver.IPv6 != nil ||
!nameserver.LastCheckAt.Equal(time.Time{}) ||
!nameserver.LastOKAt.Equal(time.Time{}) {
utils.Fatalln("Not compressing nameservers in results", nil)
}
}
for _, ds := range domain.DSSet {
if ds.Algorithm != 0 ||
len(ds.Digest) > 0 ||
ds.DigestType != 0 ||
ds.Keytag != 0 ||
!ds.ExpiresAt.Equal(time.Time{}) ||
!ds.LastCheckAt.Equal(time.Time{}) ||
!ds.LastOKAt.Equal(time.Time{}) {
utils.Fatalln("Not compressing ds set in results", nil)
}
}
}
domains, err = domainDAO.FindAll(&pagination, true, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
for _, domain := range domains {
if len(domain.Owners) == 0 {
utils.Fatalln("Compressing owners in results when it shouldn't", nil)
}
for _, nameserver := range domain.Nameservers {
if len(nameserver.Host) == 0 ||
nameserver.IPv4 == nil ||
nameserver.IPv6 == nil ||
nameserver.LastCheckAt.Equal(time.Time{}) ||
nameserver.LastOKAt.Equal(time.Time{}) ||
nameserver.LastStatus != model.NameserverStatusOK {
utils.Fatalln("Compressing nameservers in results when it shouldn't", nil)
}
}
for _, ds := range domain.DSSet {
if ds.Algorithm == 0 ||
len(ds.Digest) == 0 ||
ds.DigestType == 0 ||
ds.Keytag == 0 ||
ds.ExpiresAt.Equal(time.Time{}) ||
ds.LastCheckAt.Equal(time.Time{}) ||
ds.LastOKAt.Equal(time.Time{}) ||
ds.LastStatus != model.DSStatusOK {
utils.Fatalln("Compressing ds set in results when it shouldn't", nil)
}
}
}
domainsResult = domainDAO.RemoveMany(newDomains)
for _, domainResult := range domainsResult {
if domainResult.Error != nil {
utils.Fatalln("Error removing domains", domainResult.Error)
}
}
}
func brDomainWithoutDNSSEC(domainDAO dao.DomainDAO) {
domain := model.Domain{
FQDN: "br.",
Nameservers: []model.Nameserver{
{
Host: "a.dns.br.",
IPv4: net.ParseIP("200.160.0.10"),
IPv6: net.ParseIP("2001:12ff::10"),
},
{
Host: "b.dns.br.",
IPv4: net.ParseIP("200.189.41.10"),
},
{
Host: "c.dns.br.",
IPv4: net.ParseIP("200.192.233.10"),
},
{
Host: "d.dns.br.",
IPv4: net.ParseIP("200.219.154.10"),
IPv6: net.ParseIP("2001:12f8:4::10"),
},
{
Host: "f.dns.br.",
IPv4: net.ParseIP("200.219.159.10"),
},
},
// We are going to add the current DNSKEYs from .br but we are not going to check it.
// This is because there's a strange case that when it found a problem on a DS (such
// as bit SEP) it does not check other nameservers
DSSet: []model.DS{
{
Keytag: 41674,
Algorithm: model.DSAlgorithmRSASHA1,
DigestType: model.DSDigestTypeSHA256,
Digest: "6ec74914376b4f383ede3840088ae1d7bf13a19bfc51465cc2da57618889416a",
},
{
Keytag: 57207,
Algorithm: model.DSAlgorithmRSASHA1,
DigestType: model.DSDigestTypeSHA256,
Digest: "d46f059860d31a0965f925ac6ff97ed0975f33a14e2d01ec5ab5dd543624d307",
},
},
}
var err error
if err = domainDAO.Save(&domain); err != nil {
utils.Fatalln("Error saving the domain", err)
}
scan.ScanDomains()
domain, err = domainDAO.FindByFQDN(domain.FQDN)
if err != nil {
utils.Fatalln("Didn't find scanned domain", err)
}
for _, nameserver := range domain.Nameservers {
if nameserver.LastStatus != model.NameserverStatusOK {
utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured nameserver '%s'. Found status: %s",
nameserver.Host, model.NameserverStatusToString(nameserver.LastStatus)), nil)
}
}
if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Error removing domain %s", domain.FQDN), err)
}
}
func calculateDomainDAODurations(domainDAO dao.DomainDAO, numberOfItems int) (totalDuration, insertDuration,
queryDuration, removeDuration time.Duration) {
beginTimer := time.Now()
sectionTimer := beginTimer
// Build array to create many at once
var domains []*model.Domain
for i := 0; i < numberOfItems; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("test%d.com.br", i),
}
domains = append(domains, &domain)
}
errorInDomainsCreation := false
domainResults := domainDAO.SaveMany(domains)
// Check if there was any error while creating them
for _, domainResult := range domainResults {
if domainResult.Error != nil {
errorInDomainsCreation = true
utils.Errorln(fmt.Sprintf("Couldn't save domain %s in database during the performance test",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if errorInDomainsCreation {
utils.Fatalln("Due to errors in domain creation, the performance test will be aborted", nil)
}
insertDuration = time.Since(sectionTimer)
sectionTimer = time.Now()
// Try to find domains from different parts of the whole range to check indexes
queryRanges := numberOfItems / 4
fqdn1 := fmt.Sprintf("test%d.com.br", queryRanges)
fqdn2 := fmt.Sprintf("test%d.com.br", queryRanges*2)
fqdn3 := fmt.Sprintf("test%d.com.br", queryRanges*3)
if _, err := domainDAO.FindByFQDN(fqdn1); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn1), err)
}
if _, err := domainDAO.FindByFQDN(fqdn2); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn2), err)
}
if _, err := domainDAO.FindByFQDN(fqdn3); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn3), err)
}
queryDuration = time.Since(sectionTimer)
sectionTimer = time.Now()
errorInDomainsRemoval := false
domainResults = domainDAO.RemoveMany(domains)
// Check if there was any error while removing them
for _, domainResult := range domainResults {
if domainResult.Error != nil {
errorInDomainsRemoval = true
utils.Errorln(fmt.Sprintf("Error while trying to remove a domain %s during the performance test",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if errorInDomainsRemoval {
utils.Fatalln("Due to errors in domain removal, the performance test will be aborted", nil)
}
removeDuration = time.Since(sectionTimer)
totalDuration = time.Since(beginTimer)
return
}
// This method is the last part of the scan, when the new state of the domain object is
// persisted back to the database. It receives a go routine control group to sinalize to
// the main thread when the scan ends, a domain channel to receive each domain that need
// to be save and an error channel to send back all errors while persisting the data. It
// was created to be asynchronous and finish after receiving a poison pill from querier
// dispatcher
func (c *Collector) Start(scanGroup *sync.WaitGroup,
domainsToSaveChannel chan *model.Domain, errorsChannel chan error) {
// Add one more to the group of scan go routines
scanGroup.Add(1)
go func() {
// Initialize Domain DAO using injected database connection
domainDAO := dao.DomainDAO{
Database: c.Database,
}
// Add a safety check to avoid an infinite loop
if c.SaveAtOnce == 0 {
c.SaveAtOnce = 1
}
finished := false
nameserverStatistics := make(map[string]uint64)
dsStatistics := make(map[string]uint64)
for {
// Using make for faster allocation
domains := make([]*model.Domain, 0, c.SaveAtOnce)
for i := 0; i < c.SaveAtOnce; i++ {
domain := <-domainsToSaveChannel
// Detect poison pill. We don't return from function here because we can still
// have some domains to save in the domains array
if domain == nil {
finished = true
break
}
// Count this domain for the scan information to estimate the scan progress
model.FinishAnalyzingDomainForScan(len(domain.DSSet) > 0)
// Keep track of nameservers statistics
for _, nameserver := range domain.Nameservers {
status := model.NameserverStatusToString(nameserver.LastStatus)
nameserverStatistics[status] += 1
}
// Keep track of DS statistics
for _, ds := range domain.DSSet {
status := model.DSStatusToString(ds.LastStatus)
dsStatistics[status] += 1
}
domains = append(domains, domain)
}
domainsResults := domainDAO.SaveMany(domains)
for _, domainResult := range domainsResults {
if domainResult.Error != nil {
// Error channel should have a buffer or this will block the collector until
// someone check this error. One question here is that we are returning the
// error, but not telling wich domain got the error, we should improve the error
// communication system between the go routines
errorsChannel <- domainResult.Error
}
}
// Now that everything is done, check if we received a poison pill
if finished {
model.StoreStatisticsOfTheScan(nameserverStatistics, dsStatistics)
scanGroup.Done()
return
}
}
}()
}