// Generates a report with the amount of time for each operation in the domain DAO. For
// more realistic values it does the same operation for the same amount of data X number
// of times to get the average time of the operation. After some results, with indexes we
// get 80% better performance, another good improvements was to create and remove many
// objects at once using go routines
func domainDAOPerformanceReport(reportFile string, domainDAO dao.DomainDAO) {
// Report header
report := " # | Total | Insert | Find | Remove\n" +
"------------------------------------------------------------------------------------\n"
// Report variables
averageTurns := 5
scale := []int{10, 50, 100, 500, 1000, 5000,
10000, 50000, 100000, 500000, 1000000, 5000000}
for _, numberOfItems := range scale {
var totalDuration, insertDuration, queryDuration, removeDuration time.Duration
for i := 0; i < averageTurns; i++ {
utils.Println(fmt.Sprintf("Generating report - scale %d - turn %d", numberOfItems, i+1))
totalDurationTmp, insertDurationTmp, queryDurationTmp, removeDurationTmp :=
calculateDomainDAODurations(domainDAO, numberOfItems)
totalDuration += totalDurationTmp
insertDuration += insertDurationTmp
queryDuration += queryDurationTmp
removeDuration += removeDurationTmp
}
report += fmt.Sprintf("% -8d | % 16s | % 16s | % 16s | % 16s\n",
numberOfItems,
time.Duration(int64(totalDuration)/int64(averageTurns)).String(),
time.Duration(int64(insertDuration)/int64(averageTurns)).String(),
time.Duration(int64(queryDuration)/int64(averageTurns)).String(),
time.Duration(int64(removeDuration)/int64(averageTurns)).String(),
)
}
utils.WriteReport(reportFile, report)
}
func main() {
flag.Parse()
var clientConfig ClientDomainVerificationTestConfigFile
err := utils.ReadConfigFile(configFilePath, &clientConfig)
config.ShelterConfig = clientConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
utils.StartDNSServer(clientConfig.DNSServerPort, clientConfig.Scan.UDPMaxSize)
finishRESTServer := utils.StartRESTServer()
defer finishRESTServer()
utils.StartWebClient()
scanDomain()
queryDomain()
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var config ScanQuerierTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
utils.StartDNSServer(config.Server.Port, config.Scan.UDPMaxSize)
domainWithNoDNSErrors(config)
domainWithNoDNSSECErrors(config)
domainDNSTimeout(config)
domainDNSUnknownHost(config)
// Scan querier performance report is optional and only generated when the report file
// path parameter is given
if report {
scanQuerierReport(config)
}
if inputReport {
inputScanReport(config)
}
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var config ScanCollectorTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// Remove all data before starting the test. This is necessary because maybe in the last
// test there was an error and the data wasn't removed from the database
utils.ClearDatabase(database)
domainWithErrors(config, database)
domainWithNoErrors(config, database)
utils.Println("SUCCESS!")
}
// Generates a report with the amount of time of a scan
func scanReport(domainDAO dao.DomainDAO, scanDAO dao.ScanDAO, scanConfig ScanTestConfigFile) {
report := " # | Total | DPS | Memory (MB)\n" +
"-----------------------------------------------------\n"
// Report variables
scale := []int{10, 50, 100, 500, 1000, 5000,
10000, 50000, 100000, 500000, 1000000, 5000000}
dnskey, privateKey, err := utils.GenerateKey()
if err != nil {
utils.Fatalln("Error generating DNSKEY", err)
}
reportHandler := ReportHandler{
DNSKEY: dnskey,
PrivateKey: privateKey,
}
server.Handler = reportHandler
dns.DefaultServeMux = nil
for _, numberOfItems := range scale {
utils.Println(fmt.Sprintf("Generating report - scale %d", numberOfItems))
for i := 0; i < numberOfItems; i++ {
if i%1000 == 0 {
utils.PrintProgress("building scenario", (i*100)/numberOfItems)
}
fqdn := fmt.Sprintf("domain%d.br.", i)
generateAndSaveDomain(fqdn, domainDAO, dnskey)
}
utils.PrintProgress("building scenario", 100)
totalDuration, domainsPerSecond := calculateScanDurations(numberOfItems, scanDAO)
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
report += fmt.Sprintf("% -8d | %16s | %4d | %14.2f\n",
numberOfItems,
time.Duration(int64(totalDuration)).String(),
domainsPerSecond,
float64(memStats.Alloc)/float64(MB),
)
if err := domainDAO.RemoveAll(); err != nil {
// When the result set is too big to remove, we got a timeout error from the
// connection, but it's ok
//utils.Fatalln("Error removing domains generated for report", err)
}
}
utils.WriteReport(scanConfig.Report.File, report)
}
func main() {
flag.Parse()
var config DomainDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
domainLifeCycle(domainDAO)
domainsLifeCycle(domainDAO)
domainUniqueFQDN(domainDAO)
domainConcurrency(domainDAO)
domainsPagination(domainDAO)
domainsNotification(domainDAO)
domainsExpand(domainDAO)
domainFilter(domainDAO)
// Domain DAO performance report is optional and only generated when the report file
// path parameter is given
if report {
domainDAOPerformanceReport(config.Report.ReportFile, domainDAO)
}
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
utils.ClearDatabase(database)
invalidFQDN(database)
createDomain(database)
updateDomain(database)
retrieveDomain(database)
retrieveDomainMetadata(database)
retrieveDomainIfModifiedSince(database)
retrieveDomainIfUnmodifiedSince(database)
retrieveDomainIfMatch(database)
retrieveDomainIfNoneMatch(database)
updateDomainIfModifiedSince(database)
updateDomainIfUnmodifiedSince(database)
updateDomainIfMatch(database)
updateDomainIfNoneMatch(database)
deleteDomainIfModifiedSince(database)
deleteDomainIfUnmodifiedSince(database)
deleteDomainIfMatch(database)
deleteDomainIfNoneMatch(database)
deleteDomain(database)
retrieveUnknownDomain(database)
utils.Println("SUCCESS!")
}
func restReport(restConfig RESTTestConfigFile) {
report := " # | Operation | Total | DPS | Memory (MB)\n" +
"---------------------------------------------------------------\n"
// Report variables
scale := []int{10, 50, 100, 500, 1000, 5000,
10000, 50000, 100000, 500000, 1000000, 5000000}
content := []byte(`{
"Nameservers": [
{ "host": "ns1.example.com.br.", "ipv4": "127.0.0.1" },
{ "host": "ns2.example.com.br.", "ipv6": "::1" }
],
"Owners": [
{ "email": "*****@*****.**", "language": "pt-br" }
]
}`)
url := ""
if len(config.ShelterConfig.RESTServer.Listeners) > 0 {
url = fmt.Sprintf("http://%s:%d", config.ShelterConfig.RESTServer.Listeners[0].IP,
config.ShelterConfig.RESTServer.Listeners[0].Port)
}
if len(url) == 0 {
utils.Fatalln("There's no interface to connect to", nil)
}
for _, numberOfItems := range scale {
utils.Println(fmt.Sprintf("Generating report - scale %d", numberOfItems))
// Generate domains
report += restActionReport(numberOfItems, content, func(i int) (*http.Request, error) {
return http.NewRequest("PUT", fmt.Sprintf("%s/domain/example%d.com.br.", url, i),
bytes.NewReader(content))
}, http.StatusCreated, "CREATE")
// Retrieve domains
report += restActionReport(numberOfItems, content, func(i int) (*http.Request, error) {
return http.NewRequest("GET", fmt.Sprintf("%s/domain/example%d.com.br.", url, i), nil)
}, http.StatusOK, "RETRIEVE")
// Delete domains
report += restActionReport(numberOfItems, content, func(i int) (*http.Request, error) {
return http.NewRequest("DELETE", fmt.Sprintf("%s/domain/example%d.com.br.", url, i), nil)
}, http.StatusNoContent, "DELETE")
}
utils.WriteReport(restConfig.Report.File, report)
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the database,
// so let's clear it to don't affect this test. We avoid checking the error, because if
// the collection does not exist yet, it will be created in the first insert
utils.ClearDatabase(database)
messageChannel, errorChannel, err := utils.StartMailServer(config.ShelterConfig.Notification.SMTPServer.Port)
if err != nil {
utils.Fatalln("Error starting the mail server", err)
}
domainDAO := dao.DomainDAO{
Database: database,
}
templateName := createTemplateFile()
defer removeTemplateFile(templateName)
simpleNotification(domainDAO, templateName, messageChannel, errorChannel)
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
finishRESTServer := utils.StartRESTServer()
defer finishRESTServer()
utils.StartWebClient()
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
utils.ClearDatabase(database)
createScans(database)
retrieveScans()
retrieveScansWithPagination()
retrieveScansWithCache()
retrieveCurrentScan()
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var restConfig RESTHandlerDomainVerificationTestConfigFile
err := utils.ReadConfigFile(configFilePath, &restConfig)
config.ShelterConfig = restConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
restConfig.Database.URIs,
restConfig.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
utils.StartDNSServer(restConfig.DNSServerPort, restConfig.Scan.UDPMaxSize)
scanDomain()
scanPersistedDomain(domainDAO)
queryDomain()
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var config ScanDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
scanDAO := dao.ScanDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
scanDAO.RemoveAll()
scanLifeCycle(scanDAO)
scanConcurrency(scanDAO)
scanStatistics(scanDAO)
scansPagination(scanDAO)
scansExpand(scanDAO)
utils.Println("SUCCESS!")
}
// Generates a report with the amount of time of a scan
func scanQuerierReport(config ScanQuerierTestConfigFile) {
report := " # | Total | QPS | Memory (MB)\n" +
"-----------------------------------------------------\n"
// Report variables
scale := []int{10, 50, 100, 500, 1000, 5000,
10000, 50000, 100000, 500000, 1000000, 5000000}
fqdn := "domain.com.br."
dnskey, rrsig, err := utils.GenerateKSKAndSignZone(fqdn)
if err != nil {
utils.Fatalln("Error creating DNSSEC keys and signatures", err)
}
ds := dnskey.ToDS(uint8(model.DSDigestTypeSHA1))
dns.HandleFunc(fqdn, func(w dns.ResponseWriter, dnsRequestMessage *dns.Msg) {
defer w.Close()
if dnsRequestMessage.Question[0].Qtype == dns.TypeSOA {
dnsResponseMessage := &dns.Msg{
MsgHdr: dns.MsgHdr{
Authoritative: true,
},
Question: dnsRequestMessage.Question,
Answer: []dns.RR{
&dns.SOA{
Hdr: dns.RR_Header{
Name: fqdn,
Rrtype: dns.TypeSOA,
Class: dns.ClassINET,
Ttl: 86400,
},
Ns: "ns1." + fqdn,
Mbox: "rafael.justo.net.br.",
Serial: 2013112600,
Refresh: 86400,
Retry: 86400,
Expire: 86400,
Minttl: 900,
},
},
}
dnsResponseMessage.SetReply(dnsRequestMessage)
w.WriteMsg(dnsResponseMessage)
} else if dnsRequestMessage.Question[0].Qtype == dns.TypeDNSKEY {
dnsResponseMessage := &dns.Msg{
MsgHdr: dns.MsgHdr{
Authoritative: true,
},
Question: dnsRequestMessage.Question,
Answer: []dns.RR{
dnskey,
rrsig,
},
}
dnsResponseMessage.SetReply(dnsRequestMessage)
w.WriteMsg(dnsResponseMessage)
}
})
for _, numberOfItems := range scale {
var domains []*model.Domain
for i := 0; i < numberOfItems; i++ {
// We create an object with different nameservers because we don't want to put the
// nameserver in the query rate limit check
domains = append(domains, &model.Domain{
FQDN: fqdn,
Nameservers: []model.Nameserver{
{
Host: fmt.Sprintf("ns%d.%s", i, fqdn),
IPv4: net.ParseIP("127.0.0.1"),
},
},
DSSet: []model.DS{
{
Keytag: dnskey.KeyTag(),
Algorithm: utils.ConvertKeyAlgorithm(dnskey.Algorithm),
DigestType: model.DSDigestTypeSHA1,
Digest: ds.Digest,
},
},
})
}
utils.Println(fmt.Sprintf("Generating report - scale %d", numberOfItems))
totalDuration, queriesPerSecond, _, _ :=
calculateScanQuerierDurations(config, domains)
var memStats runtime.MemStats
runtime.ReadMemStats(&memStats)
report += fmt.Sprintf("% -8d | %16s | %4d | %14.2f\n",
numberOfItems,
time.Duration(int64(totalDuration)).String(),
queriesPerSecond,
float64(memStats.Alloc)/float64(MB),
)
}
utils.WriteReport(config.Report.ReportFile, report)
}
func main() {
flag.Parse()
var scanConfig ScanTestConfigFile
err := utils.ReadConfigFile(configFilePath, &scanConfig)
config.ShelterConfig = scanConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
scanConfig.Database.URIs,
scanConfig.Database.Name,
scanConfig.Database.Auth.Enabled,
scanConfig.Database.Auth.Username,
scanConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// Remove all data before starting the test. This is necessary because maybe in the last
// test there was an error and the data wasn't removed from the database
utils.ClearDatabase(database)
server = utils.StartDNSServer(scanConfig.DNSServerPort, scanConfig.Scan.UDPMaxSize)
domainDAO := dao.DomainDAO{
Database: database,
}
// Register a scan only to avoid warning messages in the integration test execution
scheduler.Register(scheduler.Job{
Type: scheduler.JobTypeScan,
NextExecution: time.Now().Add(10 * time.Minute),
Task: func() {},
})
domainWithNoErrors(domainDAO)
domainWithNoErrorsOnTheFly()
domainQuery()
domainQueryWithDNSSECErrors()
// Scan performance report is optional and only generated when the report file
// path parameter is given
if report {
if cpuProfile {
f := utils.StartCPUProfile(scanConfig.Report.Profile.CPUFile)
defer f()
}
if memoryProfile {
f := utils.StartMemoryProfile(scanConfig.Report.Profile.MemoryFile)
defer f()
}
if goProfile {
f := utils.StartGoRoutinesProfile(scanConfig.Report.Profile.GoRoutinesFile)
defer f()
}
scanDAO := dao.ScanDAO{
Database: database,
}
scanReport(domainDAO, scanDAO, scanConfig)
}
// This test is after all others because it will ask on real nameservers, so we need to
// change to port 53
scan.DNSPort = 53
brDomainWithoutDNSSEC(domainDAO)
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var restConfig RESTTestConfigFile
err := utils.ReadConfigFile(configFilePath, &restConfig)
config.ShelterConfig = restConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the database,
// so let's clear it to don't affect this test. We avoid checking the error, because if
// the collection does not exist yet, it will be created in the first insert
utils.ClearDatabase(database)
finishRESTServer := utils.StartRESTServer()
defer finishRESTServer()
domainLifeCycle()
// REST performance report is optional and only generated when the report file path parameter is
// given
if report {
if cpuProfile {
f := utils.StartCPUProfile(restConfig.Report.Profile.CPUFile)
defer f()
}
if memoryProfile {
f := utils.StartMemoryProfile(restConfig.Report.Profile.MemoryFile)
defer f()
}
if goProfile {
f := utils.StartGoRoutinesProfile(restConfig.Report.Profile.GoRoutinesFile)
defer f()
}
restReport(restConfig)
}
utils.Println("SUCCESS!")
}
