func main() {
flag.Parse()
var config ScanCollectorTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// Remove all data before starting the test. This is necessary because maybe in the last
// test there was an error and the data wasn't removed from the database
utils.ClearDatabase(database)
domainWithErrors(config, database)
domainWithNoErrors(config, database)
utils.Println("SUCCESS!")
}
func (i *Database) Before(w http.ResponseWriter, r *http.Request) {
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error creating database connection. Details:", err)
return
}
i.databaseHandler.SetDatabaseSession(databaseSession)
i.databaseHandler.SetDatabase(database)
}
func main() {
flag.Parse()
var config DomainDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
domainLifeCycle(domainDAO)
domainsLifeCycle(domainDAO)
domainUniqueFQDN(domainDAO)
domainConcurrency(domainDAO)
domainsPagination(domainDAO)
domainsNotification(domainDAO)
domainsExpand(domainDAO)
domainFilter(domainDAO)
// Domain DAO performance report is optional and only generated when the report file
// path parameter is given
if report {
domainDAOPerformanceReport(config.Report.ReportFile, domainDAO)
}
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
utils.ClearDatabase(database)
invalidFQDN(database)
createDomain(database)
updateDomain(database)
retrieveDomain(database)
retrieveDomainMetadata(database)
retrieveDomainIfModifiedSince(database)
retrieveDomainIfUnmodifiedSince(database)
retrieveDomainIfMatch(database)
retrieveDomainIfNoneMatch(database)
updateDomainIfModifiedSince(database)
updateDomainIfUnmodifiedSince(database)
updateDomainIfMatch(database)
updateDomainIfNoneMatch(database)
deleteDomainIfModifiedSince(database)
deleteDomainIfUnmodifiedSince(database)
deleteDomainIfMatch(database)
deleteDomainIfNoneMatch(database)
deleteDomain(database)
retrieveUnknownDomain(database)
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the database,
// so let's clear it to don't affect this test. We avoid checking the error, because if
// the collection does not exist yet, it will be created in the first insert
utils.ClearDatabase(database)
messageChannel, errorChannel, err := utils.StartMailServer(config.ShelterConfig.Notification.SMTPServer.Port)
if err != nil {
utils.Fatalln("Error starting the mail server", err)
}
domainDAO := dao.DomainDAO{
Database: database,
}
templateName := createTemplateFile()
defer removeTemplateFile(templateName)
simpleNotification(domainDAO, templateName, messageChannel, errorChannel)
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
finishRESTServer := utils.StartRESTServer()
defer finishRESTServer()
utils.StartWebClient()
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
utils.ClearDatabase(database)
createScans(database)
retrieveScans()
retrieveScansWithPagination()
retrieveScansWithCache()
retrieveCurrentScan()
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var restConfig RESTHandlerDomainVerificationTestConfigFile
err := utils.ReadConfigFile(configFilePath, &restConfig)
config.ShelterConfig = restConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
restConfig.Database.URIs,
restConfig.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
domainDAO.RemoveAll()
utils.StartDNSServer(restConfig.DNSServerPort, restConfig.Scan.UDPMaxSize)
scanDomain()
scanPersistedDomain(domainDAO)
queryDomain()
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var config ScanDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
scanDAO := dao.ScanDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
scanDAO.RemoveAll()
scanLifeCycle(scanDAO)
scanConcurrency(scanDAO)
scanStatistics(scanDAO)
scansPagination(scanDAO)
scansExpand(scanDAO)
utils.Println("SUCCESS!")
}
// Notify is responsable for selecting the domains that should be notified in the system.
// It will send alert e-mails for each owner of a domain
func Notify() {
defer func() {
// Something went really wrong while notifying the owners. Log the error stacktrace
// and move out
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Printf("Panic detected while notifying the owners. Details: %v\n%s", r, buf)
}
}()
log.Info("Start notification job")
defer func() {
log.Info("End notification job")
}()
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
log.Println("Error while initializing database. Details:", err)
return
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
config.ShelterConfig.Notification.NameserverErrorAlertDays,
config.ShelterConfig.Notification.NameserverTimeoutAlertDays,
config.ShelterConfig.Notification.DSErrorAlertDays,
config.ShelterConfig.Notification.DSTimeoutAlertDays,
// TODO: Should we move this configuration parameter to a place were both modules can
// access it. This sounds better for configuration deployment
config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
)
if err != nil {
log.Println("Error retrieving domains to notify. Details:", err)
return
}
// Dispatch the asynchronous part of the method
for {
// Get domain from the database (one-by-one)
domainResult := <-domainChannel
// Detect errors while retrieving a specific domain. We are not going to stop all the
// process when only one domain got an error
if domainResult.Error != nil {
log.Println("Error retrieving domain to notify. Details:", domainResult.Error)
continue
}
// Problem detected while retrieving a domain or we don't have domains anymore
if domainResult.Error != nil || domainResult.Domain == nil {
break
}
if err := notifyDomain(domainResult.Domain); err != nil {
log.Println("Error notifying a domain. Details:", err)
}
}
}
func main() {
flag.Parse()
var scanConfig ScanTestConfigFile
err := utils.ReadConfigFile(configFilePath, &scanConfig)
config.ShelterConfig = scanConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
scanConfig.Database.URIs,
scanConfig.Database.Name,
scanConfig.Database.Auth.Enabled,
scanConfig.Database.Auth.Username,
scanConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// Remove all data before starting the test. This is necessary because maybe in the last
// test there was an error and the data wasn't removed from the database
utils.ClearDatabase(database)
server = utils.StartDNSServer(scanConfig.DNSServerPort, scanConfig.Scan.UDPMaxSize)
domainDAO := dao.DomainDAO{
Database: database,
}
// Register a scan only to avoid warning messages in the integration test execution
scheduler.Register(scheduler.Job{
Type: scheduler.JobTypeScan,
NextExecution: time.Now().Add(10 * time.Minute),
Task: func() {},
})
domainWithNoErrors(domainDAO)
domainWithNoErrorsOnTheFly()
domainQuery()
domainQueryWithDNSSECErrors()
// Scan performance report is optional and only generated when the report file
// path parameter is given
if report {
if cpuProfile {
f := utils.StartCPUProfile(scanConfig.Report.Profile.CPUFile)
defer f()
}
if memoryProfile {
f := utils.StartMemoryProfile(scanConfig.Report.Profile.MemoryFile)
defer f()
}
if goProfile {
f := utils.StartGoRoutinesProfile(scanConfig.Report.Profile.GoRoutinesFile)
defer f()
}
scanDAO := dao.ScanDAO{
Database: database,
}
scanReport(domainDAO, scanDAO, scanConfig)
}
// This test is after all others because it will ask on real nameservers, so we need to
// change to port 53
scan.DNSPort = 53
brDomainWithoutDNSSEC(domainDAO)
utils.Println("SUCCESS!")
}
func main() {
flag.Parse()
var restConfig RESTTestConfigFile
err := utils.ReadConfigFile(configFilePath, &restConfig)
config.ShelterConfig = restConfig.Config
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
// If there was some problem in the last test, there could be some data in the database,
// so let's clear it to don't affect this test. We avoid checking the error, because if
// the collection does not exist yet, it will be created in the first insert
utils.ClearDatabase(database)
finishRESTServer := utils.StartRESTServer()
defer finishRESTServer()
domainLifeCycle()
// REST performance report is optional and only generated when the report file path parameter is
// given
if report {
if cpuProfile {
f := utils.StartCPUProfile(restConfig.Report.Profile.CPUFile)
defer f()
}
if memoryProfile {
f := utils.StartMemoryProfile(restConfig.Report.Profile.MemoryFile)
defer f()
}
if goProfile {
f := utils.StartGoRoutinesProfile(restConfig.Report.Profile.GoRoutinesFile)
defer f()
}
restReport(restConfig)
}
utils.Println("SUCCESS!")
}
// Function responsible for running the domain scan system, checking the configuration of each
// domain in the database according to an algorithm. This method is synchronous and will return only
// after the scan proccess is done
func ScanDomains() {
defer func() {
// Something went really wrong while scanning the domains. Log the error stacktrace
// and move out
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Printf("Panic detected while scanning domains. Details: %v\n%s", r, buf)
}
}()
log.Info("Start scan job")
defer func() {
log.Info("End scan job")
}()
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
log.Println("Error while initializing database. Details:", err)
return
}
defer databaseSession.Close()
injector := NewInjector(
database,
config.ShelterConfig.Scan.DomainsBufferSize,
config.ShelterConfig.Scan.VerificationIntervals.MaxOKDays,
config.ShelterConfig.Scan.VerificationIntervals.MaxErrorDays,
config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
)
querierDispatcher := NewQuerierDispatcher(
config.ShelterConfig.Scan.NumberOfQueriers,
config.ShelterConfig.Scan.DomainsBufferSize,
config.ShelterConfig.Scan.UDPMaxSize,
time.Duration(config.ShelterConfig.Scan.Timeouts.DialSeconds)*time.Second,
time.Duration(config.ShelterConfig.Scan.Timeouts.ReadSeconds)*time.Second,
time.Duration(config.ShelterConfig.Scan.Timeouts.WriteSeconds)*time.Second,
config.ShelterConfig.Scan.ConnectionRetries,
)
collector := NewCollector(
database,
config.ShelterConfig.Scan.SaveAtOnce,
)
// Create a new scan information
model.StartNewScan()
var scanGroup sync.WaitGroup
errorsChannel := make(chan error, config.ShelterConfig.Scan.ErrorsBufferSize)
domainsToQueryChannel := injector.Start(&scanGroup, errorsChannel)
domainsToSaveChannel := querierDispatcher.Start(&scanGroup, domainsToQueryChannel)
collector.Start(&scanGroup, domainsToSaveChannel, errorsChannel)
// Keep track of errors for the scan information structure
errorDetected := false
go func() {
for {
select {
case err := <-errorsChannel:
// Detect the poison pill to finish the error listener go routine. This poison
// pill should be sent after all parts of the scan are done and we are sure that
// we don't have any error to log anymore
if err == nil {
return
} else {
errorDetected = true
log.Println("Error detected while executing the scan. Details:", err)
}
}
}
}()
// Wait for all parts of the scan to finish their job
scanGroup.Wait()
// Finish the error listener sending a poison pill
errorsChannel <- nil
scanDAO := dao.ScanDAO{
Database: database,
}
// Save the scan information for future reports
if err := model.FinishAndSaveScan(errorDetected, scanDAO.Save); err != nil {
log.Println("Error while saving scan information. Details:", err)
}
}