Exemplo n.º 1
0
func main() {
	flag.Parse()

	var config ScanCollectorTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &config)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		[]string{config.Database.URI},
		config.Database.Name,
		false, "", "",
	)
	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// Remove all data before starting the test. This is necessary because maybe in the last
	// test there was an error and the data wasn't removed from the database
	utils.ClearDatabase(database)

	domainWithErrors(config, database)
	domainWithNoErrors(config, database)

	utils.Println("SUCCESS!")
}
Exemplo n.º 2
0
func (i *Database) Before(w http.ResponseWriter, r *http.Request) {
	log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
	)

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Println("Error creating database connection. Details:", err)
		return
	}

	i.databaseHandler.SetDatabaseSession(databaseSession)
	i.databaseHandler.SetDatabase(database)
}
Exemplo n.º 3
0
func main() {
	flag.Parse()

	var config DomainDAOTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &config)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		[]string{config.Database.URI},
		config.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	domainDAO.RemoveAll()

	domainLifeCycle(domainDAO)
	domainsLifeCycle(domainDAO)
	domainUniqueFQDN(domainDAO)
	domainConcurrency(domainDAO)
	domainsPagination(domainDAO)
	domainsNotification(domainDAO)
	domainsExpand(domainDAO)
	domainFilter(domainDAO)

	// Domain DAO performance report is optional and only generated when the report file
	// path parameter is given
	if report {
		domainDAOPerformanceReport(config.Report.ReportFile, domainDAO)
	}

	utils.Println("SUCCESS!")
}
Exemplo n.º 4
0
func main() {
	flag.Parse()

	err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	utils.ClearDatabase(database)

	invalidFQDN(database)
	createDomain(database)
	updateDomain(database)
	retrieveDomain(database)
	retrieveDomainMetadata(database)
	retrieveDomainIfModifiedSince(database)
	retrieveDomainIfUnmodifiedSince(database)
	retrieveDomainIfMatch(database)
	retrieveDomainIfNoneMatch(database)
	updateDomainIfModifiedSince(database)
	updateDomainIfUnmodifiedSince(database)
	updateDomainIfMatch(database)
	updateDomainIfNoneMatch(database)
	deleteDomainIfModifiedSince(database)
	deleteDomainIfUnmodifiedSince(database)
	deleteDomainIfMatch(database)
	deleteDomainIfNoneMatch(database)
	deleteDomain(database)
	retrieveUnknownDomain(database)

	utils.Println("SUCCESS!")
}
Exemplo n.º 5
0
func main() {
	flag.Parse()

	err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// If there was some problem in the last test, there could be some data in the database,
	// so let's clear it to don't affect this test. We avoid checking the error, because if
	// the collection does not exist yet, it will be created in the first insert
	utils.ClearDatabase(database)

	messageChannel, errorChannel, err := utils.StartMailServer(config.ShelterConfig.Notification.SMTPServer.Port)
	if err != nil {
		utils.Fatalln("Error starting the mail server", err)
	}

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	templateName := createTemplateFile()
	defer removeTemplateFile(templateName)

	simpleNotification(domainDAO, templateName, messageChannel, errorChannel)

	utils.Println("SUCCESS!")
}
Exemplo n.º 6
0
func main() {
	flag.Parse()

	err := utils.ReadConfigFile(configFilePath, &config.ShelterConfig)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	finishRESTServer := utils.StartRESTServer()
	defer finishRESTServer()
	utils.StartWebClient()

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	utils.ClearDatabase(database)

	createScans(database)
	retrieveScans()
	retrieveScansWithPagination()
	retrieveScansWithCache()
	retrieveCurrentScan()

	utils.Println("SUCCESS!")
}
func main() {
	flag.Parse()

	var restConfig RESTHandlerDomainVerificationTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &restConfig)
	config.ShelterConfig = restConfig.Config

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		restConfig.Database.URIs,
		restConfig.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	domainDAO.RemoveAll()

	utils.StartDNSServer(restConfig.DNSServerPort, restConfig.Scan.UDPMaxSize)

	scanDomain()
	scanPersistedDomain(domainDAO)
	queryDomain()

	utils.Println("SUCCESS!")
}
Exemplo n.º 8
0
func main() {
	flag.Parse()

	var config ScanDAOTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &config)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		[]string{config.Database.URI},
		config.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	scanDAO := dao.ScanDAO{
		Database: database,
	}

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	scanDAO.RemoveAll()

	scanLifeCycle(scanDAO)
	scanConcurrency(scanDAO)
	scanStatistics(scanDAO)
	scansPagination(scanDAO)
	scansExpand(scanDAO)

	utils.Println("SUCCESS!")
}
Exemplo n.º 9
0
// Notify is responsable for selecting the domains that should be notified in the system.
// It will send alert e-mails for each owner of a domain
func Notify() {
	defer func() {
		// Something went really wrong while notifying the owners. Log the error stacktrace
		// and move out
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("Panic detected while notifying the owners. Details: %v\n%s", r, buf)
		}
	}()

	log.Info("Start notification job")
	defer func() {
		log.Info("End notification job")
	}()

	log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
	)

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		log.Println("Error while initializing database. Details:", err)
		return
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
		config.ShelterConfig.Notification.NameserverErrorAlertDays,
		config.ShelterConfig.Notification.NameserverTimeoutAlertDays,
		config.ShelterConfig.Notification.DSErrorAlertDays,
		config.ShelterConfig.Notification.DSTimeoutAlertDays,

		// TODO: Should we move this configuration parameter to a place were both modules can
		// access it. This sounds better for configuration deployment
		config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
	)

	if err != nil {
		log.Println("Error retrieving domains to notify. Details:", err)
		return
	}

	// Dispatch the asynchronous part of the method
	for {
		// Get domain from the database (one-by-one)
		domainResult := <-domainChannel

		// Detect errors while retrieving a specific domain. We are not going to stop all the
		// process when only one domain got an error
		if domainResult.Error != nil {
			log.Println("Error retrieving domain to notify. Details:", domainResult.Error)
			continue
		}

		// Problem detected while retrieving a domain or we don't have domains anymore
		if domainResult.Error != nil || domainResult.Domain == nil {
			break
		}

		if err := notifyDomain(domainResult.Domain); err != nil {
			log.Println("Error notifying a domain. Details:", err)
		}
	}
}
Exemplo n.º 10
0
func main() {
	flag.Parse()

	var scanConfig ScanTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &scanConfig)
	config.ShelterConfig = scanConfig.Config

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		scanConfig.Database.URIs,
		scanConfig.Database.Name,
		scanConfig.Database.Auth.Enabled,
		scanConfig.Database.Auth.Username,
		scanConfig.Database.Auth.Password,
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// Remove all data before starting the test. This is necessary because maybe in the last
	// test there was an error and the data wasn't removed from the database
	utils.ClearDatabase(database)

	server = utils.StartDNSServer(scanConfig.DNSServerPort, scanConfig.Scan.UDPMaxSize)

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	// Register a scan only to avoid warning messages in the integration test execution
	scheduler.Register(scheduler.Job{
		Type:          scheduler.JobTypeScan,
		NextExecution: time.Now().Add(10 * time.Minute),
		Task:          func() {},
	})

	domainWithNoErrors(domainDAO)
	domainWithNoErrorsOnTheFly()
	domainQuery()
	domainQueryWithDNSSECErrors()

	// Scan performance report is optional and only generated when the report file
	// path parameter is given
	if report {
		if cpuProfile {
			f := utils.StartCPUProfile(scanConfig.Report.Profile.CPUFile)
			defer f()
		}

		if memoryProfile {
			f := utils.StartMemoryProfile(scanConfig.Report.Profile.MemoryFile)
			defer f()
		}

		if goProfile {
			f := utils.StartGoRoutinesProfile(scanConfig.Report.Profile.GoRoutinesFile)
			defer f()
		}

		scanDAO := dao.ScanDAO{
			Database: database,
		}

		scanReport(domainDAO, scanDAO, scanConfig)
	}

	// This test is after all others because it will ask on real nameservers, so we need to
	// change to port 53
	scan.DNSPort = 53
	brDomainWithoutDNSSEC(domainDAO)

	utils.Println("SUCCESS!")
}
Exemplo n.º 11
0
func main() {
	flag.Parse()

	var restConfig RESTTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &restConfig)
	config.ShelterConfig = restConfig.Config

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	// If there was some problem in the last test, there could be some data in the database,
	// so let's clear it to don't affect this test. We avoid checking the error, because if
	// the collection does not exist yet, it will be created in the first insert
	utils.ClearDatabase(database)

	finishRESTServer := utils.StartRESTServer()
	defer finishRESTServer()

	domainLifeCycle()

	// REST performance report is optional and only generated when the report file path parameter is
	// given
	if report {
		if cpuProfile {
			f := utils.StartCPUProfile(restConfig.Report.Profile.CPUFile)
			defer f()
		}

		if memoryProfile {
			f := utils.StartMemoryProfile(restConfig.Report.Profile.MemoryFile)
			defer f()
		}

		if goProfile {
			f := utils.StartGoRoutinesProfile(restConfig.Report.Profile.GoRoutinesFile)
			defer f()
		}

		restReport(restConfig)
	}

	utils.Println("SUCCESS!")
}
Exemplo n.º 12
0
// Function responsible for running the domain scan system, checking the configuration of each
// domain in the database according to an algorithm. This method is synchronous and will return only
// after the scan proccess is done
func ScanDomains() {
	defer func() {
		// Something went really wrong while scanning the domains. Log the error stacktrace
		// and move out
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("Panic detected while scanning domains. Details: %v\n%s", r, buf)
		}
	}()

	log.Info("Start scan job")
	defer func() {
		log.Info("End scan job")
	}()

	log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
	)

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		log.Println("Error while initializing database. Details:", err)
		return
	}
	defer databaseSession.Close()

	injector := NewInjector(
		database,
		config.ShelterConfig.Scan.DomainsBufferSize,
		config.ShelterConfig.Scan.VerificationIntervals.MaxOKDays,
		config.ShelterConfig.Scan.VerificationIntervals.MaxErrorDays,
		config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
	)

	querierDispatcher := NewQuerierDispatcher(
		config.ShelterConfig.Scan.NumberOfQueriers,
		config.ShelterConfig.Scan.DomainsBufferSize,
		config.ShelterConfig.Scan.UDPMaxSize,
		time.Duration(config.ShelterConfig.Scan.Timeouts.DialSeconds)*time.Second,
		time.Duration(config.ShelterConfig.Scan.Timeouts.ReadSeconds)*time.Second,
		time.Duration(config.ShelterConfig.Scan.Timeouts.WriteSeconds)*time.Second,
		config.ShelterConfig.Scan.ConnectionRetries,
	)

	collector := NewCollector(
		database,
		config.ShelterConfig.Scan.SaveAtOnce,
	)

	// Create a new scan information
	model.StartNewScan()

	var scanGroup sync.WaitGroup
	errorsChannel := make(chan error, config.ShelterConfig.Scan.ErrorsBufferSize)
	domainsToQueryChannel := injector.Start(&scanGroup, errorsChannel)
	domainsToSaveChannel := querierDispatcher.Start(&scanGroup, domainsToQueryChannel)
	collector.Start(&scanGroup, domainsToSaveChannel, errorsChannel)

	// Keep track of errors for the scan information structure
	errorDetected := false

	go func() {
		for {
			select {
			case err := <-errorsChannel:
				// Detect the poison pill to finish the error listener go routine. This poison
				// pill should be sent after all parts of the scan are done and we are sure that
				// we don't have any error to log anymore
				if err == nil {
					return

				} else {
					errorDetected = true
					log.Println("Error detected while executing the scan. Details:", err)
				}
			}
		}
	}()

	// Wait for all parts of the scan to finish their job
	scanGroup.Wait()

	// Finish the error listener sending a poison pill
	errorsChannel <- nil

	scanDAO := dao.ScanDAO{
		Database: database,
	}

	// Save the scan information for future reports
	if err := model.FinishAndSaveScan(errorDetected, scanDAO.Save); err != nil {
		log.Println("Error while saving scan information. Details:", err)
	}
}