Beispiel #1
0
// Function to mock a domain
func generateAndSaveDomain(fqdn string, domainDAO dao.DomainDAO, language string) {
	lastOKAt := time.Now().Add(time.Duration(-config.ShelterConfig.Notification.NameserverErrorAlertDays*24) * time.Hour)
	owner, _ := mail.ParseAddress("*****@*****.**")

	domain := model.Domain{
		FQDN: fqdn,
		Nameservers: []model.Nameserver{
			{
				Host:       fmt.Sprintf("ns1.%s", fqdn),
				IPv4:       net.ParseIP("127.0.0.1"),
				LastStatus: model.NameserverStatusServerFailure,
				LastOKAt:   lastOKAt,
			},
		},
		Owners: []model.Owner{
			{
				Email:    owner,
				Language: language,
			},
		},
	}

	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln(fmt.Sprintf("Fail to save domain %s", domain.FQDN), err)
	}
}
Beispiel #2
0
// Generates a report with the amount of time of a scan
func scanReport(domainDAO dao.DomainDAO, scanDAO dao.ScanDAO, scanConfig ScanTestConfigFile) {
	report := " #       | Total            | DPS  | Memory (MB)\n" +
		"-----------------------------------------------------\n"

	// Report variables
	scale := []int{10, 50, 100, 500, 1000, 5000,
		10000, 50000, 100000, 500000, 1000000, 5000000}

	dnskey, privateKey, err := utils.GenerateKey()
	if err != nil {
		utils.Fatalln("Error generating DNSKEY", err)
	}

	reportHandler := ReportHandler{
		DNSKEY:     dnskey,
		PrivateKey: privateKey,
	}

	server.Handler = reportHandler
	dns.DefaultServeMux = nil

	for _, numberOfItems := range scale {
		utils.Println(fmt.Sprintf("Generating report - scale %d", numberOfItems))

		for i := 0; i < numberOfItems; i++ {
			if i%1000 == 0 {
				utils.PrintProgress("building scenario", (i*100)/numberOfItems)
			}

			fqdn := fmt.Sprintf("domain%d.br.", i)
			generateAndSaveDomain(fqdn, domainDAO, dnskey)
		}

		utils.PrintProgress("building scenario", 100)
		totalDuration, domainsPerSecond := calculateScanDurations(numberOfItems, scanDAO)

		var memStats runtime.MemStats
		runtime.ReadMemStats(&memStats)

		report += fmt.Sprintf("% -8d | %16s | %4d | %14.2f\n",
			numberOfItems,
			time.Duration(int64(totalDuration)).String(),
			domainsPerSecond,
			float64(memStats.Alloc)/float64(MB),
		)

		if err := domainDAO.RemoveAll(); err != nil {
			// When the result set is too big to remove, we got a timeout error from the
			// connection, but it's ok
			//utils.Fatalln("Error removing domains generated for report", err)
		}
	}

	utils.WriteReport(scanConfig.Report.File, report)
}
Beispiel #3
0
// Function created to remove all entries from the database to ensure that the tests
// enviroments are always equal
func ClearDatabase(database *mgo.Database) {
	domainDAO := dao.DomainDAO{
		Database: database,
	}
	domainDAO.RemoveAll()

	scanDAO := dao.ScanDAO{
		Database: database,
	}
	scanDAO.RemoveAll()
}
Beispiel #4
0
func main() {
	flag.Parse()

	var config DomainDAOTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &config)

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		[]string{config.Database.URI},
		config.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	domainDAO.RemoveAll()

	domainLifeCycle(domainDAO)
	domainsLifeCycle(domainDAO)
	domainUniqueFQDN(domainDAO)
	domainConcurrency(domainDAO)
	domainsPagination(domainDAO)
	domainsNotification(domainDAO)
	domainsExpand(domainDAO)
	domainFilter(domainDAO)

	// Domain DAO performance report is optional and only generated when the report file
	// path parameter is given
	if report {
		domainDAOPerformanceReport(config.Report.ReportFile, domainDAO)
	}

	utils.Println("SUCCESS!")
}
Beispiel #5
0
// Function to mock a domain
func generateSignAndSaveDomain(fqdn string, domainDAO dao.DomainDAO) (
	model.Domain, *dns.DNSKEY, *dns.RRSIG, time.Time, time.Time,
) {
	domain, dnskey, rrsig, lastCheckAt, lastOKAt := generateAndSignDomain(fqdn)

	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Error saving domain", err)
	}

	return domain, dnskey, rrsig, lastCheckAt, lastOKAt
}
Beispiel #6
0
// Function to mock a domain
func generateAndSaveDomain(fqdn string, domainDAO dao.DomainDAO, dnskey *dns.DNSKEY) {
	ds := dnskey.ToDS(uint8(model.DSDigestTypeSHA1))

	domain := model.Domain{
		FQDN: fqdn,
		Nameservers: []model.Nameserver{
			{
				Host: fmt.Sprintf("ns1.%s", fqdn),
				IPv4: net.ParseIP("127.0.0.1"),
			},
		},
		DSSet: []model.DS{
			{
				Keytag:     dnskey.KeyTag(),
				Algorithm:  utils.ConvertKeyAlgorithm(dnskey.Algorithm),
				DigestType: model.DSDigestTypeSHA1,
				Digest:     ds.Digest,
			},
		},
	}

	owner, _ := mail.ParseAddress("*****@*****.**")
	domain.Owners = []model.Owner{
		{
			Email:    owner,
			Language: "pt-BR",
		},
	}

	lastCheckAt := time.Now().Add(-72 * time.Hour)
	lastOKAt := lastCheckAt.Add(-24 * time.Hour)

	// Set all nameservers with error and the last check equal of the error check interval,
	// this will force the domain to be checked
	for index, _ := range domain.Nameservers {
		domain.Nameservers[index].LastCheckAt = lastCheckAt
		domain.Nameservers[index].LastOKAt = lastOKAt
		domain.Nameservers[index].LastStatus = model.NameserverStatusServerFailure
	}

	// Set all DS records with error and the last check equal of the error check interval,
	// this will force the domain to be checked
	for index, _ := range domain.DSSet {
		domain.DSSet[index].LastCheckAt = lastCheckAt
		domain.DSSet[index].LastOKAt = lastOKAt
		domain.DSSet[index].LastStatus = model.DSStatusTimeout
	}

	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln(fmt.Sprintf("Fail to save domain %s", domain.FQDN), err)
	}
}
Beispiel #7
0
func (h *DomainHandler) Delete(w http.ResponseWriter, r *http.Request) {
	domainDAO := dao.DomainDAO{
		Database: h.GetDatabase(),
	}

	if err := domainDAO.Remove(&h.domain); err != nil {
		log.Println("Error while removing domain object. Details:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
Beispiel #8
0
func (i *Domain) Before(w http.ResponseWriter, r *http.Request) {
	domainDAO := dao.DomainDAO{
		Database: i.domainHandler.GetDatabase(),
	}

	domain, err := domainDAO.FindByFQDN(i.domainHandler.GetFQDN())

	// For PUT method if the domain does not exist yet thats alright because we will create
	// it
	if r.Method != "PUT" && err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	i.domainHandler.SetDomain(domain)
}
func main() {
	flag.Parse()

	var restConfig RESTHandlerDomainVerificationTestConfigFile
	err := utils.ReadConfigFile(configFilePath, &restConfig)
	config.ShelterConfig = restConfig.Config

	if err == utils.ErrConfigFileUndefined {
		fmt.Println(err.Error())
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return

	} else if err != nil {
		utils.Fatalln("Error reading configuration file", err)
	}

	database, databaseSession, err := mongodb.Open(
		restConfig.Database.URIs,
		restConfig.Database.Name,
		false, "", "",
	)

	if err != nil {
		utils.Fatalln("Error connecting the database", err)
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	// If there was some problem in the last test, there could be some data in the
	// database, so let's clear it to don't affect this test. We avoid checking the error,
	// because if the collection does not exist yet, it will be created in the first
	// insert
	domainDAO.RemoveAll()

	utils.StartDNSServer(restConfig.DNSServerPort, restConfig.Scan.UDPMaxSize)

	scanDomain()
	scanPersistedDomain(domainDAO)
	queryDomain()

	utils.Println("SUCCESS!")
}
Beispiel #10
0
func domainWithNoErrors(config ScanInjectorTestConfigFile, domainDAO dao.DomainDAO) {
	domain := newDomain()

	// Set all nameservers as configured correctly and the last check as now, this domain is
	// unlikely to be selected
	for index, _ := range domain.Nameservers {
		domain.Nameservers[index].LastCheckAt = time.Now()
		domain.Nameservers[index].LastStatus = model.NameserverStatusOK
	}

	// Set all DS records as configured correctly and the last check as now, this domain is
	// unlikely to be selected
	for index, _ := range domain.DSSet {
		domain.DSSet[index].LastCheckAt = time.Now()
		domain.DSSet[index].LastStatus = model.DSStatusOK
	}

	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Error saving domain for scan scenario", err)
	}

	model.StartNewScan()
	if domains := runScan(config, domainDAO); len(domains) > 0 {
		utils.Fatalln(fmt.Sprintf("Selected a domain configured correctly for the scan. "+
			"Expected 0 got %d", len(domains)), nil)
	}

	currentScan := model.GetCurrentScan()
	if currentScan.Status != model.ScanStatusRunning {
		utils.Fatalln("Not changing the scan info status for domain with no errors", nil)
	}

	if currentScan.DomainsToBeScanned > 0 {
		utils.Fatalln("Not counting the domains to be scanned for domain with no errors", nil)
	}

	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Error removing domain", err)
	}
}
Beispiel #11
0
func domainWithDNSErrors(config ScanInjectorTestConfigFile, domainDAO dao.DomainDAO) {
	domain := newDomain()

	// Set all nameservers with error and the last check equal of the error check interval,
	// this will force the domain to be checked
	for index, _ := range domain.Nameservers {
		maxErrorHours := config.Scan.VerificationIntervals.MaxErrorDays * 24
		lessThreeDays, _ := time.ParseDuration("-" + strconv.Itoa(maxErrorHours) + "h")

		domain.Nameservers[index].LastCheckAt = time.Now().Add(lessThreeDays)
		domain.Nameservers[index].LastStatus = model.NameserverStatusServerFailure
	}

	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Error saving domain for scan scenario", err)
	}

	model.StartNewScan()
	if domains := runScan(config, domainDAO); len(domains) != 1 {
		utils.Fatalln(fmt.Sprintf("Couldn't load a domain with DNS errors for scan. "+
			"Expected 1 got %d", len(domains)), nil)
	}

	currentScan := model.GetCurrentScan()
	if currentScan.Status != model.ScanStatusRunning {
		utils.Fatalln("Not changing the scan info status with DNS errors", nil)
	}

	if currentScan.DomainsToBeScanned != 1 {
		utils.Fatalln("Not counting the domains to be scanned with DNS errors", nil)
	}

	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Error removing domain", err)
	}
}
Beispiel #12
0
func domainFilter(domainDAO dao.DomainDAO) {
	numberOfItems := 20

	for i := 0; i < numberOfItems; i++ {
		domain := model.Domain{
			FQDN: fmt.Sprintf("example%d.com.br", i),
		}

		if err := domainDAO.Save(&domain); err != nil {
			utils.Fatalln("Error saving domain in database", err)
		}
	}

	pagination := dao.DomainDAOPagination{
		PageSize: 10,
		Page:     5,
		OrderBy: []dao.DomainDAOSort{
			{
				Field:     dao.DomainDAOOrderByFieldFQDN,
				Direction: dao.DAOOrderByDirectionAscending,
			},
		},
	}

	domains, err := domainDAO.FindAll(&pagination, true, "example1\\.com.*")
	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	if len(domains) != 1 {
		utils.Fatalln(fmt.Sprintf("Wrong number of domains when there's filter. "+
			"Expected '1' and got '%d'", len(domains)), nil)
	}

	if domains[0].FQDN != "example1.com.br" {
		utils.Fatalln("Wrong domain returned", nil)
	}

	for i := 0; i < numberOfItems; i++ {
		fqdn := fmt.Sprintf("example%d.com.br", i)
		if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
			utils.Fatalln("Error removing domain from database", err)
		}
	}
}
Beispiel #13
0
// FQDN must be unique in the database, today we limit this using an unique index key
func domainUniqueFQDN(domainDAO dao.DomainDAO) {
	domain1 := newDomain()

	// Create domain
	if err := domainDAO.Save(&domain1); err != nil {
		utils.Fatalln("Couldn't save domain in database", err)
	}

	domain2 := newDomain()

	// Create another domain with the same FQDN
	if err := domainDAO.Save(&domain2); err == nil {
		utils.Fatalln("Allowing more than one object with the same FQDN", nil)
	}

	// Remove domain
	if err := domainDAO.RemoveByFQDN(domain1.FQDN); err != nil {
		utils.Fatalln("Error while trying to remove a domain", err)
	}
}
Beispiel #14
0
// Check if the revision field avoid data concurrency. Is better to fail than to store the
// wrong state
func domainConcurrency(domainDAO dao.DomainDAO) {
	domain := newDomain()

	// Create domain
	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Couldn't save domain in database", err)
	}

	domain1, err := domainDAO.FindByFQDN(domain.FQDN)
	if err != nil {
		utils.Fatalln("Couldn't find created domain in database", err)
	}

	domain2, err := domainDAO.FindByFQDN(domain.FQDN)
	if err != nil {
		utils.Fatalln("Couldn't find created domain in database", err)
	}

	if err := domainDAO.Save(&domain1); err != nil {
		utils.Fatalln("Couldn't save domain in database", err)
	}

	if err := domainDAO.Save(&domain2); err == nil {
		utils.Fatalln("Not controlling domain concurrency", nil)
	}

	// Remove domain
	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Error while trying to remove a domain", err)
	}
}
Beispiel #15
0
// Test all phases from a domain lyfe cycle, but now working with a group of domains
func domainsLifeCycle(domainDAO dao.DomainDAO) {
	domains := newDomains()

	// Create domains
	domainResults := domainDAO.SaveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	for _, domain := range domains {
		// Search and compare created domains
		if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't find created domain %s in database", domain.FQDN), err)

		} else if !utils.CompareDomain(*domain, domainRetrieved) {
			utils.Fatalln(fmt.Sprintf("Domain %s created is being persisted wrongly", domain.FQDN), nil)
		}
	}

	// Update domains
	for _, domain := range domains {
		domain.Owners = []model.Owner{}
	}

	domainResults = domainDAO.SaveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't update domain %s in database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	for _, domain := range domains {
		// Search and compare updated domains
		if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't find updated domain %s in database", domain.FQDN), err)

		} else if !utils.CompareDomain(*domain, domainRetrieved) {
			utils.Fatalln(fmt.Sprintf("Domain %s updated in being persisted wrongly", domain.FQDN), nil)
		}
	}

	// Check if find all really return all domains
	allDomainsChannel, err := domainDAO.FindAllAsync()
	if err != nil {
		utils.Fatalln("Error while retrieving all domains from database", err)
	}

	var allDomains []model.Domain
	for {
		domainRetrieved := <-allDomainsChannel
		if domainRetrieved.Error != nil {
			utils.Fatalln("Error while retrieving all domains from database", err)
		} else if domainRetrieved.Domain == nil {
			break
		}

		allDomains = append(allDomains, *domainRetrieved.Domain)
	}

	if len(allDomains) != len(domains) {
		utils.Fatalln(fmt.Sprintf("FindAll method is not returning all domains we expected %d but got %d",
			len(domains), len(allDomains)), nil)
	}

	// Detected a problem in FindAsync method on 2014-01-17 where we were returning the same
	// object many times because we were reusing the same pointer. For that reason we are
	// going to add a test to check if the items returned are the same set of the inserted
	// ones
	for _, domain := range domains {
		found := false
		for _, domainRetrieved := range allDomains {
			if domainRetrieved.Id.Hex() == domain.Id.Hex() {
				found = true
				break
			}
		}

		if !found {
			utils.Fatalln("FindAll method is not returning all objects "+
				"that were inserted, apparently there are duplicated objects in the result set", nil)
		}
	}

	// Remove domains
	domainResults = domainDAO.RemoveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Error while trying to remove domain %s from database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	for _, domain := range domains {
		// Check removals
		if _, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
			utils.Fatalln(fmt.Sprintf("Domain %s was not removed from database", domain.FQDN), nil)
		}
	}

	// Let's add and remove the domains again to test the remove all method

	domainResults = domainDAO.SaveMany(domains)
	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	if err := domainDAO.RemoveAll(); err != nil {
		utils.Fatalln("Couldn't remove all domains", err)
	}

	allDomainsChannel, err = domainDAO.FindAllAsync()
	if err != nil {
		utils.Fatalln("Error while retrieving all domains from database", err)
	}

	allDomains = []model.Domain{}
	for {
		domainRetrieved := <-allDomainsChannel
		if domainRetrieved.Error != nil {
			utils.Fatalln("Error while retrieving all domains from database", err)
		} else if domainRetrieved.Domain == nil {
			break
		}

		allDomains = append(allDomains, *domainRetrieved.Domain)
	}

	if len(allDomains) > 0 {
		utils.Fatalln("RemoveAll method is not removing the domains from the database", nil)
	}
}
Beispiel #16
0
// Test all phases of the domain life cycle
func domainLifeCycle(domainDAO dao.DomainDAO) {
	domain := newDomain()

	// Create domain
	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Couldn't save domain in database", err)
	}

	// Search and compare created domain
	if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Couldn't find created domain in database", err)

	} else if !utils.CompareDomain(domain, domainRetrieved) {
		utils.Fatalln("Domain created is being persisted wrongly", nil)
	}

	// Update domain
	domain.Owners = []model.Owner{}
	if err := domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Couldn't save domain in database", err)
	}

	// Search and compare updated domain
	if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Couldn't find updated domain in database", err)

	} else if !utils.CompareDomain(domain, domainRetrieved) {
		utils.Fatalln("Domain updated is being persisted wrongly", nil)
	}

	// Remove domain
	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln("Error while trying to remove a domain", err)
	}

	// Check removal
	if _, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
		utils.Fatalln("Domain was not removed from database", nil)
	}
}
Beispiel #17
0
// Notify is responsable for selecting the domains that should be notified in the system.
// It will send alert e-mails for each owner of a domain
func Notify() {
	defer func() {
		// Something went really wrong while notifying the owners. Log the error stacktrace
		// and move out
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("Panic detected while notifying the owners. Details: %v\n%s", r, buf)
		}
	}()

	log.Info("Start notification job")
	defer func() {
		log.Info("End notification job")
	}()

	log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
	)

	database, databaseSession, err := mongodb.Open(
		config.ShelterConfig.Database.URIs,
		config.ShelterConfig.Database.Name,
		config.ShelterConfig.Database.Auth.Enabled,
		config.ShelterConfig.Database.Auth.Username,
		config.ShelterConfig.Database.Auth.Password,
	)

	if err != nil {
		log.Println("Error while initializing database. Details:", err)
		return
	}
	defer databaseSession.Close()

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
		config.ShelterConfig.Notification.NameserverErrorAlertDays,
		config.ShelterConfig.Notification.NameserverTimeoutAlertDays,
		config.ShelterConfig.Notification.DSErrorAlertDays,
		config.ShelterConfig.Notification.DSTimeoutAlertDays,

		// TODO: Should we move this configuration parameter to a place were both modules can
		// access it. This sounds better for configuration deployment
		config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
	)

	if err != nil {
		log.Println("Error retrieving domains to notify. Details:", err)
		return
	}

	// Dispatch the asynchronous part of the method
	for {
		// Get domain from the database (one-by-one)
		domainResult := <-domainChannel

		// Detect errors while retrieving a specific domain. We are not going to stop all the
		// process when only one domain got an error
		if domainResult.Error != nil {
			log.Println("Error retrieving domain to notify. Details:", domainResult.Error)
			continue
		}

		// Problem detected while retrieving a domain or we don't have domains anymore
		if domainResult.Error != nil || domainResult.Domain == nil {
			break
		}

		if err := notifyDomain(domainResult.Domain); err != nil {
			log.Println("Error notifying a domain. Details:", err)
		}
	}
}
Beispiel #18
0
// The HEAD method is identical to GET except that the server MUST NOT return a message-
// body in the response. But now the responsability for don't adding the body is from the
// mux while writing the response
func (h *DomainsHandler) retrieveDomains(w http.ResponseWriter, r *http.Request) {
	var pagination dao.DomainDAOPagination
	expand := false
	filter := ""

	for key, values := range r.URL.Query() {
		key = strings.TrimSpace(key)
		key = strings.ToLower(key)

		// A key can have multiple values in a query string, we are going to always consider
		// the last one (overwrite strategy)
		for _, value := range values {
			value = strings.TrimSpace(value)
			value = strings.ToLower(value)

			switch key {
			case "orderby":
				// OrderBy parameter will store the fields that the user want to be the keys of the sort
				// algorithm in the result set and the direction that each sort field will have. The format
				// that will be used is:
				//
				// <field1>:<direction1>@<field2>:<direction2>@...@<fieldN>:<directionN>

				orderByParts := strings.Split(value, "@")

				for _, orderByPart := range orderByParts {
					orderByPart = strings.TrimSpace(orderByPart)
					orderByAndDirection := strings.Split(orderByPart, ":")

					var field, direction string

					if len(orderByAndDirection) == 1 {
						field, direction = orderByAndDirection[0], "asc"

					} else if len(orderByAndDirection) == 2 {
						field, direction = orderByAndDirection[0], orderByAndDirection[1]

					} else {
						if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
							w.WriteHeader(http.StatusBadRequest)

						} else {
							log.Println("Error while writing response. Details:", err)
							w.WriteHeader(http.StatusInternalServerError)
						}
						return
					}

					orderByField, err := dao.DomainDAOOrderByFieldFromString(field)
					if err != nil {
						if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
							w.WriteHeader(http.StatusBadRequest)

						} else {
							log.Println("Error while writing response. Details:", err)
							w.WriteHeader(http.StatusInternalServerError)
						}
						return
					}

					orderByDirection, err := dao.DAOOrderByDirectionFromString(direction)
					if err != nil {
						if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
							w.WriteHeader(http.StatusBadRequest)

						} else {
							log.Println("Error while writing response. Details:", err)
							w.WriteHeader(http.StatusInternalServerError)
						}
						return
					}

					pagination.OrderBy = append(pagination.OrderBy, dao.DomainDAOSort{
						Field:     orderByField,
						Direction: orderByDirection,
					})
				}

			case "pagesize":
				var err error
				pagination.PageSize, err = strconv.Atoi(value)
				if err != nil {
					if err := h.MessageResponse("invalid-query-page-size", ""); err == nil {
						w.WriteHeader(http.StatusBadRequest)

					} else {
						log.Println("Error while writing response. Details:", err)
						w.WriteHeader(http.StatusInternalServerError)
					}
					return
				}

			case "page":
				var err error
				pagination.Page, err = strconv.Atoi(value)
				if err != nil {
					if err := h.MessageResponse("invalid-query-page", ""); err == nil {
						w.WriteHeader(http.StatusBadRequest)

					} else {
						log.Println("Error while writing response. Details:", err)
						w.WriteHeader(http.StatusInternalServerError)
					}
					return
				}

			case "expand":
				expand = true

			case "filter":
				filter = value
			}
		}
	}

	domainDAO := dao.DomainDAO{
		Database: h.GetDatabase(),
	}

	domains, err := domainDAO.FindAll(&pagination, expand, filter)
	if err != nil {
		log.Println("Error while filtering domains objects. Details:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	domainsResponse := protocol.ToDomainsResponse(domains, pagination, expand, filter)
	h.Response = &domainsResponse

	// Last-Modified is going to be the most recent date of the list
	for _, domain := range domains {
		if domain.LastModifiedAt.After(h.lastModifiedAt) {
			h.lastModifiedAt = domain.LastModifiedAt
		}
	}

	w.Header().Add("ETag", h.GetETag())
	w.Header().Add("Last-Modified", h.lastModifiedAt.Format(time.RFC1123))
	w.WriteHeader(http.StatusOK)
}
// Put is responsable for checking a domain object on-the-fly without persisting in
// database, useful for pre-registration validations in the registry
func (h *DomainVerificationHandler) Put(w http.ResponseWriter, r *http.Request) {
	// We need to set the FQDN in the domain request object because it is sent only in the
	// URI and not in the domain request body to avoid information redudancy
	h.Request.FQDN = h.GetFQDN()

	var domain model.Domain
	var err error

	if domain, err = protocol.Merge(domain, h.Request); err != nil {
		messageId := ""

		switch err {
		case protocol.ErrInvalidDNSKEY:
			messageId = "invalid-dnskey"
		case protocol.ErrInvalidDSAlgorithm:
			messageId = "invalid-ds-algorithm"
		case protocol.ErrInvalidDSDigestType:
			messageId = "invalid-ds-digest-type"
		case protocol.ErrInvalidIP:
			messageId = "invalid-ip"
		case protocol.ErrInvalidLanguage:
			messageId = "invalid-language"
		}

		if len(messageId) == 0 {
			log.Println("Error while merging domain objects for domain verification "+
				"operation. Details:", err)
			w.WriteHeader(http.StatusInternalServerError)

		} else {
			if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
				w.WriteHeader(http.StatusBadRequest)

			} else {
				log.Println("Error while writing response. Details:", err)
				w.WriteHeader(http.StatusInternalServerError)
			}
		}
		return
	}

	scan.ScanDomain(&domain)

	// As we alredy did the scan, if the domain is registered in the system, we update it for this
	// results. This also gives a more intuitive design for when the user wants to force a check a
	// specific domain in the Shelter system
	domainDAO := dao.DomainDAO{
		Database: h.GetDatabase(),
	}

	if dbDomain, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
		update := true

		// Check if we have the same nameservers, and if so update the last status
		if len(dbDomain.Nameservers) == len(domain.Nameservers) {
			for i := range dbDomain.Nameservers {
				dbNameserver := dbDomain.Nameservers[i]
				nameserver := domain.Nameservers[i]

				if dbNameserver.Host == nameserver.Host &&
					dbNameserver.IPv4.Equal(nameserver.IPv4) &&
					dbNameserver.IPv6.Equal(nameserver.IPv6) {

					dbDomain.Nameservers[i].ChangeStatus(nameserver.LastStatus)

				} else {
					update = false
					break
				}
			}

		} else {
			update = false
		}

		// Check if we have the same DS set, and if so update the last status
		if len(dbDomain.DSSet) == len(domain.DSSet) {
			for i := range dbDomain.DSSet {
				dbDS := dbDomain.DSSet[i]
				ds := domain.DSSet[i]

				if dbDS.Keytag == ds.Keytag &&
					dbDS.Algorithm == ds.Algorithm &&
					dbDS.DigestType == ds.DigestType &&
					dbDS.Digest == ds.Digest {

					dbDomain.DSSet[i].ChangeStatus(ds.LastStatus)

				} else {
					update = false
					break
				}
			}

		} else {
			update = false
		}

		if update {
			// We don't care about errors resulted here, because the main idea of this service is to scan
			// a domaion, not persist the results
			domainDAO.Save(&dbDomain)
		}
	}

	w.WriteHeader(http.StatusOK)
	domainResponse := protocol.ToDomainResponse(domain, false)
	h.Response = &domainResponse
}
Beispiel #20
0
// Method that starts the injector job, retrieving the data from the database and adding
// the same data into a channel for a querier start sending DNS requests. There are two
// parameters, one to control the scan go routines and sinalize to the main thread the
// end, and other to define a channel to report errors while loading the data. This method
// is asynchronous and will finish sending a poison pill (error or nil domain) to indicate
// to the querier that there are no more domains
func (i *Injector) Start(scanGroup *sync.WaitGroup, errorsChannel chan error) chan *model.Domain {

	// Create the output channel where we are going to add the domains retrieved from the
	// database for the querier
	domainsToQueryChannel := make(chan *model.Domain, i.DomainsBufferSize)

	// Add one more to the group of scan go routines
	scanGroup.Add(1)

	go func() {
		// Initialize Domain DAO using injected database connection
		domainDAO := dao.DomainDAO{
			Database: i.Database,
		}

		// Load all domains from database to begin the scan
		domainChannel, err := domainDAO.FindAllAsync()

		// Low level error was detected. No domain was processed yet, but we still need to
		// shutdown the querier and by consequence the collector, so we send back the error
		// and add the poison pill
		if err != nil {
			errorsChannel <- err
			domainsToQueryChannel <- nil

			// Tells the scan information structure that the injector is done
			model.FinishLoadingDomainsForScan()

			scanGroup.Done()
			return
		}

		// Dispatch the asynchronous part of the method
		for {
			// Get domain from the database (one-by-one)
			domainResult := <-domainChannel

			// Send back the error to the caller thread. We don't log the error here directly
			// into the log interface because sometimes we want to do something when an error
			// occurs, like in a test enviroment
			if domainResult.Error != nil {
				errorsChannel <- domainResult.Error
			}

			// Problem detected while retrieving a domain or we don't have domains anymore, send
			// the poison pill to alert the querier
			if domainResult.Error != nil || domainResult.Domain == nil {
				domainsToQueryChannel <- nil

				// Tells the scan information structure that the injector is done
				model.FinishLoadingDomainsForScan()

				scanGroup.Done()
				return
			}

			// The logic that decides if a domain is going to be a part of this scan or not is
			// inside the domain object for better unit testing
			if domainResult.Domain.ShouldBeScanned(i.MaxOKVerificationDays,
				i.MaxErrorVerificationDays, i.MaxExpirationAlertDays) {
				// Send to the querier
				domainsToQueryChannel <- domainResult.Domain

				// Count domain for the scan information to estimate the scan progress
				model.LoadedDomainForScan()
			}
		}
	}()

	return domainsToQueryChannel
}
Beispiel #21
0
func (h *DomainHandler) Put(w http.ResponseWriter, r *http.Request) {
	// We need to set the FQDN in the domain request object because it is sent only in the
	// URI and not in the domain request body to avoid information redudancy
	h.Request.FQDN = h.GetFQDN()

	var err error
	if h.domain, err = protocol.Merge(h.domain, h.Request); err != nil {
		messageId := ""

		switch err {
		case model.ErrInvalidFQDN:
			messageId = "invalid-fqdn"
		case protocol.ErrInvalidDNSKEY:
			messageId = "invalid-dnskey"
		case protocol.ErrInvalidDSAlgorithm:
			messageId = "invalid-ds-algorithm"
		case protocol.ErrInvalidDSDigestType:
			messageId = "invalid-ds-digest-type"
		case protocol.ErrInvalidIP:
			messageId = "invalid-ip"
		case protocol.ErrInvalidLanguage:
			messageId = "invalid-language"
		}

		if len(messageId) == 0 {
			log.Println("Error while merging domain objects for create or "+
				"update operation. Details:", err)
			w.WriteHeader(http.StatusInternalServerError)

		} else {
			if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
				w.WriteHeader(http.StatusBadRequest)

			} else {
				log.Println("Error while writing response. Details:", err)
				w.WriteHeader(http.StatusInternalServerError)
			}
		}
		return
	}

	domainDAO := dao.DomainDAO{
		Database: h.GetDatabase(),
	}

	if err := domainDAO.Save(&h.domain); err != nil {
		if strings.Index(err.Error(), "duplicate key error index") != -1 {
			if err := h.MessageResponse("conflict", r.URL.RequestURI()); err == nil {
				w.WriteHeader(http.StatusConflict)

			} else {
				log.Println("Error while writing response. Details:", err)
				w.WriteHeader(http.StatusInternalServerError)
			}

		} else {
			log.Println("Error while saving domain object for create or "+
				"update operation. Details:", err)
			w.WriteHeader(http.StatusInternalServerError)
		}

		return
	}

	w.Header().Add("ETag", h.GetETag())
	w.Header().Add("Last-Modified", h.GetLastModifiedAt().Format(time.RFC1123))

	if h.domain.Revision == 1 {
		w.Header().Add("Location", "/domain/"+h.domain.FQDN)
		w.WriteHeader(http.StatusCreated)

	} else {
		w.WriteHeader(http.StatusNoContent)
	}
}
Beispiel #22
0
func domainWithNoErrors(domainDAO dao.DomainDAO) {
	domain, dnskey, rrsig, lastCheckAt, lastOKAt := generateSignAndSaveDomain("br.", domainDAO)

	dns.HandleFunc("br.", func(w dns.ResponseWriter, dnsRequestMessage *dns.Msg) {
		defer w.Close()

		if dnsRequestMessage.Question[0].Qtype == dns.TypeSOA {
			dnsResponseMessage := &dns.Msg{
				MsgHdr: dns.MsgHdr{
					Authoritative: true,
				},
				Question: dnsRequestMessage.Question,
				Answer: []dns.RR{
					&dns.SOA{
						Hdr: dns.RR_Header{
							Name:   "br.",
							Rrtype: dns.TypeSOA,
							Class:  dns.ClassINET,
							Ttl:    86400,
						},
						Ns:      "ns1.br.",
						Mbox:    "rafael.justo.net.br.",
						Serial:  2013112600,
						Refresh: 86400,
						Retry:   86400,
						Expire:  86400,
						Minttl:  900,
					},
				},
			}
			dnsResponseMessage.SetReply(dnsRequestMessage)
			w.WriteMsg(dnsResponseMessage)

		} else if dnsRequestMessage.Question[0].Qtype == dns.TypeDNSKEY {
			dnsResponseMessage := &dns.Msg{
				MsgHdr: dns.MsgHdr{
					Authoritative: true,
				},
				Question: dnsRequestMessage.Question,
				Answer: []dns.RR{
					dnskey,
					rrsig,
				},
			}

			dnsResponseMessage.SetReply(dnsRequestMessage)
			w.WriteMsg(dnsResponseMessage)
		}
	})

	scan.ScanDomains()

	domain, err := domainDAO.FindByFQDN(domain.FQDN)
	if err != nil {
		utils.Fatalln("Didn't find scanned domain", err)
	}

	for _, nameserver := range domain.Nameservers {
		if nameserver.LastStatus != model.NameserverStatusOK {
			utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured nameserver '%s'. Found status: %s",
				nameserver.Host, model.NameserverStatusToString(nameserver.LastStatus)), nil)
		}

		if nameserver.LastCheckAt.Before(lastCheckAt) ||
			nameserver.LastCheckAt.Equal(lastCheckAt) {
			utils.Fatalln(fmt.Sprintf("Last check date was not updated in nameserver '%s'",
				nameserver.Host), nil)
		}

		if nameserver.LastOKAt.Before(lastOKAt) || nameserver.LastOKAt.Equal(lastOKAt) {
			utils.Fatalln(fmt.Sprintf("Last OK date was not updated in nameserver '%s'",
				nameserver.Host), nil)
		}
	}

	for _, ds := range domain.DSSet {
		if ds.LastStatus != model.DSStatusOK {
			utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured DS %d. "+
				"Found status: %s", ds.Keytag, model.DSStatusToString(ds.LastStatus)), nil)
		}

		if ds.LastCheckAt.Before(lastCheckAt) || ds.LastCheckAt.Equal(lastCheckAt) {
			utils.Fatalln(fmt.Sprintf("Last check date was not updated in DS %d",
				ds.Keytag), nil)
		}

		if ds.LastOKAt.Before(lastOKAt) || ds.LastOKAt.Equal(lastOKAt) {
			utils.Fatalln(fmt.Sprintf("Last OK date was not updated in DS %d",
				ds.Keytag), nil)
		}
	}

	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln(fmt.Sprintf("Error removing domain %s", domain.FQDN), err)
	}
}
Beispiel #23
0
func simpleNotification(domainDAO dao.DomainDAO, templateName string,
	messageChannel chan *mail.Message, errorChannel chan error) {

	generateAndSaveDomain("example.com.br.", domainDAO, templateName)

	notification.TemplateExtension = ""
	if err := notification.LoadTemplates(); err != nil {
		utils.Fatalln("Error loading templates", err)
	}

	notification.Notify()

	timeout := make(chan bool, 1)
	go func() {
		time.Sleep(5 * time.Second)
		timeout <- true
	}()

	select {
	case message := <-messageChannel:
		if message.Header.Get("From") != "*****@*****.**" {
			utils.Fatalln(fmt.Sprintf("E-mail from header is different. Expected "+
				"[email protected] but found %s", message.Header.Get("From")), nil)
		}

		if message.Header.Get("To") != "*****@*****.**" {
			utils.Fatalln("E-mail to header is different", nil)
		}

		if message.Header.Get("Subject") != "=?UTF-8?B?TWlzY29uZmlndXJhdGlvbiBvbiBkb21haW4gZXhhbXBsZS5jb20uYnIu?=" {
			utils.Fatalln("E-mail subject header is different", nil)
		}

		body, err := ioutil.ReadAll(message.Body)
		if err != nil {
			utils.Fatalln("Error reading e-mail body", err)
		}

		expectedBody := "Dear Sir/Madam,\r\n" +
			"\r\n" +
			"During our periodically domain verification, a configuration problem was detected with the\r\n" +
			"domain example.com.br..\r\n" +
			"\r\n" +
			"  * Nameserver ns1.example.com.br. got an internal error while receiving the DNS request.\r\n" +
			"    Please check the DNS server log to detect and solve the problem.\r\n" +
			"\r\n" +
			"Best regards,\r\n" +
			"LACTLD\r\n" +
			".\r\n"

		if string(body) != expectedBody {
			utils.Fatalln(fmt.Sprintf("E-mail body is different from what we expected. "+
				"Expected [%s], but found [%s]", expectedBody, body), nil)
		}

	case err := <-errorChannel:
		utils.Fatalln("Error receiving message", err)

	case <-timeout:
		utils.Fatalln("No mail sent", nil)
	}

	if err := domainDAO.RemoveByFQDN("example.com.br."); err != nil {
		utils.Fatalln("Error removing domain", err)
	}
}
Beispiel #24
0
func domainsPagination(domainDAO dao.DomainDAO) {
	numberOfItems := 1000

	for i := 0; i < numberOfItems; i++ {
		domain := model.Domain{
			FQDN: fmt.Sprintf("example%d.com.br", i),
		}

		if err := domainDAO.Save(&domain); err != nil {
			utils.Fatalln("Error saving domain in database", err)
		}
	}

	pagination := dao.DomainDAOPagination{
		PageSize: 10,
		Page:     5,
		OrderBy: []dao.DomainDAOSort{
			{
				Field:     dao.DomainDAOOrderByFieldFQDN,
				Direction: dao.DAOOrderByDirectionAscending,
			},
		},
	}

	domains, err := domainDAO.FindAll(&pagination, true, "")
	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	if pagination.NumberOfItems != numberOfItems {
		utils.Errorln("Number of items not calculated correctly", nil)
	}

	if pagination.NumberOfPages != numberOfItems/pagination.PageSize {
		utils.Errorln("Number of pages not calculated correctly", nil)
	}

	if len(domains) != pagination.PageSize {
		utils.Errorln("Number of domains not following page size", nil)
	}

	pagination = dao.DomainDAOPagination{
		PageSize: 10000,
		Page:     1,
		OrderBy: []dao.DomainDAOSort{
			{
				Field:     dao.DomainDAOOrderByFieldFQDN,
				Direction: dao.DAOOrderByDirectionAscending,
			},
		},
	}

	domains, err = domainDAO.FindAll(&pagination, true, "")
	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	if pagination.NumberOfPages != 1 {
		utils.Fatalln("Calculating wrong number of pages when there's only one page", nil)
	}

	for i := 0; i < numberOfItems; i++ {
		fqdn := fmt.Sprintf("example%d.com.br", i)
		if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
			utils.Fatalln("Error removing domain from database", err)
		}
	}
}
Beispiel #25
0
// Verify if the method that choose the domains that needs to be verified is correct
func domainsNotification(domainDAO dao.DomainDAO) {
	numberOfItemsToBeVerified := 1000
	numberOfItemsToDontBeVerified := 1000
	nameserverErrorAlertDays := 7
	nameserverTimeoutAlertDays := 30
	dsErrorAlertDays := 1
	dsTimeoutAlertDays := 7
	maxExpirationAlertDays := 5

	data := []struct {
		name                      string
		numberOfItems             int
		nameserverTimeoutLastOKAt time.Time
		nameserverErrorLastOKAt   time.Time
		dsTimeoutLastOkAt         time.Time
		dsErrorLastOkAt           time.Time
		dsExpiresAt               time.Time
	}{
		{
			name:                      "shouldbenotified",
			numberOfItems:             numberOfItemsToBeVerified,
			nameserverTimeoutLastOKAt: time.Now().Add(time.Duration(-nameserverTimeoutAlertDays*24) * time.Hour),
			nameserverErrorLastOKAt:   time.Now().Add(time.Duration(-nameserverErrorAlertDays*24) * time.Hour),
			dsTimeoutLastOkAt:         time.Now().Add(time.Duration(-dsTimeoutAlertDays*24) * time.Hour),
			dsErrorLastOkAt:           time.Now().Add(time.Duration(-dsErrorAlertDays*24) * time.Hour),
			dsExpiresAt:               time.Now().Add(time.Duration((maxExpirationAlertDays)*24) * time.Hour),
		},
		{
			name:                      "shouldnotbenotified",
			numberOfItems:             numberOfItemsToDontBeVerified,
			nameserverTimeoutLastOKAt: time.Now().Add(time.Duration((-nameserverTimeoutAlertDays+1)*24) * time.Hour),
			nameserverErrorLastOKAt:   time.Now().Add(time.Duration((-nameserverErrorAlertDays+1)*24) * time.Hour),
			dsTimeoutLastOkAt:         time.Now().Add(time.Duration((-dsTimeoutAlertDays+1)*24) * time.Hour),
			dsErrorLastOkAt:           time.Now().Add(time.Duration((-dsErrorAlertDays+1)*24) * time.Hour),
			dsExpiresAt:               time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
		},
	}

	for _, item := range data {
		for i := 0; i < item.numberOfItems/5; i++ {
			domain := model.Domain{
				FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
				Nameservers: []model.Nameserver{
					{
						LastStatus: model.NameserverStatusTimeout,
						LastOKAt:   item.nameserverTimeoutLastOKAt,
					},
				},
			}

			if err := domainDAO.Save(&domain); err != nil {
				utils.Fatalln("Error saving domain in database", err)
			}
		}

		for i := item.numberOfItems / 5; i < item.numberOfItems/5*2; i++ {
			domain := model.Domain{
				FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
				Nameservers: []model.Nameserver{
					{
						LastStatus: model.NameserverStatusNoAuthority,
						LastOKAt:   item.nameserverErrorLastOKAt,
					},
				},
			}

			if err := domainDAO.Save(&domain); err != nil {
				utils.Fatalln("Error saving domain in database", err)
			}
		}

		for i := item.numberOfItems / 5 * 2; i < item.numberOfItems/5*3; i++ {
			domain := model.Domain{
				FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
				DSSet: []model.DS{
					{
						LastStatus: model.DSStatusTimeout,
						LastOKAt:   item.dsTimeoutLastOkAt,
						ExpiresAt:  time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
					},
				},
			}

			if err := domainDAO.Save(&domain); err != nil {
				utils.Fatalln("Error saving domain in database", err)
			}
		}

		for i := item.numberOfItems / 5 * 3; i < item.numberOfItems/5*4; i++ {
			domain := model.Domain{
				FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
				DSSet: []model.DS{
					{
						LastStatus: model.DSStatusExpiredSignature,
						LastOKAt:   item.dsErrorLastOkAt,
						ExpiresAt:  time.Now().Add(time.Duration((maxExpirationAlertDays+1)*24) * time.Hour),
					},
				},
			}

			if err := domainDAO.Save(&domain); err != nil {
				utils.Fatalln("Error saving domain in database", err)
			}
		}

		for i := item.numberOfItems / 5 * 4; i < item.numberOfItems; i++ {
			domain := model.Domain{
				FQDN: fmt.Sprintf("%s%d.com.br", item.name, i),
				DSSet: []model.DS{
					{
						LastStatus: model.DSStatusOK,
						LastOKAt:   time.Now(),
						ExpiresAt:  item.dsExpiresAt,
					},
				},
			}

			if err := domainDAO.Save(&domain); err != nil {
				utils.Fatalln("Error saving domain in database", err)
			}
		}
	}

	domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
		nameserverErrorAlertDays,
		nameserverTimeoutAlertDays,
		dsErrorAlertDays,
		dsTimeoutAlertDays,
		maxExpirationAlertDays,
	)

	if err != nil {
		utils.Fatalln("Error retrieving domains to be notified", err)
	}

	var domains []*model.Domain
	for {
		domainResult := <-domainChannel

		if domainResult.Error != nil {
			utils.Fatalln("Error retrieving domain to be notified", domainResult.Error)
		}

		if domainResult.Error != nil || domainResult.Domain == nil {
			break
		}

		domains = append(domains, domainResult.Domain)
	}

	if len(domains) != numberOfItemsToBeVerified {
		utils.Fatalln(fmt.Sprintf("Did not select all the domains ready for notification. "+
			"Expected %d and got %d", numberOfItemsToBeVerified, len(domains)), nil)
	}

	for _, item := range data {
		for i := 0; i < item.numberOfItems; i++ {
			fqdn := fmt.Sprintf("%s%d.com.br", item.name, i)
			if err := domainDAO.RemoveByFQDN(fqdn); err != nil {
				utils.Fatalln("Error removing domain from database", err)
			}
		}
	}
}
Beispiel #26
0
func domainWithErrors(config ScanCollectorTestConfigFile, database *mgo.Database) {
	domainsToSave := make(chan *model.Domain, config.Scan.DomainsBufferSize)
	domainsToSave <- &model.Domain{
		FQDN: "br.",
		Nameservers: []model.Nameserver{
			{
				Host:       "ns1.br",
				IPv4:       net.ParseIP("127.0.0.1"),
				LastStatus: model.NameserverStatusTimeout,
			},
		},
		DSSet: []model.DS{
			{
				Keytag:     1234,
				Algorithm:  model.DSAlgorithmRSASHA1NSEC3,
				DigestType: model.DSDigestTypeSHA1,
				Digest:     "EAA0978F38879DB70A53F9FF1ACF21D046A98B5C",
				LastStatus: model.DSStatusExpiredSignature,
			},
		},
	}
	domainsToSave <- nil

	model.StartNewScan()
	runScan(config, database, domainsToSave)

	domainDAO := dao.DomainDAO{
		Database: database,
	}

	domain, err := domainDAO.FindByFQDN("br.")
	if err != nil {
		utils.Fatalln("Error loading domain with problems", err)
	}

	if len(domain.Nameservers) == 0 {
		utils.Fatalln("Error saving nameservers", nil)
	}

	if domain.Nameservers[0].LastStatus != model.NameserverStatusTimeout {
		utils.Fatalln("Error setting status in the nameserver", nil)
	}

	if len(domain.DSSet) == 0 {
		utils.Fatalln("Error saving the DS set", nil)
	}

	if domain.DSSet[0].LastStatus != model.DSStatusExpiredSignature {
		utils.Fatalln("Error setting status in the DS", nil)
	}

	if err := domainDAO.RemoveByFQDN("br."); err != nil {
		utils.Fatalln("Error removing test domain", err)
	}

	currentScan := model.GetCurrentScan()
	if currentScan.DomainsScanned != 1 || currentScan.DomainsWithDNSSECScanned != 1 {
		utils.Fatalln("Not counting domains for scan progress when there're errors", nil)
	}

	if currentScan.NameserverStatistics[model.NameserverStatusToString(model.NameserverStatusTimeout)] != 1 ||
		currentScan.DSStatistics[model.DSStatusToString(model.DSStatusExpiredSignature)] != 1 {
		utils.Fatalln("Not counting statistics properly when there're errors", nil)
	}
}
Beispiel #27
0
func domainsExpand(domainDAO dao.DomainDAO) {
	newDomains := newDomains()
	domainsResult := domainDAO.SaveMany(newDomains)
	for _, domainResult := range domainsResult {
		if domainResult.Error != nil {
			utils.Fatalln("Error creating domains", domainResult.Error)
		}
	}

	pagination := dao.DomainDAOPagination{}
	domains, err := domainDAO.FindAll(&pagination, false, "")

	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	for _, domain := range domains {
		if len(domain.Owners) > 0 {
			utils.Fatalln("Not compressing owners in results", nil)
		}

		for _, nameserver := range domain.Nameservers {
			if len(nameserver.Host) > 0 ||
				nameserver.IPv4 != nil ||
				nameserver.IPv6 != nil ||
				!nameserver.LastCheckAt.Equal(time.Time{}) ||
				!nameserver.LastOKAt.Equal(time.Time{}) {
				utils.Fatalln("Not compressing nameservers in results", nil)
			}
		}

		for _, ds := range domain.DSSet {
			if ds.Algorithm != 0 ||
				len(ds.Digest) > 0 ||
				ds.DigestType != 0 ||
				ds.Keytag != 0 ||
				!ds.ExpiresAt.Equal(time.Time{}) ||
				!ds.LastCheckAt.Equal(time.Time{}) ||
				!ds.LastOKAt.Equal(time.Time{}) {
				utils.Fatalln("Not compressing ds set in results", nil)
			}
		}
	}

	domains, err = domainDAO.FindAll(&pagination, true, "")

	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	for _, domain := range domains {
		if len(domain.Owners) == 0 {
			utils.Fatalln("Compressing owners in results when it shouldn't", nil)
		}

		for _, nameserver := range domain.Nameservers {
			if len(nameserver.Host) == 0 ||
				nameserver.IPv4 == nil ||
				nameserver.IPv6 == nil ||
				nameserver.LastCheckAt.Equal(time.Time{}) ||
				nameserver.LastOKAt.Equal(time.Time{}) ||
				nameserver.LastStatus != model.NameserverStatusOK {
				utils.Fatalln("Compressing nameservers in results when it shouldn't", nil)
			}
		}

		for _, ds := range domain.DSSet {
			if ds.Algorithm == 0 ||
				len(ds.Digest) == 0 ||
				ds.DigestType == 0 ||
				ds.Keytag == 0 ||
				ds.ExpiresAt.Equal(time.Time{}) ||
				ds.LastCheckAt.Equal(time.Time{}) ||
				ds.LastOKAt.Equal(time.Time{}) ||
				ds.LastStatus != model.DSStatusOK {
				utils.Fatalln("Compressing ds set in results when it shouldn't", nil)
			}
		}
	}

	domainsResult = domainDAO.RemoveMany(newDomains)
	for _, domainResult := range domainsResult {
		if domainResult.Error != nil {
			utils.Fatalln("Error removing domains", domainResult.Error)
		}
	}
}
Beispiel #28
0
func brDomainWithoutDNSSEC(domainDAO dao.DomainDAO) {
	domain := model.Domain{
		FQDN: "br.",
		Nameservers: []model.Nameserver{
			{
				Host: "a.dns.br.",
				IPv4: net.ParseIP("200.160.0.10"),
				IPv6: net.ParseIP("2001:12ff::10"),
			},
			{
				Host: "b.dns.br.",
				IPv4: net.ParseIP("200.189.41.10"),
			},
			{
				Host: "c.dns.br.",
				IPv4: net.ParseIP("200.192.233.10"),
			},
			{
				Host: "d.dns.br.",
				IPv4: net.ParseIP("200.219.154.10"),
				IPv6: net.ParseIP("2001:12f8:4::10"),
			},
			{
				Host: "f.dns.br.",
				IPv4: net.ParseIP("200.219.159.10"),
			},
		},

		// We are going to add the current DNSKEYs from .br but we are not going to check it.
		// This is because there's a strange case that when it found a problem on a DS (such
		// as bit SEP) it does not check other nameservers
		DSSet: []model.DS{
			{
				Keytag:     41674,
				Algorithm:  model.DSAlgorithmRSASHA1,
				DigestType: model.DSDigestTypeSHA256,
				Digest:     "6ec74914376b4f383ede3840088ae1d7bf13a19bfc51465cc2da57618889416a",
			},
			{
				Keytag:     57207,
				Algorithm:  model.DSAlgorithmRSASHA1,
				DigestType: model.DSDigestTypeSHA256,
				Digest:     "d46f059860d31a0965f925ac6ff97ed0975f33a14e2d01ec5ab5dd543624d307",
			},
		},
	}

	var err error

	if err = domainDAO.Save(&domain); err != nil {
		utils.Fatalln("Error saving the domain", err)
	}

	scan.ScanDomains()

	domain, err = domainDAO.FindByFQDN(domain.FQDN)
	if err != nil {
		utils.Fatalln("Didn't find scanned domain", err)
	}

	for _, nameserver := range domain.Nameservers {
		if nameserver.LastStatus != model.NameserverStatusOK {
			utils.Fatalln(fmt.Sprintf("Fail to validate a supposedly well configured nameserver '%s'. Found status: %s",
				nameserver.Host, model.NameserverStatusToString(nameserver.LastStatus)), nil)
		}
	}

	if err := domainDAO.RemoveByFQDN(domain.FQDN); err != nil {
		utils.Fatalln(fmt.Sprintf("Error removing domain %s", domain.FQDN), err)
	}
}
Beispiel #29
0
func calculateDomainDAODurations(domainDAO dao.DomainDAO, numberOfItems int) (totalDuration, insertDuration,
	queryDuration, removeDuration time.Duration) {

	beginTimer := time.Now()
	sectionTimer := beginTimer

	// Build array to create many at once
	var domains []*model.Domain
	for i := 0; i < numberOfItems; i++ {
		domain := model.Domain{
			FQDN: fmt.Sprintf("test%d.com.br", i),
		}

		domains = append(domains, &domain)
	}

	errorInDomainsCreation := false
	domainResults := domainDAO.SaveMany(domains)

	// Check if there was any error while creating them
	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			errorInDomainsCreation = true
			utils.Errorln(fmt.Sprintf("Couldn't save domain %s in database during the performance test",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	if errorInDomainsCreation {
		utils.Fatalln("Due to errors in domain creation, the performance test will be aborted", nil)
	}

	insertDuration = time.Since(sectionTimer)
	sectionTimer = time.Now()

	// Try to find domains from different parts of the whole range to check indexes
	queryRanges := numberOfItems / 4
	fqdn1 := fmt.Sprintf("test%d.com.br", queryRanges)
	fqdn2 := fmt.Sprintf("test%d.com.br", queryRanges*2)
	fqdn3 := fmt.Sprintf("test%d.com.br", queryRanges*3)

	if _, err := domainDAO.FindByFQDN(fqdn1); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn1), err)
	}

	if _, err := domainDAO.FindByFQDN(fqdn2); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn2), err)
	}

	if _, err := domainDAO.FindByFQDN(fqdn3); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn3), err)
	}

	queryDuration = time.Since(sectionTimer)
	sectionTimer = time.Now()

	errorInDomainsRemoval := false
	domainResults = domainDAO.RemoveMany(domains)

	// Check if there was any error while removing them
	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			errorInDomainsRemoval = true
			utils.Errorln(fmt.Sprintf("Error while trying to remove a domain %s during the performance test",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	if errorInDomainsRemoval {
		utils.Fatalln("Due to errors in domain removal, the performance test will be aborted", nil)
	}

	removeDuration = time.Since(sectionTimer)
	totalDuration = time.Since(beginTimer)

	return
}
Beispiel #30
0
// This method is the last part of the scan, when the new state of the domain object is
// persisted back to the database. It receives a go routine control group to sinalize to
// the main thread when the scan ends, a domain channel to receive each domain that need
// to be save and an error channel to send back all errors while persisting the data. It
// was created to be asynchronous and finish after receiving a poison pill from querier
// dispatcher
func (c *Collector) Start(scanGroup *sync.WaitGroup,
	domainsToSaveChannel chan *model.Domain, errorsChannel chan error) {

	// Add one more to the group of scan go routines
	scanGroup.Add(1)

	go func() {
		// Initialize Domain DAO using injected database connection
		domainDAO := dao.DomainDAO{
			Database: c.Database,
		}

		// Add a safety check to avoid an infinite loop
		if c.SaveAtOnce == 0 {
			c.SaveAtOnce = 1
		}

		finished := false
		nameserverStatistics := make(map[string]uint64)
		dsStatistics := make(map[string]uint64)

		for {
			// Using make for faster allocation
			domains := make([]*model.Domain, 0, c.SaveAtOnce)

			for i := 0; i < c.SaveAtOnce; i++ {
				domain := <-domainsToSaveChannel

				// Detect poison pill. We don't return from function here because we can still
				// have some domains to save in the domains array
				if domain == nil {
					finished = true
					break
				}

				// Count this domain for the scan information to estimate the scan progress
				model.FinishAnalyzingDomainForScan(len(domain.DSSet) > 0)

				// Keep track of nameservers statistics
				for _, nameserver := range domain.Nameservers {
					status := model.NameserverStatusToString(nameserver.LastStatus)
					nameserverStatistics[status] += 1
				}

				// Keep track of DS statistics
				for _, ds := range domain.DSSet {
					status := model.DSStatusToString(ds.LastStatus)
					dsStatistics[status] += 1
				}

				domains = append(domains, domain)
			}

			domainsResults := domainDAO.SaveMany(domains)
			for _, domainResult := range domainsResults {
				if domainResult.Error != nil {
					// Error channel should have a buffer or this will block the collector until
					// someone check this error. One question here is that we are returning the
					// error, but not telling wich domain got the error, we should improve the error
					// communication system between the go routines
					errorsChannel <- domainResult.Error
				}
			}

			// Now that everything is done, check if we received a poison pill
			if finished {
				model.StoreStatisticsOfTheScan(nameserverStatistics, dsStatistics)
				scanGroup.Done()
				return
			}
		}
	}()
}