func calculateDomainDAODurations(domainDAO dao.DomainDAO, numberOfItems int) (totalDuration, insertDuration,
queryDuration, removeDuration time.Duration) {
beginTimer := time.Now()
sectionTimer := beginTimer
// Build array to create many at once
var domains []*model.Domain
for i := 0; i < numberOfItems; i++ {
domain := model.Domain{
FQDN: fmt.Sprintf("test%d.com.br", i),
}
domains = append(domains, &domain)
}
errorInDomainsCreation := false
domainResults := domainDAO.SaveMany(domains)
// Check if there was any error while creating them
for _, domainResult := range domainResults {
if domainResult.Error != nil {
errorInDomainsCreation = true
utils.Errorln(fmt.Sprintf("Couldn't save domain %s in database during the performance test",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if errorInDomainsCreation {
utils.Fatalln("Due to errors in domain creation, the performance test will be aborted", nil)
}
insertDuration = time.Since(sectionTimer)
sectionTimer = time.Now()
// Try to find domains from different parts of the whole range to check indexes
queryRanges := numberOfItems / 4
fqdn1 := fmt.Sprintf("test%d.com.br", queryRanges)
fqdn2 := fmt.Sprintf("test%d.com.br", queryRanges*2)
fqdn3 := fmt.Sprintf("test%d.com.br", queryRanges*3)
if _, err := domainDAO.FindByFQDN(fqdn1); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn1), err)
}
if _, err := domainDAO.FindByFQDN(fqdn2); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn2), err)
}
if _, err := domainDAO.FindByFQDN(fqdn3); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
"the performance test", fqdn3), err)
}
queryDuration = time.Since(sectionTimer)
sectionTimer = time.Now()
errorInDomainsRemoval := false
domainResults = domainDAO.RemoveMany(domains)
// Check if there was any error while removing them
for _, domainResult := range domainResults {
if domainResult.Error != nil {
errorInDomainsRemoval = true
utils.Errorln(fmt.Sprintf("Error while trying to remove a domain %s during the performance test",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if errorInDomainsRemoval {
utils.Fatalln("Due to errors in domain removal, the performance test will be aborted", nil)
}
removeDuration = time.Since(sectionTimer)
totalDuration = time.Since(beginTimer)
return
}
// Test all phases from a domain lyfe cycle, but now working with a group of domains
func domainsLifeCycle(domainDAO dao.DomainDAO) {
domains := newDomains()
// Create domains
domainResults := domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Search and compare created domains
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find created domain %s in database", domain.FQDN), err)
} else if !utils.CompareDomain(*domain, domainRetrieved) {
utils.Fatalln(fmt.Sprintf("Domain %s created is being persisted wrongly", domain.FQDN), nil)
}
}
// Update domains
for _, domain := range domains {
domain.Owners = []model.Owner{}
}
domainResults = domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't update domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Search and compare updated domains
if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
utils.Fatalln(fmt.Sprintf("Couldn't find updated domain %s in database", domain.FQDN), err)
} else if !utils.CompareDomain(*domain, domainRetrieved) {
utils.Fatalln(fmt.Sprintf("Domain %s updated in being persisted wrongly", domain.FQDN), nil)
}
}
// Check if find all really return all domains
allDomainsChannel, err := domainDAO.FindAllAsync()
if err != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
}
var allDomains []model.Domain
for {
domainRetrieved := <-allDomainsChannel
if domainRetrieved.Error != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
} else if domainRetrieved.Domain == nil {
break
}
allDomains = append(allDomains, *domainRetrieved.Domain)
}
if len(allDomains) != len(domains) {
utils.Fatalln(fmt.Sprintf("FindAll method is not returning all domains we expected %d but got %d",
len(domains), len(allDomains)), nil)
}
// Detected a problem in FindAsync method on 2014-01-17 where we were returning the same
// object many times because we were reusing the same pointer. For that reason we are
// going to add a test to check if the items returned are the same set of the inserted
// ones
for _, domain := range domains {
found := false
for _, domainRetrieved := range allDomains {
if domainRetrieved.Id.Hex() == domain.Id.Hex() {
found = true
break
}
}
if !found {
utils.Fatalln("FindAll method is not returning all objects "+
"that were inserted, apparently there are duplicated objects in the result set", nil)
}
}
// Remove domains
domainResults = domainDAO.RemoveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Error while trying to remove domain %s from database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
for _, domain := range domains {
// Check removals
if _, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
utils.Fatalln(fmt.Sprintf("Domain %s was not removed from database", domain.FQDN), nil)
}
}
// Let's add and remove the domains again to test the remove all method
domainResults = domainDAO.SaveMany(domains)
for _, domainResult := range domainResults {
if domainResult.Error != nil {
utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
domainResult.Domain.FQDN), domainResult.Error)
}
}
if err := domainDAO.RemoveAll(); err != nil {
utils.Fatalln("Couldn't remove all domains", err)
}
allDomainsChannel, err = domainDAO.FindAllAsync()
if err != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
}
allDomains = []model.Domain{}
for {
domainRetrieved := <-allDomainsChannel
if domainRetrieved.Error != nil {
utils.Fatalln("Error while retrieving all domains from database", err)
} else if domainRetrieved.Domain == nil {
break
}
allDomains = append(allDomains, *domainRetrieved.Domain)
}
if len(allDomains) > 0 {
utils.Fatalln("RemoveAll method is not removing the domains from the database", nil)
}
}
func domainsExpand(domainDAO dao.DomainDAO) {
newDomains := newDomains()
domainsResult := domainDAO.SaveMany(newDomains)
for _, domainResult := range domainsResult {
if domainResult.Error != nil {
utils.Fatalln("Error creating domains", domainResult.Error)
}
}
pagination := dao.DomainDAOPagination{}
domains, err := domainDAO.FindAll(&pagination, false, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
for _, domain := range domains {
if len(domain.Owners) > 0 {
utils.Fatalln("Not compressing owners in results", nil)
}
for _, nameserver := range domain.Nameservers {
if len(nameserver.Host) > 0 ||
nameserver.IPv4 != nil ||
nameserver.IPv6 != nil ||
!nameserver.LastCheckAt.Equal(time.Time{}) ||
!nameserver.LastOKAt.Equal(time.Time{}) {
utils.Fatalln("Not compressing nameservers in results", nil)
}
}
for _, ds := range domain.DSSet {
if ds.Algorithm != 0 ||
len(ds.Digest) > 0 ||
ds.DigestType != 0 ||
ds.Keytag != 0 ||
!ds.ExpiresAt.Equal(time.Time{}) ||
!ds.LastCheckAt.Equal(time.Time{}) ||
!ds.LastOKAt.Equal(time.Time{}) {
utils.Fatalln("Not compressing ds set in results", nil)
}
}
}
domains, err = domainDAO.FindAll(&pagination, true, "")
if err != nil {
utils.Fatalln("Error retrieving domains", err)
}
for _, domain := range domains {
if len(domain.Owners) == 0 {
utils.Fatalln("Compressing owners in results when it shouldn't", nil)
}
for _, nameserver := range domain.Nameservers {
if len(nameserver.Host) == 0 ||
nameserver.IPv4 == nil ||
nameserver.IPv6 == nil ||
nameserver.LastCheckAt.Equal(time.Time{}) ||
nameserver.LastOKAt.Equal(time.Time{}) ||
nameserver.LastStatus != model.NameserverStatusOK {
utils.Fatalln("Compressing nameservers in results when it shouldn't", nil)
}
}
for _, ds := range domain.DSSet {
if ds.Algorithm == 0 ||
len(ds.Digest) == 0 ||
ds.DigestType == 0 ||
ds.Keytag == 0 ||
ds.ExpiresAt.Equal(time.Time{}) ||
ds.LastCheckAt.Equal(time.Time{}) ||
ds.LastOKAt.Equal(time.Time{}) ||
ds.LastStatus != model.DSStatusOK {
utils.Fatalln("Compressing ds set in results when it shouldn't", nil)
}
}
}
domainsResult = domainDAO.RemoveMany(newDomains)
for _, domainResult := range domainsResult {
if domainResult.Error != nil {
utils.Fatalln("Error removing domains", domainResult.Error)
}
}
}
// This method is the last part of the scan, when the new state of the domain object is
// persisted back to the database. It receives a go routine control group to sinalize to
// the main thread when the scan ends, a domain channel to receive each domain that need
// to be save and an error channel to send back all errors while persisting the data. It
// was created to be asynchronous and finish after receiving a poison pill from querier
// dispatcher
func (c *Collector) Start(scanGroup *sync.WaitGroup,
domainsToSaveChannel chan *model.Domain, errorsChannel chan error) {
// Add one more to the group of scan go routines
scanGroup.Add(1)
go func() {
// Initialize Domain DAO using injected database connection
domainDAO := dao.DomainDAO{
Database: c.Database,
}
// Add a safety check to avoid an infinite loop
if c.SaveAtOnce == 0 {
c.SaveAtOnce = 1
}
finished := false
nameserverStatistics := make(map[string]uint64)
dsStatistics := make(map[string]uint64)
for {
// Using make for faster allocation
domains := make([]*model.Domain, 0, c.SaveAtOnce)
for i := 0; i < c.SaveAtOnce; i++ {
domain := <-domainsToSaveChannel
// Detect poison pill. We don't return from function here because we can still
// have some domains to save in the domains array
if domain == nil {
finished = true
break
}
// Count this domain for the scan information to estimate the scan progress
model.FinishAnalyzingDomainForScan(len(domain.DSSet) > 0)
// Keep track of nameservers statistics
for _, nameserver := range domain.Nameservers {
status := model.NameserverStatusToString(nameserver.LastStatus)
nameserverStatistics[status] += 1
}
// Keep track of DS statistics
for _, ds := range domain.DSSet {
status := model.DSStatusToString(ds.LastStatus)
dsStatistics[status] += 1
}
domains = append(domains, domain)
}
domainsResults := domainDAO.SaveMany(domains)
for _, domainResult := range domainsResults {
if domainResult.Error != nil {
// Error channel should have a buffer or this will block the collector until
// someone check this error. One question here is that we are returning the
// error, but not telling wich domain got the error, we should improve the error
// communication system between the go routines
errorsChannel <- domainResult.Error
}
}
// Now that everything is done, check if we received a poison pill
if finished {
model.StoreStatisticsOfTheScan(nameserverStatistics, dsStatistics)
scanGroup.Done()
return
}
}
}()
}