func createScans(database *mgo.Database) {
scanDAO := dao.ScanDAO{
Database: database,
}
for i := 0; i < 100; i++ {
scan := model.Scan{
Status: model.ScanStatusExecuted,
StartedAt: time.Now().Add(time.Duration(-i*2) * time.Minute),
FinishedAt: time.Now().Add(time.Duration(-i) * time.Minute),
DomainsScanned: 5000000,
DomainsWithDNSSECScanned: 250000,
NameserverStatistics: map[string]uint64{
"OK": 4800000,
"TIMEOUT": 200000,
},
DSStatistics: map[string]uint64{
"OK": 220000,
"EXPSIG": 30000,
},
}
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Error creating scan", err)
}
}
}
func (i *Scan) Before(w http.ResponseWriter, r *http.Request) {
date, err := time.Parse(time.RFC3339Nano, strings.ToUpper(i.scanHandler.GetStartedAt()))
if err != nil {
if err := i.scanHandler.MessageResponse("invalid-uri", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
scanDAO := dao.ScanDAO{
Database: i.scanHandler.GetDatabase(),
}
scan, err := scanDAO.FindByStartedAt(date)
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
i.scanHandler.SetScan(scan)
}
func createScan(database *mgo.Database) time.Time {
scan := model.Scan{
Status: model.ScanStatusExecuted,
StartedAt: time.Now().Add(-60 * time.Minute),
FinishedAt: time.Now().Add(-10 * time.Minute),
DomainsScanned: 5000000,
DomainsWithDNSSECScanned: 250000,
NameserverStatistics: map[string]uint64{
"OK": 4800000,
"TIMEOUT": 200000,
},
DSStatistics: map[string]uint64{
"OK": 220000,
"EXPSIG": 30000,
},
}
scanDAO := dao.ScanDAO{
Database: database,
}
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Error creating scan", err)
}
return scan.StartedAt
}
func deleteScan(database *mgo.Database, startedAt time.Time) {
scanDAO := dao.ScanDAO{
Database: database,
}
if err := scanDAO.RemoveByStartedAt(startedAt); err != nil {
utils.Fatalln("Error removing scan", err)
}
}
func deleteScans(database *mgo.Database) {
scanDAO := dao.ScanDAO{
Database: database,
}
if err := scanDAO.RemoveAll(); err != nil {
utils.Fatalln("Error removing scans", err)
}
}
// Function created to remove all entries from the database to ensure that the tests
// enviroments are always equal
func ClearDatabase(database *mgo.Database) {
domainDAO := dao.DomainDAO{
Database: database,
}
domainDAO.RemoveAll()
scanDAO := dao.ScanDAO{
Database: database,
}
scanDAO.RemoveAll()
}
func main() {
flag.Parse()
var config ScanDAOTestConfigFile
err := utils.ReadConfigFile(configFilePath, &config)
if err == utils.ErrConfigFileUndefined {
fmt.Println(err.Error())
fmt.Println("Usage:")
flag.PrintDefaults()
return
} else if err != nil {
utils.Fatalln("Error reading configuration file", err)
}
database, databaseSession, err := mongodb.Open(
[]string{config.Database.URI},
config.Database.Name,
false, "", "",
)
if err != nil {
utils.Fatalln("Error connecting the database", err)
}
defer databaseSession.Close()
scanDAO := dao.ScanDAO{
Database: database,
}
// If there was some problem in the last test, there could be some data in the
// database, so let's clear it to don't affect this test. We avoid checking the error,
// because if the collection does not exist yet, it will be created in the first
// insert
scanDAO.RemoveAll()
scanLifeCycle(scanDAO)
scanConcurrency(scanDAO)
scanStatistics(scanDAO)
scansPagination(scanDAO)
scansExpand(scanDAO)
utils.Println("SUCCESS!")
}
func scansExpand(scanDAO dao.ScanDAO) {
newScans := newScans()
for _, scan := range newScans {
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Error creating scans", err)
}
}
pagination := dao.ScanDAOPagination{}
scans, err := scanDAO.FindAll(&pagination, false)
if err != nil {
utils.Fatalln("Error retrieving scans", err)
}
for _, scan := range scans {
if scan.DomainsScanned != 0 ||
scan.DomainsWithDNSSECScanned != 0 ||
!scan.FinishedAt.Equal(time.Time{}) ||
len(scan.NameserverStatistics) > 0 ||
len(scan.DSStatistics) > 0 {
utils.Fatalln("Not compressing scan in results", nil)
}
}
scans, err = scanDAO.FindAll(&pagination, true)
if err != nil {
utils.Fatalln("Error retrieving scans", err)
}
for _, scan := range scans {
if scan.DomainsScanned == 0 ||
scan.DomainsWithDNSSECScanned == 0 ||
scan.FinishedAt.Equal(time.Time{}) ||
scan.StartedAt.Equal(time.Time{}) ||
len(scan.NameserverStatistics) == 0 ||
len(scan.DSStatistics) == 0 {
fmt.Println(scan.DomainsScanned == 0,
scan.DomainsWithDNSSECScanned == 0,
scan.FinishedAt.Equal(time.Time{}),
scan.StartedAt.Equal(time.Time{}),
len(scan.NameserverStatistics) == 0,
len(scan.DSStatistics) == 0)
utils.Fatalln("Compressing scan in results when it shouldn't", nil)
}
}
if err := scanDAO.RemoveAll(); err != nil {
utils.Fatalln("Error removing scans", err)
}
}
func calculateScanDurations(numberOfDomains int, scanDAO dao.ScanDAO) (
totalDuration time.Duration, domainsPerSecond int64,
) {
beginTimer := time.Now()
scan.ScanDomains()
totalDuration = time.Since(beginTimer)
totalDurationSeconds := int64(totalDuration / time.Second)
if totalDurationSeconds > 0 {
domainsPerSecond = int64(numberOfDomains) / totalDurationSeconds
} else {
domainsPerSecond = int64(numberOfDomains)
}
// As we are running a lot of scans at the same time, and the scan information unique
// key is the start time of the scan, we must clear the database to avoid log messages
// of scan insert errors
scanDAO.RemoveAll()
return
}
func scanStatistics(scanDAO dao.ScanDAO) {
scan := newScan()
scan.NameserverStatistics = map[string]uint64{
"OK": 40,
"TIMEOUT": 10,
}
scan.DSStatistics = map[string]uint64{
"OK": 8,
"DNSERR": 2,
}
// Create scan
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Couldn't save scan in database", err)
}
var err error
scan, err = scanDAO.FindByStartedAt(scan.StartedAt)
if err != nil {
utils.Fatalln("Couldn't find created scan in database", err)
}
if len(scan.NameserverStatistics) != 2 ||
scan.NameserverStatistics["OK"] != 40 ||
scan.NameserverStatistics["TIMEOUT"] != 10 {
utils.Fatalln("Not retrieving nameserver statistics correctly", nil)
}
if len(scan.DSStatistics) != 2 ||
scan.DSStatistics["OK"] != 8 ||
scan.DSStatistics["DNSERR"] != 2 {
utils.Fatalln("Not retrieving DS statistics correctly", nil)
}
if err := scanDAO.RemoveAll(); err != nil {
utils.Fatalln("Error removing scans from database", err)
}
}
// Test all phases of the scan life cycle
func scanLifeCycle(scanDAO dao.ScanDAO) {
scan := newScan()
// Create scan
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Couldn't save scan in database", err)
}
// Search and compare created scan
if scanRetrieved, err := scanDAO.FindByStartedAt(scan.StartedAt); err != nil {
utils.Fatalln("Couldn't find created scan in database", err)
} else if !utils.CompareScan(scan, scanRetrieved) {
utils.Fatalln("Scan created in being persisted wrongly", nil)
}
// Update scan
scan.DomainsScanned = 100
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Couldn't save scan in database", err)
}
// Search and compare updated scan
if scanRetrieved, err := scanDAO.FindByStartedAt(scan.StartedAt); err != nil {
utils.Fatalln("Couldn't find updated scan in database", err)
} else if !utils.CompareScan(scan, scanRetrieved) {
utils.Fatalln("Scan updated in being persisted wrongly", nil)
}
// Remove scan
if err := scanDAO.RemoveByStartedAt(scan.StartedAt); err != nil {
utils.Fatalln("Error while trying to remove a scan", err)
}
// Check removal
if _, err := scanDAO.FindByStartedAt(scan.StartedAt); err == nil {
utils.Fatalln("Scan was not removed from database", nil)
}
}
func scansPagination(scanDAO dao.ScanDAO) {
numberOfItems := 1000
for i := 0; i < numberOfItems; i++ {
scan := model.Scan{
StartedAt: time.Now().Add(time.Duration(-i) * time.Minute),
}
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Error saving scan in database", err)
}
}
pagination := dao.ScanDAOPagination{
PageSize: 10,
Page: 5,
OrderBy: []dao.ScanDAOSort{
{
Field: dao.ScanDAOOrderByFieldStartedAt,
Direction: dao.DAOOrderByDirectionAscending,
},
},
}
scans, err := scanDAO.FindAll(&pagination, true)
if err != nil {
utils.Fatalln("Error retrieving scans", err)
}
if pagination.NumberOfItems != numberOfItems {
utils.Errorln("Number of items not calculated correctly", nil)
}
if pagination.NumberOfPages != numberOfItems/pagination.PageSize {
utils.Errorln("Number of pages not calculated correctly", nil)
}
if len(scans) != pagination.PageSize {
utils.Errorln("Number of scans not following page size", nil)
}
pagination = dao.ScanDAOPagination{
PageSize: 10000,
Page: 1,
OrderBy: []dao.ScanDAOSort{
{
Field: dao.ScanDAOOrderByFieldStartedAt,
Direction: dao.DAOOrderByDirectionAscending,
},
},
}
scans, err = scanDAO.FindAll(&pagination, true)
if err != nil {
utils.Fatalln("Error retrieving scans", err)
}
if pagination.NumberOfPages != 1 {
utils.Fatalln("Calculating wrong number of pages when there's only one page", nil)
}
if err := scanDAO.RemoveAll(); err != nil {
utils.Fatalln("Error removing scans from database", err)
}
}
// Check if the revision field avoid data concurrency. Is better to fail than to store the
// wrong state
func scanConcurrency(scanDAO dao.ScanDAO) {
scan := newScan()
// Create scan
if err := scanDAO.Save(&scan); err != nil {
utils.Fatalln("Couldn't save scan in database", err)
}
scan1, err := scanDAO.FindByStartedAt(scan.StartedAt)
if err != nil {
utils.Fatalln("Couldn't find created scan in database", err)
}
scan2, err := scanDAO.FindByStartedAt(scan.StartedAt)
if err != nil {
utils.Fatalln("Couldn't find created scan in database", err)
}
if err := scanDAO.Save(&scan1); err != nil {
utils.Fatalln("Couldn't save scan in database", err)
}
if err := scanDAO.Save(&scan2); err == nil {
utils.Fatalln("Not controlling scan concurrency", nil)
}
// Remove scan
if err := scanDAO.RemoveByStartedAt(scan.StartedAt); err != nil {
utils.Fatalln("Error while trying to remove a scan", err)
}
}
// The HEAD method is identical to GET except that the server MUST NOT return a message-
// body in the response. But now the responsability for don't adding the body is from the
// mux while writing the response
func (h *ScansHandler) retrieveScans(w http.ResponseWriter, r *http.Request) {
var pagination dao.ScanDAOPagination
expand := false
returnCurrent := false
for key, values := range r.URL.Query() {
key = strings.TrimSpace(key)
key = strings.ToLower(key)
// A key can have multiple values in a query string, we are going to always consider
// the last one (overwrite strategy)
for _, value := range values {
value = strings.TrimSpace(value)
value = strings.ToLower(value)
switch key {
case "orderby":
// OrderBy parameter will store the fields that the user want to be the keys of the sort
// algorithm in the result set and the direction that each sort field will have. The format
// that will be used is:
//
// <field1>:<direction1>@<field2>:<direction2>@...@<fieldN>:<directionN>
orderByParts := strings.Split(value, "@")
for _, orderByPart := range orderByParts {
orderByPart = strings.TrimSpace(orderByPart)
orderByAndDirection := strings.Split(orderByPart, ":")
var field, direction string
if len(orderByAndDirection) == 1 {
field, direction = orderByAndDirection[0], "asc"
} else if len(orderByAndDirection) == 2 {
field, direction = orderByAndDirection[0], orderByAndDirection[1]
} else {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByField, err := dao.ScanDAOOrderByFieldFromString(field)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByDirection, err := dao.DAOOrderByDirectionFromString(direction)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
pagination.OrderBy = append(pagination.OrderBy, dao.ScanDAOSort{
Field: orderByField,
Direction: orderByDirection,
})
}
case "pagesize":
var err error
pagination.PageSize, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page-size", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "page":
var err error
pagination.Page, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "expand":
expand = true
case "current":
returnCurrent = true
}
}
}
scanDAO := dao.ScanDAO{
Database: h.GetDatabase(),
}
// As we need to inform the user about the number of items, we always try to retrieve the scan
// objects even if is requested only the current object
scans, err := scanDAO.FindAll(&pagination, expand)
if err != nil {
log.Println("Error while searching scans objects. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
var scansResponse protocol.ScansResponse
var current model.CurrentScan
if returnCurrent {
// The current page will be page zero to avoid misunderstandment
pagination.Page = 0
current = model.GetCurrentScan()
scansResponse = protocol.CurrentScanToScansResponse(current, pagination)
} else {
scansResponse = protocol.ScansToScansResponse(scans, pagination)
}
h.Response = &scansResponse
// Last-Modified is going to be the most recent date of the list
if returnCurrent {
h.lastModifiedAt = current.LastModifiedAt
} else {
for _, scan := range scans {
if scan.LastModifiedAt.After(h.lastModifiedAt) {
h.lastModifiedAt = scan.LastModifiedAt
}
}
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.lastModifiedAt.Format(time.RFC1123))
w.WriteHeader(http.StatusOK)
}