func (c *JSONCodec) After(w http.ResponseWriter, r *http.Request) {
st := reflect.ValueOf(c.codecHandler).Elem()
// We are going to always send the content HTTP header fields even if we don't have a
// content, because if we don't set the GoLang HTTP server will add "text/plain"
w.Header().Set("Content-Type", fmt.Sprintf("application/vnd.shelter+json; charset=%s", check.SupportedCharset))
if c.errPosition >= 0 {
if elem := st.Field(c.errPosition).Interface(); elem != nil {
elemType := reflect.TypeOf(elem)
if elemType.Kind() == reflect.Ptr && !st.Field(c.errPosition).IsNil() {
body, err := json.Marshal(elem)
if err != nil {
log.Println("Error writing message response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
c.addResponseHeaders(w, body)
w.Write(body)
return
}
}
}
if c.resPosition >= 0 {
elem := st.Field(c.resPosition).Interface()
elemType := reflect.TypeOf(elem)
// We are also checking for map types because the work like pointers
if (elemType.Kind() == reflect.Ptr ||
elemType.Kind() == reflect.Map ||
elemType.Kind() == reflect.Slice) &&
st.Field(c.resPosition).IsNil() {
// Nothing to write, add the default headers and move out
c.addResponseHeaders(w, nil)
return
}
body, err := json.Marshal(elem)
if err != nil {
log.Println("Error writing message response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
c.addResponseHeaders(w, body)
// For HTTP HEAD method we never add the body, but we add the other headers as a
// normal GET method. For more information check the RFC 2616 - 14.13.
if len(body) > 0 && r.Method != "HEAD" {
w.Write(body)
}
} else {
c.addResponseHeaders(w, nil)
}
}
func (h *ScanHandler) handleScan(w http.ResponseWriter, r *http.Request) {
restAddress, err := retrieveRESTAddress()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error while retrieving the REST address. Details:", err)
return
}
request, err := http.NewRequest(
"GET",
fmt.Sprintf("%s%s", restAddress, r.RequestURI),
nil,
)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error creating a request in web client. Details:", err)
return
}
request.Header.Set("Accept-Language", r.Header.Get("Accept-Language"))
request.Header.Set("If-None-Match", r.Header.Get("If-None-Match"))
response, err := signAndSend(request, nil)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error signing and sending a request in web client. Details:", err)
return
}
if response.StatusCode != http.StatusOK &&
response.StatusCode != http.StatusNotModified &&
response.StatusCode != http.StatusBadRequest {
w.WriteHeader(http.StatusInternalServerError)
log.Println(fmt.Sprintf("Unexepected status code %d from /scan/{started-at} "+
"result in web client", response.StatusCode))
return
}
if response.ContentLength > 0 {
w.Header().Add("Content-Type", "application/json")
w.Header().Add("Etag", response.Header.Get("Etag"))
}
w.WriteHeader(response.StatusCode)
if response.ContentLength > 0 {
if _, err := io.Copy(w, response.Body); err != nil {
// Here we already set the response code, so the client will receive a OK result
// without body
log.Println("Error copying REST response to web client response. Details:", err)
return
}
}
}
// We are going to use the initialization function to read command line arguments, load
// the configuration file and register system signals
func init() {
flag.StringVar(&configFilePath, "config", "", "Configuration file")
showVersion = flag.Bool("version", false, "System version")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "%s\nUsage of %s:\n", copyright, os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
if *showVersion {
fmt.Println(copyright)
os.Exit(0)
}
if len(configFilePath) == 0 {
fmt.Println("The configuration file was not informed")
os.Exit(ErrInputParameters)
}
if err := loadSettings(); err != nil {
log.Println("Error loading the configuration file. Details:", err)
os.Exit(ErrLoadingConfig)
}
manageSystemSignals()
}
func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, handler HTTPCacheHandler) bool {
noneMatch := check.HTTPIfNoneMatch(r, handler.GetETag())
if !noneMatch {
handler.ClearResponse()
// Instead, if the request method was GET or HEAD, the server SHOULD respond with a
// 304 (Not Modified) response, including the cache-related header fields
// (particularly ETag) of one of the entities that matched. For all other request
// methods, the server MUST respond with a status of 412 (Precondition Failed)
if r.Method == "GET" || r.Method == "HEAD" {
// The 304 response MUST NOT contain a message-body, and thus is always terminated
// by the first empty line after the header fields.
w.Header().Add("ETag", handler.GetETag())
w.WriteHeader(http.StatusNotModified)
} else {
if err := handler.MessageResponse("if-none-match-failed", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusPreconditionFailed)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
return false
}
return true
}
func (i *Scan) Before(w http.ResponseWriter, r *http.Request) {
date, err := time.Parse(time.RFC3339Nano, strings.ToUpper(i.scanHandler.GetStartedAt()))
if err != nil {
if err := i.scanHandler.MessageResponse("invalid-uri", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
scanDAO := dao.ScanDAO{
Database: i.scanHandler.GetDatabase(),
}
scan, err := scanDAO.FindByStartedAt(date)
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
i.scanHandler.SetScan(scan)
}
func checkIfModifiedSince(w http.ResponseWriter, r *http.Request, handler HTTPCacheHandler) bool {
modifiedSince, err := check.HTTPIfModifiedSince(r, handler.GetLastModifiedAt())
if err != nil {
handler.ClearResponse()
if err := handler.MessageResponse("invalid-header-date", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return false
} else if !modifiedSince {
// If the requested variant has not been modified since the time specified in this
// field, an entity will not be returned from the server; instead, a 304 (not
// modified) response will be returned without any message-body
handler.ClearResponse()
w.WriteHeader(http.StatusNotModified)
return false
}
return true
}
func checkIfUnmodifiedSince(w http.ResponseWriter, r *http.Request, handler HTTPCacheHandler) bool {
unmodifiedSince, err := check.HTTPIfUnmodifiedSince(r, handler.GetLastModifiedAt())
if err != nil {
handler.ClearResponse()
if err := handler.MessageResponse("invalid-header-date", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return false
} else if !unmodifiedSince {
// If the requested variant has been modified since the specified time, the server
// MUST NOT perform the requested operation, and MUST return a 412 (Precondition
// Failed)
handler.ClearResponse()
w.WriteHeader(http.StatusPreconditionFailed)
return false
}
return true
}
func (i *Database) Before(w http.ResponseWriter, r *http.Request) {
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error creating database connection. Details:", err)
return
}
i.databaseHandler.SetDatabaseSession(databaseSession)
i.databaseHandler.SetDatabase(database)
}
// Build the domain object doing a DNS query. To this function works the domain must be
// registered correctly and delegated in the DNS tree
func (h *DomainVerificationHandler) queryDomain(w http.ResponseWriter, r *http.Request) {
domain, err := scan.QueryDomain(h.FQDN)
if err != nil {
log.Println("Error while resolving FQDN. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
domainResponse := protocol.ToDomainResponse(domain, false)
h.Response = &domainResponse
}
func (c *JSONCodec) Before(w http.ResponseWriter, r *http.Request) {
m := strings.ToLower(r.Method)
c.parse(m)
if c.reqPosition >= 0 {
st := reflect.ValueOf(c.codecHandler).Elem()
decoder := json.NewDecoder(r.Body)
if err := decoder.Decode(st.Field(c.reqPosition).Addr().Interface()); err != nil {
log.Println("Received an invalid JSON. Details:", err)
if err := c.codecHandler.MessageResponse("invalid-json-content", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
}
}
func (h *DomainHandler) Delete(w http.ResponseWriter, r *http.Request) {
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if err := domainDAO.Remove(&h.domain); err != nil {
log.Println("Error while removing domain object. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
// Shelter could receive system signals for OS, so this method catch the signals to create
// smothly actions for each one. For example, when receives a KILL signal, we should wait
// to process all requests before finishing the server
func manageSystemSignals() {
go func() {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGTERM, syscall.SIGHUP)
for {
sig := <-sigs
if sig == syscall.SIGHUP {
if err := loadSettings(); err != nil {
log.Println("Error reloading confirguration file. Details:", err)
}
} else if sig == syscall.SIGTERM {
for _, listener := range restListeners {
if err := listener.Close(); err != nil {
log.Println("Error closing listener. Details:", err)
}
}
restListeners = []net.Listener{}
for _, listener := range clientListeners {
if err := listener.Close(); err != nil {
log.Println("Error closing listener. Details:", err)
}
}
clientListeners = []net.Listener{}
// TODO: Wait the last requests to be processed? On epossibly solution is to
// create a request counter in MUX, we wait while this counter is non-zero. If
// there's a scan running what are we going to do?
os.Exit(NoError)
}
}
}()
}
func (i *FQDN) Before(w http.ResponseWriter, r *http.Request) {
fqdn, err := model.NormalizeDomainName(i.fqdnHandler.GetFQDN())
if err != nil {
if err := i.fqdnHandler.MessageResponse("invalid-uri", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
i.fqdnHandler.SetFQDN(fqdn)
}
func checkIfMatch(w http.ResponseWriter, r *http.Request, handler HTTPCacheHandler) bool {
match := check.HTTPIfMatch(r, handler.GetETag())
if !match {
handler.ClearResponse()
// If "*" is given and no current entity exists or if none of the entity tags match
// the server MUST NOT perform the requested method, and MUST return a 412
// (Precondition Failed) response
if err := handler.MessageResponse("if-match-failed", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusPreconditionFailed)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return false
}
return true
}
// The HEAD method is identical to GET except that the server MUST NOT return a message-
// body in the response. But now the responsability for don't adding the body is from the
// mux while writing the response
func (h *DomainsHandler) retrieveDomains(w http.ResponseWriter, r *http.Request) {
var pagination dao.DomainDAOPagination
expand := false
filter := ""
for key, values := range r.URL.Query() {
key = strings.TrimSpace(key)
key = strings.ToLower(key)
// A key can have multiple values in a query string, we are going to always consider
// the last one (overwrite strategy)
for _, value := range values {
value = strings.TrimSpace(value)
value = strings.ToLower(value)
switch key {
case "orderby":
// OrderBy parameter will store the fields that the user want to be the keys of the sort
// algorithm in the result set and the direction that each sort field will have. The format
// that will be used is:
//
// <field1>:<direction1>@<field2>:<direction2>@...@<fieldN>:<directionN>
orderByParts := strings.Split(value, "@")
for _, orderByPart := range orderByParts {
orderByPart = strings.TrimSpace(orderByPart)
orderByAndDirection := strings.Split(orderByPart, ":")
var field, direction string
if len(orderByAndDirection) == 1 {
field, direction = orderByAndDirection[0], "asc"
} else if len(orderByAndDirection) == 2 {
field, direction = orderByAndDirection[0], orderByAndDirection[1]
} else {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByField, err := dao.DomainDAOOrderByFieldFromString(field)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByDirection, err := dao.DAOOrderByDirectionFromString(direction)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
pagination.OrderBy = append(pagination.OrderBy, dao.DomainDAOSort{
Field: orderByField,
Direction: orderByDirection,
})
}
case "pagesize":
var err error
pagination.PageSize, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page-size", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "page":
var err error
pagination.Page, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "expand":
expand = true
case "filter":
filter = value
}
}
}
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
domains, err := domainDAO.FindAll(&pagination, expand, filter)
if err != nil {
log.Println("Error while filtering domains objects. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
domainsResponse := protocol.ToDomainsResponse(domains, pagination, expand, filter)
h.Response = &domainsResponse
// Last-Modified is going to be the most recent date of the list
for _, domain := range domains {
if domain.LastModifiedAt.After(h.lastModifiedAt) {
h.lastModifiedAt = domain.LastModifiedAt
}
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.lastModifiedAt.Format(time.RFC1123))
w.WriteHeader(http.StatusOK)
}
// Put is responsable for checking a domain object on-the-fly without persisting in
// database, useful for pre-registration validations in the registry
func (h *DomainVerificationHandler) Put(w http.ResponseWriter, r *http.Request) {
// We need to set the FQDN in the domain request object because it is sent only in the
// URI and not in the domain request body to avoid information redudancy
h.Request.FQDN = h.GetFQDN()
var domain model.Domain
var err error
if domain, err = protocol.Merge(domain, h.Request); err != nil {
messageId := ""
switch err {
case protocol.ErrInvalidDNSKEY:
messageId = "invalid-dnskey"
case protocol.ErrInvalidDSAlgorithm:
messageId = "invalid-ds-algorithm"
case protocol.ErrInvalidDSDigestType:
messageId = "invalid-ds-digest-type"
case protocol.ErrInvalidIP:
messageId = "invalid-ip"
case protocol.ErrInvalidLanguage:
messageId = "invalid-language"
}
if len(messageId) == 0 {
log.Println("Error while merging domain objects for domain verification "+
"operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
} else {
if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
return
}
scan.ScanDomain(&domain)
// As we alredy did the scan, if the domain is registered in the system, we update it for this
// results. This also gives a more intuitive design for when the user wants to force a check a
// specific domain in the Shelter system
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if dbDomain, err := domainDAO.FindByFQDN(domain.FQDN); err == nil {
update := true
// Check if we have the same nameservers, and if so update the last status
if len(dbDomain.Nameservers) == len(domain.Nameservers) {
for i := range dbDomain.Nameservers {
dbNameserver := dbDomain.Nameservers[i]
nameserver := domain.Nameservers[i]
if dbNameserver.Host == nameserver.Host &&
dbNameserver.IPv4.Equal(nameserver.IPv4) &&
dbNameserver.IPv6.Equal(nameserver.IPv6) {
dbDomain.Nameservers[i].ChangeStatus(nameserver.LastStatus)
} else {
update = false
break
}
}
} else {
update = false
}
// Check if we have the same DS set, and if so update the last status
if len(dbDomain.DSSet) == len(domain.DSSet) {
for i := range dbDomain.DSSet {
dbDS := dbDomain.DSSet[i]
ds := domain.DSSet[i]
if dbDS.Keytag == ds.Keytag &&
dbDS.Algorithm == ds.Algorithm &&
dbDS.DigestType == ds.DigestType &&
dbDS.Digest == ds.Digest {
dbDomain.DSSet[i].ChangeStatus(ds.LastStatus)
} else {
update = false
break
}
}
} else {
update = false
}
if update {
// We don't care about errors resulted here, because the main idea of this service is to scan
// a domaion, not persist the results
domainDAO.Save(&dbDomain)
}
}
w.WriteHeader(http.StatusOK)
domainResponse := protocol.ToDomainResponse(domain, false)
h.Response = &domainResponse
}
// Notify is responsable for selecting the domains that should be notified in the system.
// It will send alert e-mails for each owner of a domain
func Notify() {
defer func() {
// Something went really wrong while notifying the owners. Log the error stacktrace
// and move out
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Printf("Panic detected while notifying the owners. Details: %v\n%s", r, buf)
}
}()
log.Info("Start notification job")
defer func() {
log.Info("End notification job")
}()
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
log.Println("Error while initializing database. Details:", err)
return
}
defer databaseSession.Close()
domainDAO := dao.DomainDAO{
Database: database,
}
domainChannel, err := domainDAO.FindAllAsyncToBeNotified(
config.ShelterConfig.Notification.NameserverErrorAlertDays,
config.ShelterConfig.Notification.NameserverTimeoutAlertDays,
config.ShelterConfig.Notification.DSErrorAlertDays,
config.ShelterConfig.Notification.DSTimeoutAlertDays,
// TODO: Should we move this configuration parameter to a place were both modules can
// access it. This sounds better for configuration deployment
config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
)
if err != nil {
log.Println("Error retrieving domains to notify. Details:", err)
return
}
// Dispatch the asynchronous part of the method
for {
// Get domain from the database (one-by-one)
domainResult := <-domainChannel
// Detect errors while retrieving a specific domain. We are not going to stop all the
// process when only one domain got an error
if domainResult.Error != nil {
log.Println("Error retrieving domain to notify. Details:", domainResult.Error)
continue
}
// Problem detected while retrieving a domain or we don't have domains anymore
if domainResult.Error != nil || domainResult.Domain == nil {
break
}
if err := notifyDomain(domainResult.Domain); err != nil {
log.Println("Error notifying a domain. Details:", err)
}
}
}
func (h *DomainVerificationHandler) handleDomainVerification(w http.ResponseWriter, r *http.Request) {
restAddress, err := retrieveRESTAddress()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error while retrieving the REST address. Details:", err)
return
}
var content []byte
if r.ContentLength > 0 {
content, err = ioutil.ReadAll(r.Body)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error while reading request body in web client. Details:", err)
return
}
}
var request *http.Request
if content == nil {
request, err = http.NewRequest(
r.Method,
fmt.Sprintf("%s%s", restAddress, r.RequestURI),
nil,
)
} else {
request, err = http.NewRequest(
r.Method,
fmt.Sprintf("%s%s", restAddress, r.RequestURI),
strings.NewReader(string(content)),
)
}
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error creating a request in web client. Details:", err)
return
}
request.Header.Set("Accept-Language", r.Header.Get("Accept-Language"))
response, err := signAndSend(request, content)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Println("Error signing and sending a request in web client. Details:", err)
return
}
if response.StatusCode != http.StatusOK &&
response.StatusCode != http.StatusBadRequest {
w.WriteHeader(http.StatusInternalServerError)
log.Println(fmt.Sprintf("Unexepected status code %d from /domain result "+
"in web client", response.StatusCode))
return
}
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(response.StatusCode)
if _, err := io.Copy(w, response.Body); err != nil {
// Here we already set the response code, so the client will receive a OK result
// without body
log.Println("Error copying REST response to web client response. Details:", err)
return
}
}
// The HEAD method is identical to GET except that the server MUST NOT return a message-
// body in the response. But now the responsability for don't adding the body is from the
// mux while writing the response
func (h *ScansHandler) retrieveScans(w http.ResponseWriter, r *http.Request) {
var pagination dao.ScanDAOPagination
expand := false
returnCurrent := false
for key, values := range r.URL.Query() {
key = strings.TrimSpace(key)
key = strings.ToLower(key)
// A key can have multiple values in a query string, we are going to always consider
// the last one (overwrite strategy)
for _, value := range values {
value = strings.TrimSpace(value)
value = strings.ToLower(value)
switch key {
case "orderby":
// OrderBy parameter will store the fields that the user want to be the keys of the sort
// algorithm in the result set and the direction that each sort field will have. The format
// that will be used is:
//
// <field1>:<direction1>@<field2>:<direction2>@...@<fieldN>:<directionN>
orderByParts := strings.Split(value, "@")
for _, orderByPart := range orderByParts {
orderByPart = strings.TrimSpace(orderByPart)
orderByAndDirection := strings.Split(orderByPart, ":")
var field, direction string
if len(orderByAndDirection) == 1 {
field, direction = orderByAndDirection[0], "asc"
} else if len(orderByAndDirection) == 2 {
field, direction = orderByAndDirection[0], orderByAndDirection[1]
} else {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByField, err := dao.ScanDAOOrderByFieldFromString(field)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
orderByDirection, err := dao.DAOOrderByDirectionFromString(direction)
if err != nil {
if err := h.MessageResponse("invalid-query-order-by", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
pagination.OrderBy = append(pagination.OrderBy, dao.ScanDAOSort{
Field: orderByField,
Direction: orderByDirection,
})
}
case "pagesize":
var err error
pagination.PageSize, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page-size", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "page":
var err error
pagination.Page, err = strconv.Atoi(value)
if err != nil {
if err := h.MessageResponse("invalid-query-page", ""); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
case "expand":
expand = true
case "current":
returnCurrent = true
}
}
}
scanDAO := dao.ScanDAO{
Database: h.GetDatabase(),
}
// As we need to inform the user about the number of items, we always try to retrieve the scan
// objects even if is requested only the current object
scans, err := scanDAO.FindAll(&pagination, expand)
if err != nil {
log.Println("Error while searching scans objects. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
var scansResponse protocol.ScansResponse
var current model.CurrentScan
if returnCurrent {
// The current page will be page zero to avoid misunderstandment
pagination.Page = 0
current = model.GetCurrentScan()
scansResponse = protocol.CurrentScanToScansResponse(current, pagination)
} else {
scansResponse = protocol.ScansToScansResponse(scans, pagination)
}
h.Response = &scansResponse
// Last-Modified is going to be the most recent date of the list
if returnCurrent {
h.lastModifiedAt = current.LastModifiedAt
} else {
for _, scan := range scans {
if scan.LastModifiedAt.After(h.lastModifiedAt) {
h.lastModifiedAt = scan.LastModifiedAt
}
}
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.lastModifiedAt.Format(time.RFC1123))
w.WriteHeader(http.StatusOK)
}
func (h *DomainHandler) Put(w http.ResponseWriter, r *http.Request) {
// We need to set the FQDN in the domain request object because it is sent only in the
// URI and not in the domain request body to avoid information redudancy
h.Request.FQDN = h.GetFQDN()
var err error
if h.domain, err = protocol.Merge(h.domain, h.Request); err != nil {
messageId := ""
switch err {
case model.ErrInvalidFQDN:
messageId = "invalid-fqdn"
case protocol.ErrInvalidDNSKEY:
messageId = "invalid-dnskey"
case protocol.ErrInvalidDSAlgorithm:
messageId = "invalid-ds-algorithm"
case protocol.ErrInvalidDSDigestType:
messageId = "invalid-ds-digest-type"
case protocol.ErrInvalidIP:
messageId = "invalid-ip"
case protocol.ErrInvalidLanguage:
messageId = "invalid-language"
}
if len(messageId) == 0 {
log.Println("Error while merging domain objects for create or "+
"update operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
} else {
if err := h.MessageResponse(messageId, r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
return
}
domainDAO := dao.DomainDAO{
Database: h.GetDatabase(),
}
if err := domainDAO.Save(&h.domain); err != nil {
if strings.Index(err.Error(), "duplicate key error index") != -1 {
if err := h.MessageResponse("conflict", r.URL.RequestURI()); err == nil {
w.WriteHeader(http.StatusConflict)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
} else {
log.Println("Error while saving domain object for create or "+
"update operation. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
w.Header().Add("ETag", h.GetETag())
w.Header().Add("Last-Modified", h.GetLastModifiedAt().Format(time.RFC1123))
if h.domain.Revision == 1 {
w.Header().Add("Location", "/domain/"+h.domain.FQDN)
w.WriteHeader(http.StatusCreated)
} else {
w.WriteHeader(http.StatusNoContent)
}
}
// Verify HTTP headers and fill context with user preferences
func (i *Validator) Before(w http.ResponseWriter, r *http.Request) {
// We first check the language header, because if it's acceptable the next messages are
// going to be returned in the language choosen by the user
language, ok := check.HTTPAcceptLanguage(r)
if ok {
i.validatorHandler.SetLanguage(language)
} else {
if err := i.validatorHandler.MessageResponse("accept-language-error", r.RequestURI); err == nil {
w.WriteHeader(http.StatusNotAcceptable)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
if !check.HTTPAccept(r) {
if err := i.validatorHandler.MessageResponse("accept-error", r.RequestURI); err == nil {
w.WriteHeader(http.StatusNotAcceptable)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
if !check.HTTPAcceptCharset(r) {
if err := i.validatorHandler.MessageResponse("accept-charset-error", r.RequestURI); err == nil {
w.WriteHeader(http.StatusNotAcceptable)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
if !check.HTTPContentType(r) {
if err := i.validatorHandler.MessageResponse("invalid-content-type", r.RequestURI); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
if r.Body != nil {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("Error reading the request. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
defer func() {
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
}()
if !check.HTTPContentMD5(r, body) {
if err := i.validatorHandler.MessageResponse("invalid-content-md5", r.RequestURI); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
}
timeFrameOK, err := check.HTTPDate(r)
if err != nil {
if err := i.validatorHandler.MessageResponse("invalid-header-date", r.RequestURI); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
} else if !timeFrameOK {
if err := i.validatorHandler.MessageResponse("invalid-date-time-frame", r.RequestURI); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
authorized, err := check.HTTPAuthorization(r, func(secretId string) (string, error) {
s, ok := config.ShelterConfig.RESTServer.Secrets[secretId]
if !ok {
return "", ErrSecretNotFound
}
s, err = secret.Decrypt(s)
if err != nil {
return "", err
}
return s, nil
})
if err == nil && !authorized {
w.WriteHeader(http.StatusUnauthorized)
return
} else if authorized {
return
}
messageId := ""
switch err {
case check.ErrHTTPContentTypeNotFound:
messageId = "content-type-missing"
case check.ErrHTTPContentMD5NotFound:
messageId = "content-md5-missing"
case check.ErrHTTPDateNotFound:
messageId = "date-missing"
case check.ErrHTTPAuthorizationNotFound:
messageId = "authorization-missing"
case check.ErrInvalidHTTPAuthorization:
messageId = "invalid-authorization"
case ErrSecretNotFound:
messageId = "secret-not-found"
}
if len(messageId) == 0 {
log.Println("Error checking authorization. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
} else {
if err := i.validatorHandler.MessageResponse(messageId, r.RequestURI); err == nil {
w.WriteHeader(http.StatusBadRequest)
} else {
log.Println("Error while writing response. Details:", err)
w.WriteHeader(http.StatusInternalServerError)
}
}
}
// The main function of the system is responsable for deploying all system components that
// are enabled. For now we have the REST server and the scan system
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
logPath := filepath.Join(
config.ShelterConfig.BasePath,
config.ShelterConfig.LogFilename,
)
if err := log.SetOutput(logPath); err != nil {
log.Println(err)
return
}
defer log.Close()
if config.ShelterConfig.RESTServer.Enabled {
var err error
restListeners, err = rest.Listen()
if err != nil {
log.Println("Error while aquiring interfaces for REST server. Details:", err)
os.Exit(ErrListeningRESTInterfaces)
}
if err := rest.Start(restListeners); err != nil {
log.Println("Error starting the REST server. Details:", err)
os.Exit(ErrStartingRESTServer)
}
log.Info("REST server started")
}
if config.ShelterConfig.WebClient.Enabled {
var err error
clientListeners, err = client.Listen()
if err != nil {
log.Println("Error while aquiring interfaces for Client server. Details:", err)
os.Exit(ErrListeningClientInterfaces)
}
if err := client.Start(clientListeners); err != nil {
log.Println("Error starting the Client server. Details:", err)
os.Exit(ErrStartingWebClient)
}
log.Info("Web client started")
}
if config.ShelterConfig.Scan.Enabled {
// Attention: Cannot use timezone abbreviations
// http://stackoverflow.com/questions/25368415/golang-timezone-parsing
scanTime, err := time.Parse("15:04:05 -0700", config.ShelterConfig.Scan.Time)
if err != nil {
log.Println("Scan time not in a valid format. Details:", err)
os.Exit(ErrScanTimeFormat)
}
scanTime = scanTime.UTC()
now := time.Now().UTC()
nextExecution := time.Date(
now.Year(),
now.Month(),
now.Day(),
scanTime.Hour(),
scanTime.Minute(),
scanTime.Second(),
scanTime.Nanosecond(),
scanTime.Location(),
)
scheduler.Register(scheduler.Job{
Type: scheduler.JobTypeScan,
NextExecution: nextExecution,
Interval: time.Duration(config.ShelterConfig.Scan.IntervalHours) * time.Hour,
Task: scan.ScanDomains,
})
// Must be called after registering in scheduler, because we retrieve the next execution time
// from it
if err := model.InitializeCurrentScan(); err != nil {
log.Println("Current scan information got an error while initializing. Details:", err)
os.Exit(ErrCurrentScanInitialize)
}
}
if config.ShelterConfig.Notification.Enabled {
if err := notification.LoadTemplates(); err != nil {
log.Println("Error loading notification templates. Details:", err)
os.Exit(ErrNotificationTemplates)
}
// Attention: Cannot use timezone abbreviations
// http://stackoverflow.com/questions/25368415/golang-timezone-parsing
notificationTime, err := time.Parse("15:04:05 -0700", config.ShelterConfig.Scan.Time)
if err != nil {
log.Println("Scan time not in a valid format. Details:", err)
os.Exit(ErrScanTimeFormat)
}
notificationTime = notificationTime.UTC()
now := time.Now().UTC()
nextExecution := time.Date(
now.Year(),
now.Month(),
now.Day(),
notificationTime.Hour(),
notificationTime.Minute(),
notificationTime.Second(),
notificationTime.Nanosecond(),
notificationTime.Location(),
)
scheduler.Register(scheduler.Job{
Type: scheduler.JobTypeNotification,
NextExecution: nextExecution,
Interval: time.Duration(config.ShelterConfig.Notification.IntervalHours) * time.Hour,
Task: notification.Notify,
})
}
scheduler.Start()
select {}
}
// Function responsible for running the domain scan system, checking the configuration of each
// domain in the database according to an algorithm. This method is synchronous and will return only
// after the scan proccess is done
func ScanDomains() {
defer func() {
// Something went really wrong while scanning the domains. Log the error stacktrace
// and move out
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Printf("Panic detected while scanning domains. Details: %v\n%s", r, buf)
}
}()
log.Info("Start scan job")
defer func() {
log.Info("End scan job")
}()
log.Debugf("Initializing database with the parameters: URIS - %v | Name - %s | Auth - %t | Username - %s",
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
)
database, databaseSession, err := mongodb.Open(
config.ShelterConfig.Database.URIs,
config.ShelterConfig.Database.Name,
config.ShelterConfig.Database.Auth.Enabled,
config.ShelterConfig.Database.Auth.Username,
config.ShelterConfig.Database.Auth.Password,
)
if err != nil {
log.Println("Error while initializing database. Details:", err)
return
}
defer databaseSession.Close()
injector := NewInjector(
database,
config.ShelterConfig.Scan.DomainsBufferSize,
config.ShelterConfig.Scan.VerificationIntervals.MaxOKDays,
config.ShelterConfig.Scan.VerificationIntervals.MaxErrorDays,
config.ShelterConfig.Scan.VerificationIntervals.MaxExpirationAlertDays,
)
querierDispatcher := NewQuerierDispatcher(
config.ShelterConfig.Scan.NumberOfQueriers,
config.ShelterConfig.Scan.DomainsBufferSize,
config.ShelterConfig.Scan.UDPMaxSize,
time.Duration(config.ShelterConfig.Scan.Timeouts.DialSeconds)*time.Second,
time.Duration(config.ShelterConfig.Scan.Timeouts.ReadSeconds)*time.Second,
time.Duration(config.ShelterConfig.Scan.Timeouts.WriteSeconds)*time.Second,
config.ShelterConfig.Scan.ConnectionRetries,
)
collector := NewCollector(
database,
config.ShelterConfig.Scan.SaveAtOnce,
)
// Create a new scan information
model.StartNewScan()
var scanGroup sync.WaitGroup
errorsChannel := make(chan error, config.ShelterConfig.Scan.ErrorsBufferSize)
domainsToQueryChannel := injector.Start(&scanGroup, errorsChannel)
domainsToSaveChannel := querierDispatcher.Start(&scanGroup, domainsToQueryChannel)
collector.Start(&scanGroup, domainsToSaveChannel, errorsChannel)
// Keep track of errors for the scan information structure
errorDetected := false
go func() {
for {
select {
case err := <-errorsChannel:
// Detect the poison pill to finish the error listener go routine. This poison
// pill should be sent after all parts of the scan are done and we are sure that
// we don't have any error to log anymore
if err == nil {
return
} else {
errorDetected = true
log.Println("Error detected while executing the scan. Details:", err)
}
}
}
}()
// Wait for all parts of the scan to finish their job
scanGroup.Wait()
// Finish the error listener sending a poison pill
errorsChannel <- nil
scanDAO := dao.ScanDAO{
Database: database,
}
// Save the scan information for future reports
if err := model.FinishAndSaveScan(errorDetected, scanDAO.Save); err != nil {
log.Println("Error while saving scan information. Details:", err)
}
}
