//updateCert takes the input certificate and updates the map holding all the certificates to be pushed.
//If the certificates has already been inserted it updates the existing record else it creates it.
func updateCert(cert *x509.Certificate, parentSignature string, domain, ip, TSName string, valInfo *certificate.ValidationInfo, certmap map[string]certificate.Certificate) {
id := certificate.SHA256Hash(cert.Raw)
if storedCert, ok := certmap[id]; !ok {
certmap[id] = certificate.CertToStored(cert, parentSignature, domain, ip, TSName, valInfo)
} else {
parentFound := false
for _, p := range storedCert.ParentSignature {
if parentSignature == p {
parentFound = true
break
}
}
if !parentFound {
storedCert.ParentSignature = append(storedCert.ParentSignature, parentSignature)
}
if !storedCert.CA {
if storedCert.ScanTarget != domain {
log.WithFields(logrus.Fields{
"domain": storedCert.ScanTarget,
"domain_input": domain,
"certificate": storedCert.Hashes.SHA256,
}).Warning("Different domain input")
}
//add IP ( single domain may be served by multiple IPs )
ipFound := false
for _, i := range storedCert.IPs {
if ip == i {
ipFound = true
break
}
}
if !ipFound {
storedCert.IPs = append(storedCert.IPs, ip)
}
}
storedCert.ValidationInfo[TSName] = *valInfo
certmap[id] = storedCert
}
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
batch := 0
for {
fmt.Printf("\nProcessing batch %d to %d: ", batch, batch+100)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE x509_certificatepolicies IS NULL OR x509_certificatepolicies='null'
OR permitted_names IS NULL OR permitted_names='null'
ORDER BY id ASC
LIMIT 100`)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
var valInfo certificate.ValidationInfo
cert := certificate.CertToStored(c, "", "", "", "", &valInfo)
policies, err := json.Marshal(cert.X509v3Extensions.PolicyIdentifiers)
if err != nil {
log.Printf("error while marshalling policies for cert %d: %v", id, err)
continue
}
permittednames, err := json.Marshal(cert.X509v3Extensions.PermittedNames)
if err != nil {
log.Printf("error while marshalling permitted names for cert %d: %v", id, err)
continue
}
log.Printf("id=%d, subject=%s, policies=%s, permittednames=%s", id, cert.Subject.String(), policies, permittednames)
_, err = db.Exec(`UPDATE certificates
SET x509_certificatepolicies=$1,
permitted_names=$2,
is_name_constrained=$3
WHERE id=$4`,
policies,
permittednames,
cert.X509v3Extensions.IsNameConstrained,
id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
fmt.Printf(".")
}
if i == 0 {
fmt.Println("done!")
break
}
//offset += limit
batch += 100
}
}
// PostCertificateHandler handles the POST /certificate endpoint of the api.
// It receives a single PEM encoded certificate, parses it, inserts it
// into the database and returns results in JSON.
func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
setResponseHeader(w)
log := logger.GetLogger()
log.WithFields(logrus.Fields{
"form values": r.Form.Encode(),
"headers": r.Header,
"uri": r.URL.RequestURI(),
}).Debug("PostCertificate Endpoint received request")
val := r.Context().Value(dbKey)
if val == nil {
httpError(w, http.StatusInternalServerError, "Could not find database handler in request context")
return
}
db := val.(*pg.DB)
_, certHeader, err := r.FormFile("certificate")
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
certReader, err := certHeader.Open()
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
certPEM, err := ioutil.ReadAll(certReader)
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
block, _ := pem.Decode([]byte(certPEM))
if block == nil {
httpError(w, http.StatusBadRequest,
"Failed to parse certificate PEM")
return
}
certX509, err := x509.ParseCertificate(block.Bytes)
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not parse X.509 certificate: %v", err))
return
}
certHash := certificate.SHA256Hash(certX509.Raw)
id, err := db.GetCertIDBySHA256Fingerprint(certHash)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to lookup certificate hash in database: %v", err))
return
}
if id > 0 {
// if the cert already exists in DB, return early
log.Printf("cert id %d already exists in database, returning it", id)
jsonCertFromID(w, r, id)
return
}
var valInfo certificate.ValidationInfo
cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo)
id, err = db.InsertCertificate(&cert)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to store certificate in database: %v", err))
return
}
cert.ID = id
// If the cert is self-signed (aka. Root CA), we're done here
if cert.IsSelfSigned() {
jsonCertFromID(w, r, cert.ID)
return
}
// to insert the trust, first build the certificate paths, then insert one trust
// entry for each known parent of the cert
paths, err := db.GetCertPaths(&cert)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to retrieve chains from database: %v", err))
return
}
for _, parent := range paths.Parents {
cert.ValidationInfo = parent.GetValidityMap()
_, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to store trust in database: %v", err))
return
}
}
jsonCertFromID(w, r, cert.ID)
return
}
func Setup(c config.Config) {
ts := c.TrustStores
for _, tsName := range allowedTruststoreNames {
path := ""
switch tsName {
case certificate.Ubuntu_TS_name:
path = ts.UbuntuTS
case certificate.Mozilla_TS_name:
path = ts.MozillaTS
case certificate.Microsoft_TS_name:
path = ts.MicrosoftTS
case certificate.Apple_TS_name:
path = ts.AppleTS
case certificate.Android_TS_name:
path = ts.AndroidTS
default:
log.WithFields(logrus.Fields{
"tsname": tsName,
}).Warning("Invalid Truststore name.")
}
log.WithFields(logrus.Fields{
"tsname": tsName,
"path": path,
}).Debug("Loading Truststore")
// load the entire trustore into pooldata, then iterate over each PEM block
// until all of pooldata is read
poolData, err := ioutil.ReadFile(path)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"error": err.Error(),
}).Warning("Failed to load truststore")
}
certPool := x509.NewCertPool()
poollen := 0
// keep a list of cert hashes currently in this truststore
// to remove certs no longer in it
certHashes := make([]string, 0)
for len(poolData) > 0 {
// read the next PEM block, ignore non CERTIFICATE entries
var block *pem.Block
block, poolData = pem.Decode(poolData)
if block == nil {
break
}
if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
continue
}
// parse the current PEM block into a certificate, ignore failures
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"cert no": poollen + 1,
"error": err.Error(),
}).Warning("Could not parse PEM block")
continue
}
// if the cert version is 1 or 2, the cert will not contain a CA: True extension
// so we set it manually instead. This assumes that all certs found in truststoresfile:///media/Projects/GoProjects/src/github.com/mozilla/TLS-Observer/certificate/analyserPool.go
// should be considered valid certificate authorities
if cert.Version < 3 {
cert.IsCA = true
}
if !cert.IsCA {
log.WithFields(logrus.Fields{
"tsname": tsName,
"cert no": poollen + 1,
"SHA1": certificate.SHA1Hash(cert.Raw),
}).Warning("Certificate in truststore is not a CA cert")
}
certPool.AddCert(cert)
//Push current certificate to DB as trusted
v := &certificate.ValidationInfo{}
v.IsValid = true
certHash := certificate.SHA256Hash(cert.Raw)
certHashes = append(certHashes, certHash)
parentSignature := ""
if cert.Subject.CommonName == cert.Issuer.CommonName {
// self-signed, parent sig is self sig
parentSignature = certHash
}
var id int64 = -1
id, err = db.GetCertIDBySHA256Fingerprint(certHash)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"certificate": certificate.SHA256Hash(cert.Raw),
"error": err.Error(),
}).Error("Could not check if certificate is in db")
}
if id == -1 {
// insert certificate for the first time
vinfo := &certificate.ValidationInfo{}
vinfo.IsValid = true
vinfo.ValidationError = ""
st := certificate.CertToStored(cert, parentSignature, "", "", tsName, vinfo)
id, err = db.InsertCertificate(&st)
if err != nil {
log.WithFields(logrus.Fields{
"certificate": certificate.SHA256Hash(cert.Raw),
"error": err.Error(),
}).Error("Could not insert certificate in db")
}
}
switch tsName {
case certificate.Ubuntu_TS_name:
err = db.AddCertToUbuntuTruststore(id)
case certificate.Mozilla_TS_name:
err = db.AddCertToMozillaTruststore(id)
case certificate.Microsoft_TS_name:
err = db.AddCertToMicrosoftTruststore(id)
case certificate.Apple_TS_name:
err = db.AddCertToAppleTruststore(id)
case certificate.Android_TS_name:
err = db.AddCertToAndroidTruststore(id)
}
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"id": id,
"error": err.Error(),
}).Error("Could not update certificate trust in db")
}
poollen++
}
// We have a list of certificates in the current truststore and
// we use it to disable certs no longer in in
err = db.RemoveCACertFromTruststore(certHashes, tsName)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"error": err.Error(),
}).Fatal("Failed to update trust of certificates no longer in truststore")
}
trustStores = append(trustStores, certificate.TrustStore{tsName, certPool})
log.WithFields(logrus.Fields{
"tsname": tsName,
"certificates loaded": poollen,
}).Info("Successfully loaded TS ")
}
if len(trustStores) == 0 {
log.Error("No truststores loaded, TLS certificate retrieval & analysis won't be available")
}
}