//updateCert takes the input certificate and updates the map holding all the certificates to be pushed.
//If the certificates has already been inserted it updates the existing record else it creates it.
func updateCert(cert *x509.Certificate, parentSignature string, domain, ip, TSName string, valInfo *certificate.ValidationInfo, certmap map[string]certificate.Certificate) {
id := certificate.SHA256Hash(cert.Raw)
if storedCert, ok := certmap[id]; !ok {
certmap[id] = certificate.CertToStored(cert, parentSignature, domain, ip, TSName, valInfo)
} else {
parentFound := false
for _, p := range storedCert.ParentSignature {
if parentSignature == p {
parentFound = true
break
}
}
if !parentFound {
storedCert.ParentSignature = append(storedCert.ParentSignature, parentSignature)
}
if !storedCert.CA {
if storedCert.ScanTarget != domain {
log.WithFields(logrus.Fields{
"domain": storedCert.ScanTarget,
"domain_input": domain,
"certificate": storedCert.Hashes.SHA256,
}).Warning("Different domain input")
}
//add IP ( single domain may be served by multiple IPs )
ipFound := false
for _, i := range storedCert.IPs {
if ip == i {
ipFound = true
break
}
}
if !ipFound {
storedCert.IPs = append(storedCert.IPs, ip)
}
}
storedCert.ValidationInfo[TSName] = *valInfo
certmap[id] = storedCert
}
}
func main() {
var (
err error
offset int
)
// create a certificate transparency client
ctLog := client.New("http://ct.googleapis.com/aviator", nil)
httpCli := &http.Client{
Transport: &http.Transport{
DisableCompression: true,
DisableKeepAlives: false,
},
Timeout: 10 * time.Second,
}
if len(os.Args) > 1 {
offset, err = strconv.Atoi(os.Args[1])
if err != nil {
log.Fatal(err)
}
}
for {
log.Printf("retrieving CT logs %d to %d", offset, offset+100)
rawEnts, err := ctLog.GetEntries(int64(offset), int64(offset+100))
if err != nil {
log.Fatal(err)
}
// loop over CT records
for i, ent := range rawEnts {
log.Printf("CT index=%d", offset+i)
var cert *x509.Certificate
switch ent.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
cert, err = x509.ParseCertificate(ent.Leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
cert, err = x509.ParseTBSCertificate(ent.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
}
if err != nil {
log.Fatal(err)
}
log.Printf("CN=%s", cert.Subject.CommonName)
log.Printf("Not Before=%s", cert.NotBefore)
log.Printf("Not After=%s", cert.NotAfter)
// Format the PEM certificate
payload := base64.StdEncoding.EncodeToString(cert.Raw)
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "-----BEGIN CERTIFICATE-----\n")
for len(payload) > 0 {
chunkLen := len(payload)
if chunkLen > 64 {
chunkLen = 64
}
fmt.Fprintf(buf, "%s\n", payload[0:chunkLen])
payload = payload[chunkLen:]
}
fmt.Fprintf(buf, "-----END CERTIFICATE-----")
// create a mime/multipart form with the certificate
var b bytes.Buffer
w := multipart.NewWriter(&b)
fw, err := w.CreateFormFile("certificate", certificate.SHA256Hash(cert.Raw))
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(fw, buf)
if err != nil {
log.Fatal(err)
}
w.Close()
// post the form to the tls-observatory api
r, err := http.NewRequest("POST", "https://tls-observatory.services.mozilla.com/api/v1/certificate", &b)
if err != nil {
log.Fatal(err)
}
r.Header.Set("Content-Type", w.FormDataContentType())
resp, err := httpCli.Do(r)
if err != nil {
log.Printf("%v\n\n", err)
continue
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
if resp.StatusCode != http.StatusCreated {
log.Fatalf("Expected HTTP 201 Created, got %q\n%s", resp.Status, body)
}
// parse the returned cert
var tlsobs_cert certificate.Certificate
err = json.Unmarshal(body, &tlsobs_cert)
if err != nil {
log.Fatal(err)
}
log.Printf("https://tls-observatory.services.mozilla.com/api/v1/certificate?id=%d\n\n", tlsobs_cert.ID)
}
offset += 100
}
}
// PostCertificateHandler handles the POST /certificate endpoint of the api.
// It receives a single PEM encoded certificate, parses it, inserts it
// into the database and returns results in JSON.
func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
setResponseHeader(w)
log := logger.GetLogger()
log.WithFields(logrus.Fields{
"form values": r.Form.Encode(),
"headers": r.Header,
"uri": r.URL.RequestURI(),
}).Debug("PostCertificate Endpoint received request")
val := r.Context().Value(dbKey)
if val == nil {
httpError(w, http.StatusInternalServerError, "Could not find database handler in request context")
return
}
db := val.(*pg.DB)
_, certHeader, err := r.FormFile("certificate")
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
certReader, err := certHeader.Open()
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
certPEM, err := ioutil.ReadAll(certReader)
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not read certificate from form data: %v", err))
return
}
block, _ := pem.Decode([]byte(certPEM))
if block == nil {
httpError(w, http.StatusBadRequest,
"Failed to parse certificate PEM")
return
}
certX509, err := x509.ParseCertificate(block.Bytes)
if err != nil {
httpError(w, http.StatusBadRequest,
fmt.Sprintf("Could not parse X.509 certificate: %v", err))
return
}
certHash := certificate.SHA256Hash(certX509.Raw)
id, err := db.GetCertIDBySHA256Fingerprint(certHash)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to lookup certificate hash in database: %v", err))
return
}
if id > 0 {
// if the cert already exists in DB, return early
log.Printf("cert id %d already exists in database, returning it", id)
jsonCertFromID(w, r, id)
return
}
var valInfo certificate.ValidationInfo
cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo)
id, err = db.InsertCertificate(&cert)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to store certificate in database: %v", err))
return
}
cert.ID = id
// If the cert is self-signed (aka. Root CA), we're done here
if cert.IsSelfSigned() {
jsonCertFromID(w, r, cert.ID)
return
}
// to insert the trust, first build the certificate paths, then insert one trust
// entry for each known parent of the cert
paths, err := db.GetCertPaths(&cert)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to retrieve chains from database: %v", err))
return
}
for _, parent := range paths.Parents {
cert.ValidationInfo = parent.GetValidityMap()
_, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID)
if err != nil {
httpError(w, http.StatusInternalServerError,
fmt.Sprintf("Failed to store trust in database: %v", err))
return
}
}
jsonCertFromID(w, r, cert.ID)
return
}
//isChainValid creates the valid certificate chains by combining the chain retrieved with the provided truststore.
//It return true if it finds at least on validation chain or false if no valid chain of trust can be created.
//It also updates the certificate map which gets pushed at the end of each iteration.
func isChainValid(endEntity *x509.Certificate, intermediates []*x509.Certificate, truststore *certificate.TrustStore, domain, IP string, certmap map[string]certificate.Certificate) bool {
valInfo := &certificate.ValidationInfo{
IsValid: true,
}
// build a CA verification pool from the list of cacerts
interPool := x509.NewCertPool()
for _, entity := range intermediates {
interPool.AddCert(entity)
}
// get a list of domains this certificate is supposedly valid for
// if the end entity is a CA, use its common name
dnsName := domain
if endEntity.IsCA {
dnsName = endEntity.Subject.CommonName
}
// configure the verification logic to use the current trustore
opts := x509.VerifyOptions{
DNSName: dnsName,
Intermediates: interPool,
Roots: truststore.Certs,
}
// Verify attempts to build all the path between the end entity and the
// root in the truststore that validate the certificate
// If no valid path is found, err is not nil and the certificate is not trusted
chains, err := endEntity.Verify(opts)
if err == nil {
// the end entity is trusted, we need to go through each
// chain of trust and store them in database
for i, chain := range chains {
log.WithFields(logrus.Fields{
"trust chain no": i,
"path len": len(chain),
}).Debug("domain: " + domain)
// loop through each certificate in the chain and
for _, cert := range chain {
parentSignature := ""
parentCert := getFirstParent(cert, chain)
if parentCert != nil {
parentSignature = certificate.SHA256Hash(parentCert.Raw)
} else {
log.Println("could not retrieve parent for " + dnsName)
}
updateCert(cert, parentSignature, domain, IP, truststore.Name, valInfo, certmap)
}
}
return true
}
// the certificate is not trusted.
// we store the cert in DB with its validation error
if len(chains) > 0 {
log.WithFields(logrus.Fields{
"domain": domain,
}).Warning("Got validation error but chains are populated")
}
valInfo.ValidationError = err.Error()
valInfo.IsValid = false
parentSignature := ""
c := getFirstParent(endEntity, intermediates)
if c != nil {
parentSignature = certificate.SHA256Hash(c.Raw)
} else {
log.WithFields(logrus.Fields{
"domain": domain,
"servercert": certificate.SHA256Hash(endEntity.Raw),
}).Info("Could not get parent")
}
updateCert(endEntity, parentSignature, domain, IP, truststore.Name, valInfo, certmap)
return false
}
//handleCertChain takes the chain retrieved from the queue and tries to validate it
//against each of the truststores provided. The function returns the ID of the end
//entity certificate (or -1 if not stored), the ID of the trust entry (or -1 if not
//stored) and an error message.
func handleCertChain(chain *certificate.Chain) (int64, int64, error) {
var intermediates []*x509.Certificate
var endEntity *x509.Certificate
endEntity = nil
for chaincertno, data := range chain.Certs { //create certificate chain from chain struct
certRaw, err := base64.StdEncoding.DecodeString(data)
if err != nil {
log.WithFields(logrus.Fields{
"domain": chain.Domain,
"cert no": chaincertno,
"error": err.Error(),
}).Warning("Could not decode raw cert from base64")
}
var cert *x509.Certificate
cert, err = x509.ParseCertificate(certRaw)
if err != nil {
log.WithFields(logrus.Fields{
"domain": chain.Domain,
"cert no": chaincertno,
"error": err.Error(),
}).Warning("Could not parse raw cert")
}
// if certificate is an authority,
// append it to the list of intermediate certs and go to the next one
if cert.IsCA {
intermediates = append(intermediates, cert)
continue
}
// if we don't yet have an end entity in this chain
// set the current cert as the end entity
if endEntity == nil {
endEntity = cert
continue
}
// here we have a cert that's an end entity when we already
// found one in the chain. It's possible that it's an old
// V2 cert that's actually an intermediate by doesn't have the
// CA flag set
log.WithFields(logrus.Fields{
"domain": chain.Domain,
"cert no": chaincertno,
"cert fingerprint": certificate.SHA256Hash(cert.Raw),
}).Warning("Second End Entity cert found in chain received from server. Adding it to intermediates.")
if cert.Version < 3 {
log.WithFields(logrus.Fields{
"domain": chain.Domain,
"cert no": chaincertno,
"cert fingerprint": certificate.SHA256Hash(cert.Raw),
}).Debug("Probably an old root CA cert")
intermediates = append(intermediates, cert)
}
}
if endEntity == nil {
log.WithFields(logrus.Fields{
"domain": chain.Domain,
}).Warning("the certificate chain did not contain an end entity certificate")
}
var certmap = make(map[string]certificate.Certificate)
// Test the end entity cert with its chain against each of the truststore
for _, truststore := range trustStores {
if endEntity != nil && isChainValid(endEntity, intermediates, &truststore, chain.Domain, chain.IP, certmap) {
// If we have an end entity cert and its chain of trust is valid, our work
// here is done, move to the next truststore
continue
}
// to end up here either there was no leaf certificate retrieved
// or it was retrieved but it was not valid so we must check the remainder of the chain
for i, cert := range intermediates {
inter := append(intermediates[:i], intermediates[i+1:]...)
isChainValid(cert, inter, &truststore, chain.Domain, chain.IP, certmap)
}
}
log.WithFields(logrus.Fields{
"domain": chain.Domain,
"map length": len(certmap),
}).Debug("Certificate Map length")
return storeCertificates(certmap)
}
func Setup(c config.Config) {
ts := c.TrustStores
for _, tsName := range allowedTruststoreNames {
path := ""
switch tsName {
case certificate.Ubuntu_TS_name:
path = ts.UbuntuTS
case certificate.Mozilla_TS_name:
path = ts.MozillaTS
case certificate.Microsoft_TS_name:
path = ts.MicrosoftTS
case certificate.Apple_TS_name:
path = ts.AppleTS
case certificate.Android_TS_name:
path = ts.AndroidTS
default:
log.WithFields(logrus.Fields{
"tsname": tsName,
}).Warning("Invalid Truststore name.")
}
log.WithFields(logrus.Fields{
"tsname": tsName,
"path": path,
}).Debug("Loading Truststore")
// load the entire trustore into pooldata, then iterate over each PEM block
// until all of pooldata is read
poolData, err := ioutil.ReadFile(path)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"error": err.Error(),
}).Warning("Failed to load truststore")
}
certPool := x509.NewCertPool()
poollen := 0
// keep a list of cert hashes currently in this truststore
// to remove certs no longer in it
certHashes := make([]string, 0)
for len(poolData) > 0 {
// read the next PEM block, ignore non CERTIFICATE entries
var block *pem.Block
block, poolData = pem.Decode(poolData)
if block == nil {
break
}
if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
continue
}
// parse the current PEM block into a certificate, ignore failures
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"cert no": poollen + 1,
"error": err.Error(),
}).Warning("Could not parse PEM block")
continue
}
// if the cert version is 1 or 2, the cert will not contain a CA: True extension
// so we set it manually instead. This assumes that all certs found in truststoresfile:///media/Projects/GoProjects/src/github.com/mozilla/TLS-Observer/certificate/analyserPool.go
// should be considered valid certificate authorities
if cert.Version < 3 {
cert.IsCA = true
}
if !cert.IsCA {
log.WithFields(logrus.Fields{
"tsname": tsName,
"cert no": poollen + 1,
"SHA1": certificate.SHA1Hash(cert.Raw),
}).Warning("Certificate in truststore is not a CA cert")
}
certPool.AddCert(cert)
//Push current certificate to DB as trusted
v := &certificate.ValidationInfo{}
v.IsValid = true
certHash := certificate.SHA256Hash(cert.Raw)
certHashes = append(certHashes, certHash)
parentSignature := ""
if cert.Subject.CommonName == cert.Issuer.CommonName {
// self-signed, parent sig is self sig
parentSignature = certHash
}
var id int64 = -1
id, err = db.GetCertIDBySHA256Fingerprint(certHash)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"certificate": certificate.SHA256Hash(cert.Raw),
"error": err.Error(),
}).Error("Could not check if certificate is in db")
}
if id == -1 {
// insert certificate for the first time
vinfo := &certificate.ValidationInfo{}
vinfo.IsValid = true
vinfo.ValidationError = ""
st := certificate.CertToStored(cert, parentSignature, "", "", tsName, vinfo)
id, err = db.InsertCertificate(&st)
if err != nil {
log.WithFields(logrus.Fields{
"certificate": certificate.SHA256Hash(cert.Raw),
"error": err.Error(),
}).Error("Could not insert certificate in db")
}
}
switch tsName {
case certificate.Ubuntu_TS_name:
err = db.AddCertToUbuntuTruststore(id)
case certificate.Mozilla_TS_name:
err = db.AddCertToMozillaTruststore(id)
case certificate.Microsoft_TS_name:
err = db.AddCertToMicrosoftTruststore(id)
case certificate.Apple_TS_name:
err = db.AddCertToAppleTruststore(id)
case certificate.Android_TS_name:
err = db.AddCertToAndroidTruststore(id)
}
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"id": id,
"error": err.Error(),
}).Error("Could not update certificate trust in db")
}
poollen++
}
// We have a list of certificates in the current truststore and
// we use it to disable certs no longer in in
err = db.RemoveCACertFromTruststore(certHashes, tsName)
if err != nil {
log.WithFields(logrus.Fields{
"tsname": tsName,
"error": err.Error(),
}).Fatal("Failed to update trust of certificates no longer in truststore")
}
trustStores = append(trustStores, certificate.TrustStore{tsName, certPool})
log.WithFields(logrus.Fields{
"tsname": tsName,
"certificates loaded": poollen,
}).Info("Successfully loaded TS ")
}
if len(trustStores) == 0 {
log.Error("No truststores loaded, TLS certificate retrieval & analysis won't be available")
}
}