//updateCert takes the input certificate and updates the map holding all the certificates to be pushed. //If the certificates has already been inserted it updates the existing record else it creates it. func updateCert(cert *x509.Certificate, parentSignature string, domain, ip, TSName string, valInfo *certificate.ValidationInfo, certmap map[string]certificate.Certificate) { id := certificate.SHA256Hash(cert.Raw) if storedCert, ok := certmap[id]; !ok { certmap[id] = certificate.CertToStored(cert, parentSignature, domain, ip, TSName, valInfo) } else { parentFound := false for _, p := range storedCert.ParentSignature { if parentSignature == p { parentFound = true break } } if !parentFound { storedCert.ParentSignature = append(storedCert.ParentSignature, parentSignature) } if !storedCert.CA { if storedCert.ScanTarget != domain { log.WithFields(logrus.Fields{ "domain": storedCert.ScanTarget, "domain_input": domain, "certificate": storedCert.Hashes.SHA256, }).Warning("Different domain input") } //add IP ( single domain may be served by multiple IPs ) ipFound := false for _, i := range storedCert.IPs { if ip == i { ipFound = true break } } if !ipFound { storedCert.IPs = append(storedCert.IPs, ip) } } storedCert.ValidationInfo[TSName] = *valInfo certmap[id] = storedCert } }
func main() { var ( err error offset int ) // create a certificate transparency client ctLog := client.New("http://ct.googleapis.com/aviator", nil) httpCli := &http.Client{ Transport: &http.Transport{ DisableCompression: true, DisableKeepAlives: false, }, Timeout: 10 * time.Second, } if len(os.Args) > 1 { offset, err = strconv.Atoi(os.Args[1]) if err != nil { log.Fatal(err) } } for { log.Printf("retrieving CT logs %d to %d", offset, offset+100) rawEnts, err := ctLog.GetEntries(int64(offset), int64(offset+100)) if err != nil { log.Fatal(err) } // loop over CT records for i, ent := range rawEnts { log.Printf("CT index=%d", offset+i) var cert *x509.Certificate switch ent.Leaf.TimestampedEntry.EntryType { case ct.X509LogEntryType: cert, err = x509.ParseCertificate(ent.Leaf.TimestampedEntry.X509Entry) case ct.PrecertLogEntryType: cert, err = x509.ParseTBSCertificate(ent.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) } if err != nil { log.Fatal(err) } log.Printf("CN=%s", cert.Subject.CommonName) log.Printf("Not Before=%s", cert.NotBefore) log.Printf("Not After=%s", cert.NotAfter) // Format the PEM certificate payload := base64.StdEncoding.EncodeToString(cert.Raw) buf := new(bytes.Buffer) fmt.Fprintf(buf, "-----BEGIN CERTIFICATE-----\n") for len(payload) > 0 { chunkLen := len(payload) if chunkLen > 64 { chunkLen = 64 } fmt.Fprintf(buf, "%s\n", payload[0:chunkLen]) payload = payload[chunkLen:] } fmt.Fprintf(buf, "-----END CERTIFICATE-----") // create a mime/multipart form with the certificate var b bytes.Buffer w := multipart.NewWriter(&b) fw, err := w.CreateFormFile("certificate", certificate.SHA256Hash(cert.Raw)) if err != nil { log.Fatal(err) } _, err = io.Copy(fw, buf) if err != nil { log.Fatal(err) } w.Close() // post the form to the tls-observatory api r, err := http.NewRequest("POST", "https://tls-observatory.services.mozilla.com/api/v1/certificate", &b) if err != nil { log.Fatal(err) } r.Header.Set("Content-Type", w.FormDataContentType()) resp, err := httpCli.Do(r) if err != nil { log.Printf("%v\n\n", err) continue } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Fatal(err) } if resp.StatusCode != http.StatusCreated { log.Fatalf("Expected HTTP 201 Created, got %q\n%s", resp.Status, body) } // parse the returned cert var tlsobs_cert certificate.Certificate err = json.Unmarshal(body, &tlsobs_cert) if err != nil { log.Fatal(err) } log.Printf("https://tls-observatory.services.mozilla.com/api/v1/certificate?id=%d\n\n", tlsobs_cert.ID) } offset += 100 } }
// PostCertificateHandler handles the POST /certificate endpoint of the api. // It receives a single PEM encoded certificate, parses it, inserts it // into the database and returns results in JSON. func PostCertificateHandler(w http.ResponseWriter, r *http.Request) { setResponseHeader(w) log := logger.GetLogger() log.WithFields(logrus.Fields{ "form values": r.Form.Encode(), "headers": r.Header, "uri": r.URL.RequestURI(), }).Debug("PostCertificate Endpoint received request") val := r.Context().Value(dbKey) if val == nil { httpError(w, http.StatusInternalServerError, "Could not find database handler in request context") return } db := val.(*pg.DB) _, certHeader, err := r.FormFile("certificate") if err != nil { httpError(w, http.StatusBadRequest, fmt.Sprintf("Could not read certificate from form data: %v", err)) return } certReader, err := certHeader.Open() if err != nil { httpError(w, http.StatusBadRequest, fmt.Sprintf("Could not read certificate from form data: %v", err)) return } certPEM, err := ioutil.ReadAll(certReader) if err != nil { httpError(w, http.StatusBadRequest, fmt.Sprintf("Could not read certificate from form data: %v", err)) return } block, _ := pem.Decode([]byte(certPEM)) if block == nil { httpError(w, http.StatusBadRequest, "Failed to parse certificate PEM") return } certX509, err := x509.ParseCertificate(block.Bytes) if err != nil { httpError(w, http.StatusBadRequest, fmt.Sprintf("Could not parse X.509 certificate: %v", err)) return } certHash := certificate.SHA256Hash(certX509.Raw) id, err := db.GetCertIDBySHA256Fingerprint(certHash) if err != nil { httpError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to lookup certificate hash in database: %v", err)) return } if id > 0 { // if the cert already exists in DB, return early log.Printf("cert id %d already exists in database, returning it", id) jsonCertFromID(w, r, id) return } var valInfo certificate.ValidationInfo cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo) id, err = db.InsertCertificate(&cert) if err != nil { httpError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to store certificate in database: %v", err)) return } cert.ID = id // If the cert is self-signed (aka. Root CA), we're done here if cert.IsSelfSigned() { jsonCertFromID(w, r, cert.ID) return } // to insert the trust, first build the certificate paths, then insert one trust // entry for each known parent of the cert paths, err := db.GetCertPaths(&cert) if err != nil { httpError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to retrieve chains from database: %v", err)) return } for _, parent := range paths.Parents { cert.ValidationInfo = parent.GetValidityMap() _, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID) if err != nil { httpError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to store trust in database: %v", err)) return } } jsonCertFromID(w, r, cert.ID) return }
//isChainValid creates the valid certificate chains by combining the chain retrieved with the provided truststore. //It return true if it finds at least on validation chain or false if no valid chain of trust can be created. //It also updates the certificate map which gets pushed at the end of each iteration. func isChainValid(endEntity *x509.Certificate, intermediates []*x509.Certificate, truststore *certificate.TrustStore, domain, IP string, certmap map[string]certificate.Certificate) bool { valInfo := &certificate.ValidationInfo{ IsValid: true, } // build a CA verification pool from the list of cacerts interPool := x509.NewCertPool() for _, entity := range intermediates { interPool.AddCert(entity) } // get a list of domains this certificate is supposedly valid for // if the end entity is a CA, use its common name dnsName := domain if endEntity.IsCA { dnsName = endEntity.Subject.CommonName } // configure the verification logic to use the current trustore opts := x509.VerifyOptions{ DNSName: dnsName, Intermediates: interPool, Roots: truststore.Certs, } // Verify attempts to build all the path between the end entity and the // root in the truststore that validate the certificate // If no valid path is found, err is not nil and the certificate is not trusted chains, err := endEntity.Verify(opts) if err == nil { // the end entity is trusted, we need to go through each // chain of trust and store them in database for i, chain := range chains { log.WithFields(logrus.Fields{ "trust chain no": i, "path len": len(chain), }).Debug("domain: " + domain) // loop through each certificate in the chain and for _, cert := range chain { parentSignature := "" parentCert := getFirstParent(cert, chain) if parentCert != nil { parentSignature = certificate.SHA256Hash(parentCert.Raw) } else { log.Println("could not retrieve parent for " + dnsName) } updateCert(cert, parentSignature, domain, IP, truststore.Name, valInfo, certmap) } } return true } // the certificate is not trusted. // we store the cert in DB with its validation error if len(chains) > 0 { log.WithFields(logrus.Fields{ "domain": domain, }).Warning("Got validation error but chains are populated") } valInfo.ValidationError = err.Error() valInfo.IsValid = false parentSignature := "" c := getFirstParent(endEntity, intermediates) if c != nil { parentSignature = certificate.SHA256Hash(c.Raw) } else { log.WithFields(logrus.Fields{ "domain": domain, "servercert": certificate.SHA256Hash(endEntity.Raw), }).Info("Could not get parent") } updateCert(endEntity, parentSignature, domain, IP, truststore.Name, valInfo, certmap) return false }
//handleCertChain takes the chain retrieved from the queue and tries to validate it //against each of the truststores provided. The function returns the ID of the end //entity certificate (or -1 if not stored), the ID of the trust entry (or -1 if not //stored) and an error message. func handleCertChain(chain *certificate.Chain) (int64, int64, error) { var intermediates []*x509.Certificate var endEntity *x509.Certificate endEntity = nil for chaincertno, data := range chain.Certs { //create certificate chain from chain struct certRaw, err := base64.StdEncoding.DecodeString(data) if err != nil { log.WithFields(logrus.Fields{ "domain": chain.Domain, "cert no": chaincertno, "error": err.Error(), }).Warning("Could not decode raw cert from base64") } var cert *x509.Certificate cert, err = x509.ParseCertificate(certRaw) if err != nil { log.WithFields(logrus.Fields{ "domain": chain.Domain, "cert no": chaincertno, "error": err.Error(), }).Warning("Could not parse raw cert") } // if certificate is an authority, // append it to the list of intermediate certs and go to the next one if cert.IsCA { intermediates = append(intermediates, cert) continue } // if we don't yet have an end entity in this chain // set the current cert as the end entity if endEntity == nil { endEntity = cert continue } // here we have a cert that's an end entity when we already // found one in the chain. It's possible that it's an old // V2 cert that's actually an intermediate by doesn't have the // CA flag set log.WithFields(logrus.Fields{ "domain": chain.Domain, "cert no": chaincertno, "cert fingerprint": certificate.SHA256Hash(cert.Raw), }).Warning("Second End Entity cert found in chain received from server. Adding it to intermediates.") if cert.Version < 3 { log.WithFields(logrus.Fields{ "domain": chain.Domain, "cert no": chaincertno, "cert fingerprint": certificate.SHA256Hash(cert.Raw), }).Debug("Probably an old root CA cert") intermediates = append(intermediates, cert) } } if endEntity == nil { log.WithFields(logrus.Fields{ "domain": chain.Domain, }).Warning("the certificate chain did not contain an end entity certificate") } var certmap = make(map[string]certificate.Certificate) // Test the end entity cert with its chain against each of the truststore for _, truststore := range trustStores { if endEntity != nil && isChainValid(endEntity, intermediates, &truststore, chain.Domain, chain.IP, certmap) { // If we have an end entity cert and its chain of trust is valid, our work // here is done, move to the next truststore continue } // to end up here either there was no leaf certificate retrieved // or it was retrieved but it was not valid so we must check the remainder of the chain for i, cert := range intermediates { inter := append(intermediates[:i], intermediates[i+1:]...) isChainValid(cert, inter, &truststore, chain.Domain, chain.IP, certmap) } } log.WithFields(logrus.Fields{ "domain": chain.Domain, "map length": len(certmap), }).Debug("Certificate Map length") return storeCertificates(certmap) }
func Setup(c config.Config) { ts := c.TrustStores for _, tsName := range allowedTruststoreNames { path := "" switch tsName { case certificate.Ubuntu_TS_name: path = ts.UbuntuTS case certificate.Mozilla_TS_name: path = ts.MozillaTS case certificate.Microsoft_TS_name: path = ts.MicrosoftTS case certificate.Apple_TS_name: path = ts.AppleTS case certificate.Android_TS_name: path = ts.AndroidTS default: log.WithFields(logrus.Fields{ "tsname": tsName, }).Warning("Invalid Truststore name.") } log.WithFields(logrus.Fields{ "tsname": tsName, "path": path, }).Debug("Loading Truststore") // load the entire trustore into pooldata, then iterate over each PEM block // until all of pooldata is read poolData, err := ioutil.ReadFile(path) if err != nil { log.WithFields(logrus.Fields{ "tsname": tsName, "error": err.Error(), }).Warning("Failed to load truststore") } certPool := x509.NewCertPool() poollen := 0 // keep a list of cert hashes currently in this truststore // to remove certs no longer in it certHashes := make([]string, 0) for len(poolData) > 0 { // read the next PEM block, ignore non CERTIFICATE entries var block *pem.Block block, poolData = pem.Decode(poolData) if block == nil { break } if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { continue } // parse the current PEM block into a certificate, ignore failures cert, err := x509.ParseCertificate(block.Bytes) if err != nil { log.WithFields(logrus.Fields{ "tsname": tsName, "cert no": poollen + 1, "error": err.Error(), }).Warning("Could not parse PEM block") continue } // if the cert version is 1 or 2, the cert will not contain a CA: True extension // so we set it manually instead. This assumes that all certs found in truststoresfile:///media/Projects/GoProjects/src/github.com/mozilla/TLS-Observer/certificate/analyserPool.go // should be considered valid certificate authorities if cert.Version < 3 { cert.IsCA = true } if !cert.IsCA { log.WithFields(logrus.Fields{ "tsname": tsName, "cert no": poollen + 1, "SHA1": certificate.SHA1Hash(cert.Raw), }).Warning("Certificate in truststore is not a CA cert") } certPool.AddCert(cert) //Push current certificate to DB as trusted v := &certificate.ValidationInfo{} v.IsValid = true certHash := certificate.SHA256Hash(cert.Raw) certHashes = append(certHashes, certHash) parentSignature := "" if cert.Subject.CommonName == cert.Issuer.CommonName { // self-signed, parent sig is self sig parentSignature = certHash } var id int64 = -1 id, err = db.GetCertIDBySHA256Fingerprint(certHash) if err != nil { log.WithFields(logrus.Fields{ "tsname": tsName, "certificate": certificate.SHA256Hash(cert.Raw), "error": err.Error(), }).Error("Could not check if certificate is in db") } if id == -1 { // insert certificate for the first time vinfo := &certificate.ValidationInfo{} vinfo.IsValid = true vinfo.ValidationError = "" st := certificate.CertToStored(cert, parentSignature, "", "", tsName, vinfo) id, err = db.InsertCertificate(&st) if err != nil { log.WithFields(logrus.Fields{ "certificate": certificate.SHA256Hash(cert.Raw), "error": err.Error(), }).Error("Could not insert certificate in db") } } switch tsName { case certificate.Ubuntu_TS_name: err = db.AddCertToUbuntuTruststore(id) case certificate.Mozilla_TS_name: err = db.AddCertToMozillaTruststore(id) case certificate.Microsoft_TS_name: err = db.AddCertToMicrosoftTruststore(id) case certificate.Apple_TS_name: err = db.AddCertToAppleTruststore(id) case certificate.Android_TS_name: err = db.AddCertToAndroidTruststore(id) } if err != nil { log.WithFields(logrus.Fields{ "tsname": tsName, "id": id, "error": err.Error(), }).Error("Could not update certificate trust in db") } poollen++ } // We have a list of certificates in the current truststore and // we use it to disable certs no longer in in err = db.RemoveCACertFromTruststore(certHashes, tsName) if err != nil { log.WithFields(logrus.Fields{ "tsname": tsName, "error": err.Error(), }).Fatal("Failed to update trust of certificates no longer in truststore") } trustStores = append(trustStores, certificate.TrustStore{tsName, certPool}) log.WithFields(logrus.Fields{ "tsname": tsName, "certificates loaded": poollen, }).Info("Successfully loaded TS ") } if len(trustStores) == 0 { log.Error("No truststores loaded, TLS certificate retrieval & analysis won't be available") } }