示例#1
0
//updateCert takes the input certificate and updates the map holding all the certificates to be pushed.
//If the certificates has already been inserted it updates the existing record else it creates it.
func updateCert(cert *x509.Certificate, parentSignature string, domain, ip, TSName string, valInfo *certificate.ValidationInfo, certmap map[string]certificate.Certificate) {
	id := certificate.SHA256Hash(cert.Raw)

	if storedCert, ok := certmap[id]; !ok {

		certmap[id] = certificate.CertToStored(cert, parentSignature, domain, ip, TSName, valInfo)

	} else {

		parentFound := false

		for _, p := range storedCert.ParentSignature {

			if parentSignature == p {
				parentFound = true
				break
			}
		}

		if !parentFound {
			storedCert.ParentSignature = append(storedCert.ParentSignature, parentSignature)
		}

		if !storedCert.CA {

			if storedCert.ScanTarget != domain {

				log.WithFields(logrus.Fields{
					"domain":       storedCert.ScanTarget,
					"domain_input": domain,
					"certificate":  storedCert.Hashes.SHA256,
				}).Warning("Different domain input")
			}

			//add IP ( single domain may be served by multiple IPs )
			ipFound := false

			for _, i := range storedCert.IPs {
				if ip == i {
					ipFound = true
					break
				}
			}

			if !ipFound {
				storedCert.IPs = append(storedCert.IPs, ip)
			}
		}

		storedCert.ValidationInfo[TSName] = *valInfo

		certmap[id] = storedCert
	}

}
示例#2
0
func main() {
	var (
		err    error
		offset int
	)
	// create a certificate transparency client
	ctLog := client.New("http://ct.googleapis.com/aviator", nil)

	httpCli := &http.Client{
		Transport: &http.Transport{
			DisableCompression: true,
			DisableKeepAlives:  false,
		},
		Timeout: 10 * time.Second,
	}

	if len(os.Args) > 1 {
		offset, err = strconv.Atoi(os.Args[1])
		if err != nil {
			log.Fatal(err)
		}
	}
	for {
		log.Printf("retrieving CT logs %d to %d", offset, offset+100)
		rawEnts, err := ctLog.GetEntries(int64(offset), int64(offset+100))
		if err != nil {
			log.Fatal(err)
		}

		// loop over CT records
		for i, ent := range rawEnts {
			log.Printf("CT index=%d", offset+i)
			var cert *x509.Certificate
			switch ent.Leaf.TimestampedEntry.EntryType {
			case ct.X509LogEntryType:
				cert, err = x509.ParseCertificate(ent.Leaf.TimestampedEntry.X509Entry)
			case ct.PrecertLogEntryType:
				cert, err = x509.ParseTBSCertificate(ent.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
			}
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("CN=%s", cert.Subject.CommonName)
			log.Printf("Not Before=%s", cert.NotBefore)
			log.Printf("Not After=%s", cert.NotAfter)

			// Format the PEM certificate
			payload := base64.StdEncoding.EncodeToString(cert.Raw)
			buf := new(bytes.Buffer)
			fmt.Fprintf(buf, "-----BEGIN CERTIFICATE-----\n")
			for len(payload) > 0 {
				chunkLen := len(payload)
				if chunkLen > 64 {
					chunkLen = 64
				}
				fmt.Fprintf(buf, "%s\n", payload[0:chunkLen])
				payload = payload[chunkLen:]
			}
			fmt.Fprintf(buf, "-----END CERTIFICATE-----")

			// create a mime/multipart form with the certificate
			var b bytes.Buffer
			w := multipart.NewWriter(&b)
			fw, err := w.CreateFormFile("certificate", certificate.SHA256Hash(cert.Raw))
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(fw, buf)
			if err != nil {
				log.Fatal(err)
			}
			w.Close()

			// post the form to the tls-observatory api
			r, err := http.NewRequest("POST", "https://tls-observatory.services.mozilla.com/api/v1/certificate", &b)
			if err != nil {
				log.Fatal(err)
			}
			r.Header.Set("Content-Type", w.FormDataContentType())
			resp, err := httpCli.Do(r)
			if err != nil {
				log.Printf("%v\n\n", err)
				continue
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				log.Fatal(err)
			}
			if resp.StatusCode != http.StatusCreated {
				log.Fatalf("Expected HTTP 201 Created, got %q\n%s", resp.Status, body)
			}

			// parse the returned cert
			var tlsobs_cert certificate.Certificate
			err = json.Unmarshal(body, &tlsobs_cert)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("https://tls-observatory.services.mozilla.com/api/v1/certificate?id=%d\n\n", tlsobs_cert.ID)
		}
		offset += 100
	}
}
示例#3
0
// PostCertificateHandler handles the POST /certificate endpoint of the api.
// It receives a single PEM encoded certificate, parses it, inserts it
// into the database and returns results in JSON.
func PostCertificateHandler(w http.ResponseWriter, r *http.Request) {
	setResponseHeader(w)
	log := logger.GetLogger()
	log.WithFields(logrus.Fields{
		"form values": r.Form.Encode(),
		"headers":     r.Header,
		"uri":         r.URL.RequestURI(),
	}).Debug("PostCertificate Endpoint received request")

	val := r.Context().Value(dbKey)
	if val == nil {
		httpError(w, http.StatusInternalServerError, "Could not find database handler in request context")
		return
	}
	db := val.(*pg.DB)

	_, certHeader, err := r.FormFile("certificate")
	if err != nil {
		httpError(w, http.StatusBadRequest,
			fmt.Sprintf("Could not read certificate from form data: %v", err))
		return
	}

	certReader, err := certHeader.Open()
	if err != nil {
		httpError(w, http.StatusBadRequest,
			fmt.Sprintf("Could not read certificate from form data: %v", err))
		return
	}

	certPEM, err := ioutil.ReadAll(certReader)
	if err != nil {
		httpError(w, http.StatusBadRequest,
			fmt.Sprintf("Could not read certificate from form data: %v", err))
		return
	}

	block, _ := pem.Decode([]byte(certPEM))
	if block == nil {
		httpError(w, http.StatusBadRequest,
			"Failed to parse certificate PEM")
		return
	}

	certX509, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		httpError(w, http.StatusBadRequest,
			fmt.Sprintf("Could not parse X.509 certificate: %v", err))
		return
	}

	certHash := certificate.SHA256Hash(certX509.Raw)
	id, err := db.GetCertIDBySHA256Fingerprint(certHash)
	if err != nil {
		httpError(w, http.StatusInternalServerError,
			fmt.Sprintf("Failed to lookup certificate hash in database: %v", err))
		return
	}
	if id > 0 {
		// if the cert already exists in DB, return early
		log.Printf("cert id %d already exists in database, returning it", id)
		jsonCertFromID(w, r, id)
		return
	}

	var valInfo certificate.ValidationInfo
	cert := certificate.CertToStored(certX509, certHash, "", "", "", &valInfo)
	id, err = db.InsertCertificate(&cert)
	if err != nil {
		httpError(w, http.StatusInternalServerError,
			fmt.Sprintf("Failed to store certificate in database: %v", err))
		return
	}
	cert.ID = id
	// If the cert is self-signed (aka. Root CA), we're done here
	if cert.IsSelfSigned() {
		jsonCertFromID(w, r, cert.ID)
		return
	}

	// to insert the trust, first build the certificate paths, then insert one trust
	// entry for each known parent of the cert
	paths, err := db.GetCertPaths(&cert)
	if err != nil {
		httpError(w, http.StatusInternalServerError,
			fmt.Sprintf("Failed to retrieve chains from database: %v", err))
		return
	}
	for _, parent := range paths.Parents {
		cert.ValidationInfo = parent.GetValidityMap()
		_, err := db.InsertTrustToDB(cert, cert.ID, parent.Cert.ID)
		if err != nil {
			httpError(w, http.StatusInternalServerError,
				fmt.Sprintf("Failed to store trust in database: %v", err))
			return
		}
	}

	jsonCertFromID(w, r, cert.ID)
	return
}
示例#4
0
//isChainValid creates the valid certificate chains by combining the chain retrieved with the provided truststore.
//It return true if it finds at least on validation chain or false if no valid chain of trust can be created.
//It also updates the certificate map which gets pushed at the end of each iteration.
func isChainValid(endEntity *x509.Certificate, intermediates []*x509.Certificate, truststore *certificate.TrustStore, domain, IP string, certmap map[string]certificate.Certificate) bool {
	valInfo := &certificate.ValidationInfo{
		IsValid: true,
	}

	// build a CA verification pool from the list of cacerts
	interPool := x509.NewCertPool()
	for _, entity := range intermediates {
		interPool.AddCert(entity)
	}

	// get a list of domains this certificate is supposedly valid for
	// if the end entity is a CA, use its common name
	dnsName := domain
	if endEntity.IsCA {
		dnsName = endEntity.Subject.CommonName
	}

	// configure the verification logic to use the current trustore
	opts := x509.VerifyOptions{
		DNSName:       dnsName,
		Intermediates: interPool,
		Roots:         truststore.Certs,
	}

	// Verify attempts to build all the path between the end entity and the
	// root in the truststore that validate the certificate
	// If no valid path is found, err is not nil and the certificate is not trusted
	chains, err := endEntity.Verify(opts)

	if err == nil {
		// the end entity is trusted, we need to go through each
		// chain of trust and store them in database
		for i, chain := range chains {
			log.WithFields(logrus.Fields{
				"trust chain no": i,
				"path len":       len(chain),
			}).Debug("domain: " + domain)
			// loop through each certificate in the chain and
			for _, cert := range chain {
				parentSignature := ""
				parentCert := getFirstParent(cert, chain)
				if parentCert != nil {
					parentSignature = certificate.SHA256Hash(parentCert.Raw)
				} else {
					log.Println("could not retrieve parent for " + dnsName)
				}
				updateCert(cert, parentSignature, domain, IP, truststore.Name, valInfo, certmap)
			}
		}
		return true
	}

	// the certificate is not trusted.
	// we store the cert in DB with its validation error
	if len(chains) > 0 {
		log.WithFields(logrus.Fields{
			"domain": domain,
		}).Warning("Got validation error but chains are populated")
	}

	valInfo.ValidationError = err.Error()
	valInfo.IsValid = false

	parentSignature := ""
	c := getFirstParent(endEntity, intermediates)

	if c != nil {
		parentSignature = certificate.SHA256Hash(c.Raw)
	} else {
		log.WithFields(logrus.Fields{
			"domain":     domain,
			"servercert": certificate.SHA256Hash(endEntity.Raw),
		}).Info("Could not get parent")
	}

	updateCert(endEntity, parentSignature, domain, IP, truststore.Name, valInfo, certmap)

	return false

}
示例#5
0
//handleCertChain takes the chain retrieved from the queue and tries to validate it
//against each of the truststores provided. The function returns the ID of the end
//entity certificate (or -1 if not stored), the ID of the trust entry (or -1 if not
//stored) and an error message.
func handleCertChain(chain *certificate.Chain) (int64, int64, error) {

	var intermediates []*x509.Certificate
	var endEntity *x509.Certificate
	endEntity = nil

	for chaincertno, data := range chain.Certs { //create certificate chain from chain struct

		certRaw, err := base64.StdEncoding.DecodeString(data)
		if err != nil {

			log.WithFields(logrus.Fields{
				"domain":  chain.Domain,
				"cert no": chaincertno,
				"error":   err.Error(),
			}).Warning("Could not decode raw cert from base64")

		}

		var cert *x509.Certificate
		cert, err = x509.ParseCertificate(certRaw)
		if err != nil {
			log.WithFields(logrus.Fields{
				"domain":  chain.Domain,
				"cert no": chaincertno,
				"error":   err.Error(),
			}).Warning("Could not parse raw cert")
		}

		// if certificate is an authority,
		// append it to the list of intermediate certs and go to the next one
		if cert.IsCA {
			intermediates = append(intermediates, cert)
			continue
		}
		// if we don't yet have an end entity in this chain
		// set the current cert as the end entity
		if endEntity == nil {
			endEntity = cert
			continue
		}
		// here we have a cert that's an end entity when we already
		// found one in the chain. It's possible that it's an old
		// V2 cert that's actually an intermediate by doesn't have the
		// CA flag set
		log.WithFields(logrus.Fields{
			"domain":           chain.Domain,
			"cert no":          chaincertno,
			"cert fingerprint": certificate.SHA256Hash(cert.Raw),
		}).Warning("Second End Entity cert found in chain received from server. Adding it to intermediates.")
		if cert.Version < 3 {
			log.WithFields(logrus.Fields{
				"domain":           chain.Domain,
				"cert no":          chaincertno,
				"cert fingerprint": certificate.SHA256Hash(cert.Raw),
			}).Debug("Probably an old root CA cert")
			intermediates = append(intermediates, cert)
		}
	}

	if endEntity == nil {
		log.WithFields(logrus.Fields{
			"domain": chain.Domain,
		}).Warning("the certificate chain did not contain an end entity certificate")
	}

	var certmap = make(map[string]certificate.Certificate)

	// Test the end entity cert with its chain against each of the truststore
	for _, truststore := range trustStores {
		if endEntity != nil && isChainValid(endEntity, intermediates, &truststore, chain.Domain, chain.IP, certmap) {
			// If we have an end entity cert and its chain of trust is valid, our work
			// here is done, move to the next truststore
			continue
		}

		// to end up here either there was no leaf certificate retrieved
		// or it was retrieved but it was not valid so we must check the remainder of the chain
		for i, cert := range intermediates {
			inter := append(intermediates[:i], intermediates[i+1:]...)
			isChainValid(cert, inter, &truststore, chain.Domain, chain.IP, certmap)
		}
	}

	log.WithFields(logrus.Fields{
		"domain":     chain.Domain,
		"map length": len(certmap),
	}).Debug("Certificate Map length")

	return storeCertificates(certmap)
}
示例#6
0
func Setup(c config.Config) {
	ts := c.TrustStores

	for _, tsName := range allowedTruststoreNames {

		path := ""

		switch tsName {
		case certificate.Ubuntu_TS_name:
			path = ts.UbuntuTS
		case certificate.Mozilla_TS_name:
			path = ts.MozillaTS
		case certificate.Microsoft_TS_name:
			path = ts.MicrosoftTS
		case certificate.Apple_TS_name:
			path = ts.AppleTS
		case certificate.Android_TS_name:
			path = ts.AndroidTS
		default:
			log.WithFields(logrus.Fields{
				"tsname": tsName,
			}).Warning("Invalid Truststore name.")
		}

		log.WithFields(logrus.Fields{
			"tsname": tsName,
			"path":   path,
		}).Debug("Loading Truststore")

		// load the entire trustore into pooldata, then iterate over each PEM block
		// until all of pooldata is read
		poolData, err := ioutil.ReadFile(path)
		if err != nil {
			log.WithFields(logrus.Fields{
				"tsname": tsName,
				"error":  err.Error(),
			}).Warning("Failed to load truststore")
		}
		certPool := x509.NewCertPool()
		poollen := 0

		// keep a list of cert hashes currently in this truststore
		// to remove certs no longer in it
		certHashes := make([]string, 0)

		for len(poolData) > 0 {
			// read the next PEM block, ignore non CERTIFICATE entries
			var block *pem.Block
			block, poolData = pem.Decode(poolData)
			if block == nil {
				break
			}
			if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
				continue
			}
			// parse the current PEM block into a certificate, ignore failures
			cert, err := x509.ParseCertificate(block.Bytes)
			if err != nil {
				log.WithFields(logrus.Fields{
					"tsname":  tsName,
					"cert no": poollen + 1,
					"error":   err.Error(),
				}).Warning("Could not parse PEM block")
				continue
			}

			// if the cert version is 1 or 2, the cert will not contain a CA: True extension
			// so we set it manually instead. This assumes that all certs found in truststoresfile:///media/Projects/GoProjects/src/github.com/mozilla/TLS-Observer/certificate/analyserPool.go
			// should be considered valid certificate authorities
			if cert.Version < 3 {
				cert.IsCA = true
			}

			if !cert.IsCA {
				log.WithFields(logrus.Fields{
					"tsname":  tsName,
					"cert no": poollen + 1,
					"SHA1":    certificate.SHA1Hash(cert.Raw),
				}).Warning("Certificate in truststore is not a CA cert")
			}

			certPool.AddCert(cert)

			//Push current certificate to DB as trusted
			v := &certificate.ValidationInfo{}
			v.IsValid = true
			certHash := certificate.SHA256Hash(cert.Raw)
			certHashes = append(certHashes, certHash)
			parentSignature := ""
			if cert.Subject.CommonName == cert.Issuer.CommonName {
				// self-signed, parent sig is self sig
				parentSignature = certHash
			}
			var id int64 = -1
			id, err = db.GetCertIDBySHA256Fingerprint(certHash)
			if err != nil {
				log.WithFields(logrus.Fields{
					"tsname":      tsName,
					"certificate": certificate.SHA256Hash(cert.Raw),
					"error":       err.Error(),
				}).Error("Could not check if certificate is in db")
			}

			if id == -1 {
				// insert certificate for the first time
				vinfo := &certificate.ValidationInfo{}
				vinfo.IsValid = true
				vinfo.ValidationError = ""

				st := certificate.CertToStored(cert, parentSignature, "", "", tsName, vinfo)
				id, err = db.InsertCertificate(&st)
				if err != nil {
					log.WithFields(logrus.Fields{
						"certificate": certificate.SHA256Hash(cert.Raw),
						"error":       err.Error(),
					}).Error("Could not insert certificate in db")
				}
			}
			switch tsName {
			case certificate.Ubuntu_TS_name:
				err = db.AddCertToUbuntuTruststore(id)
			case certificate.Mozilla_TS_name:
				err = db.AddCertToMozillaTruststore(id)
			case certificate.Microsoft_TS_name:
				err = db.AddCertToMicrosoftTruststore(id)
			case certificate.Apple_TS_name:
				err = db.AddCertToAppleTruststore(id)
			case certificate.Android_TS_name:
				err = db.AddCertToAndroidTruststore(id)
			}
			if err != nil {
				log.WithFields(logrus.Fields{
					"tsname": tsName,
					"id":     id,
					"error":  err.Error(),
				}).Error("Could not update certificate trust in db")
			}
			poollen++
		}
		// We have a list of certificates in the current truststore and
		// we use it to disable certs no longer in in
		err = db.RemoveCACertFromTruststore(certHashes, tsName)
		if err != nil {
			log.WithFields(logrus.Fields{
				"tsname": tsName,
				"error":  err.Error(),
			}).Fatal("Failed to update trust of certificates no longer in truststore")
		}
		trustStores = append(trustStores, certificate.TrustStore{tsName, certPool})
		log.WithFields(logrus.Fields{
			"tsname":              tsName,
			"certificates loaded": poollen,
		}).Info("Successfully loaded TS ")
	}

	if len(trustStores) == 0 {
		log.Error("No truststores loaded, TLS certificate retrieval & analysis won't be available")
	}
}