// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry)) {
atomic.AddInt64(&s.certsProcessed, 1)
switch entry.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
if s.opts.PrecertOnly {
// Only interested in precerts and this is an X.509 cert, early-out.
return
}
cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
// We hit an unparseable entry, already logged inside handleParseEntryError()
return
}
if s.opts.Matcher.CertificateMatches(cert) {
entry.X509Cert = cert
foundCert(&entry)
}
case ct.PrecertLogEntryType:
c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
// We hit an unparseable entry, already logged inside handleParseEntryError()
return
}
precert := &ct.Precertificate{
Raw: entry.Chain[0],
TBSCertificate: *c,
IssuerKeyHash: entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
if s.opts.Matcher.PrecertificateMatches(precert) {
entry.Precert = precert
foundPrecert(&entry)
}
atomic.AddInt64(&s.precertsSeen, 1)
}
}
func (edb *EntriesDatabase) InsertCTEntry(entry *ct.LogEntry) error {
var cert *x509.Certificate
var err error
switch entry.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
cert, err = x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
cert, err = x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
}
if err != nil {
return err
}
// Skip unimportant entries, if configured
if edb.IssuerFilter != nil && !strings.HasPrefix(cert.Issuer.CommonName, *edb.IssuerFilter) {
return nil
}
backoff := &backoff.Backoff{
Jitter: true,
}
for count := 0; count < 10; count++ {
txn, certId, err := edb.insertCertificate(cert)
if err != nil {
if edb.Verbose {
fmt.Printf("Error inserting cert, retrying (%d/10) %s\n", count, err)
}
if txn != nil {
txn.Rollback()
}
time.Sleep(backoff.Duration())
continue
}
//
// Insert the appropriate CertificateLogEntry, ignoring errors if there was a collision
//
certLogEntry := &CertificateLogEntry{
CertID: certId,
LogID: edb.LogId,
EntryID: uint64(entry.Index),
EntryTime: Uint64ToTimestamp(entry.Leaf.TimestampedEntry.Timestamp),
}
err = txn.Insert(certLogEntry)
if errorIsNotDuplicate(err) {
txn.Rollback()
continue
}
return txn.Commit()
}
return err
}
func main() {
var (
err error
offset int
)
// create a certificate transparency client
ctLog := client.New("http://ct.googleapis.com/aviator", nil)
httpCli := &http.Client{
Transport: &http.Transport{
DisableCompression: true,
DisableKeepAlives: false,
},
Timeout: 10 * time.Second,
}
if len(os.Args) > 1 {
offset, err = strconv.Atoi(os.Args[1])
if err != nil {
log.Fatal(err)
}
}
for {
log.Printf("retrieving CT logs %d to %d", offset, offset+100)
rawEnts, err := ctLog.GetEntries(int64(offset), int64(offset+100))
if err != nil {
log.Fatal(err)
}
// loop over CT records
for i, ent := range rawEnts {
log.Printf("CT index=%d", offset+i)
var cert *x509.Certificate
switch ent.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
cert, err = x509.ParseCertificate(ent.Leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
cert, err = x509.ParseTBSCertificate(ent.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
}
if err != nil {
log.Fatal(err)
}
log.Printf("CN=%s", cert.Subject.CommonName)
log.Printf("Not Before=%s", cert.NotBefore)
log.Printf("Not After=%s", cert.NotAfter)
// Format the PEM certificate
payload := base64.StdEncoding.EncodeToString(cert.Raw)
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "-----BEGIN CERTIFICATE-----\n")
for len(payload) > 0 {
chunkLen := len(payload)
if chunkLen > 64 {
chunkLen = 64
}
fmt.Fprintf(buf, "%s\n", payload[0:chunkLen])
payload = payload[chunkLen:]
}
fmt.Fprintf(buf, "-----END CERTIFICATE-----")
// create a mime/multipart form with the certificate
var b bytes.Buffer
w := multipart.NewWriter(&b)
fw, err := w.CreateFormFile("certificate", certificate.SHA256Hash(cert.Raw))
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(fw, buf)
if err != nil {
log.Fatal(err)
}
w.Close()
// post the form to the tls-observatory api
r, err := http.NewRequest("POST", "https://tls-observatory.services.mozilla.com/api/v1/certificate", &b)
if err != nil {
log.Fatal(err)
}
r.Header.Set("Content-Type", w.FormDataContentType())
resp, err := httpCli.Do(r)
if err != nil {
log.Printf("%v\n\n", err)
continue
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
if resp.StatusCode != http.StatusCreated {
log.Fatalf("Expected HTTP 201 Created, got %q\n%s", resp.Status, body)
}
// parse the returned cert
var tlsobs_cert certificate.Certificate
err = json.Unmarshal(body, &tlsobs_cert)
if err != nil {
log.Fatal(err)
}
log.Printf("https://tls-observatory.services.mozilla.com/api/v1/certificate?id=%d\n\n", tlsobs_cert.ID)
}
offset += 100
}
}