// Processes the given |entry| in the specified log. func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry), foundPrecert func(*ct.LogEntry)) { atomic.AddInt64(&s.certsProcessed, 1) switch entry.Leaf.TimestampedEntry.EntryType { case ct.X509LogEntryType: if s.opts.PrecertOnly { // Only interested in precerts and this is an X.509 cert, early-out. return } cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry) if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil { // We hit an unparseable entry, already logged inside handleParseEntryError() return } if s.opts.Matcher.CertificateMatches(cert) { entry.X509Cert = cert foundCert(&entry) } case ct.PrecertLogEntryType: c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil { // We hit an unparseable entry, already logged inside handleParseEntryError() return } precert := &ct.Precertificate{ Raw: entry.Chain[0], TBSCertificate: *c, IssuerKeyHash: entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash} if s.opts.Matcher.PrecertificateMatches(precert) { entry.Precert = precert foundPrecert(&entry) } atomic.AddInt64(&s.precertsSeen, 1) } }
func (edb *EntriesDatabase) InsertCTEntry(entry *ct.LogEntry) error { var cert *x509.Certificate var err error switch entry.Leaf.TimestampedEntry.EntryType { case ct.X509LogEntryType: cert, err = x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry) case ct.PrecertLogEntryType: cert, err = x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) } if err != nil { return err } // Skip unimportant entries, if configured if edb.IssuerFilter != nil && !strings.HasPrefix(cert.Issuer.CommonName, *edb.IssuerFilter) { return nil } backoff := &backoff.Backoff{ Jitter: true, } for count := 0; count < 10; count++ { txn, certId, err := edb.insertCertificate(cert) if err != nil { if edb.Verbose { fmt.Printf("Error inserting cert, retrying (%d/10) %s\n", count, err) } if txn != nil { txn.Rollback() } time.Sleep(backoff.Duration()) continue } // // Insert the appropriate CertificateLogEntry, ignoring errors if there was a collision // certLogEntry := &CertificateLogEntry{ CertID: certId, LogID: edb.LogId, EntryID: uint64(entry.Index), EntryTime: Uint64ToTimestamp(entry.Leaf.TimestampedEntry.Timestamp), } err = txn.Insert(certLogEntry) if errorIsNotDuplicate(err) { txn.Rollback() continue } return txn.Commit() } return err }
func main() { var ( err error offset int ) // create a certificate transparency client ctLog := client.New("http://ct.googleapis.com/aviator", nil) httpCli := &http.Client{ Transport: &http.Transport{ DisableCompression: true, DisableKeepAlives: false, }, Timeout: 10 * time.Second, } if len(os.Args) > 1 { offset, err = strconv.Atoi(os.Args[1]) if err != nil { log.Fatal(err) } } for { log.Printf("retrieving CT logs %d to %d", offset, offset+100) rawEnts, err := ctLog.GetEntries(int64(offset), int64(offset+100)) if err != nil { log.Fatal(err) } // loop over CT records for i, ent := range rawEnts { log.Printf("CT index=%d", offset+i) var cert *x509.Certificate switch ent.Leaf.TimestampedEntry.EntryType { case ct.X509LogEntryType: cert, err = x509.ParseCertificate(ent.Leaf.TimestampedEntry.X509Entry) case ct.PrecertLogEntryType: cert, err = x509.ParseTBSCertificate(ent.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) } if err != nil { log.Fatal(err) } log.Printf("CN=%s", cert.Subject.CommonName) log.Printf("Not Before=%s", cert.NotBefore) log.Printf("Not After=%s", cert.NotAfter) // Format the PEM certificate payload := base64.StdEncoding.EncodeToString(cert.Raw) buf := new(bytes.Buffer) fmt.Fprintf(buf, "-----BEGIN CERTIFICATE-----\n") for len(payload) > 0 { chunkLen := len(payload) if chunkLen > 64 { chunkLen = 64 } fmt.Fprintf(buf, "%s\n", payload[0:chunkLen]) payload = payload[chunkLen:] } fmt.Fprintf(buf, "-----END CERTIFICATE-----") // create a mime/multipart form with the certificate var b bytes.Buffer w := multipart.NewWriter(&b) fw, err := w.CreateFormFile("certificate", certificate.SHA256Hash(cert.Raw)) if err != nil { log.Fatal(err) } _, err = io.Copy(fw, buf) if err != nil { log.Fatal(err) } w.Close() // post the form to the tls-observatory api r, err := http.NewRequest("POST", "https://tls-observatory.services.mozilla.com/api/v1/certificate", &b) if err != nil { log.Fatal(err) } r.Header.Set("Content-Type", w.FormDataContentType()) resp, err := httpCli.Do(r) if err != nil { log.Printf("%v\n\n", err) continue } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Fatal(err) } if resp.StatusCode != http.StatusCreated { log.Fatalf("Expected HTTP 201 Created, got %q\n%s", resp.Status, body) } // parse the returned cert var tlsobs_cert certificate.Certificate err = json.Unmarshal(body, &tlsobs_cert) if err != nil { log.Fatal(err) } log.Printf("https://tls-observatory.services.mozilla.com/api/v1/certificate?id=%d\n\n", tlsobs_cert.ID) } offset += 100 } }