func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // batch side: do 100 certs at a time limit := 100 batch := 0 for { fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit) rows, err := db.Query(`SELECT id, raw_cert FROM certificates WHERE sha256_subject_spki IS NULL ORDER BY id ASC LIMIT $1`, limit) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var raw string var id int64 err = rows.Scan(&id, &raw) if err != nil { fmt.Println("error while parsing cert", id, ":", err) continue } certdata, err := base64.StdEncoding.DecodeString(raw) if err != nil { fmt.Println("error decoding base64 of cert", id, ":", err) continue } c, err := x509.ParseCertificate(certdata) if err != nil { fmt.Println("error while x509 parsing cert", id, ":", err) continue } _, err = db.Exec(`UPDATE certificates SET sha256_subject_spki=$1 WHERE id=$2`, certificate.SHA256SubjectSPKI(c), id) if err != nil { fmt.Println("error while updating cert", id, "in database:", err) } } if i == 0 { fmt.Println("done!") break } //offset += limit batch++ } }
func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // get all the sha256 fingerprints of dup certificates rows, err := db.Query(`select sha256_fingerprint from certificates group by sha256_fingerprint having count(sha256_fingerprint) > 1`) if rows != nil { defer rows.Close() } if err != nil { log.Fatal(err) } for rows.Next() { var fp string err = rows.Scan(&fp) if err != nil { log.Fatal(err) } // get all the ids of certificates with this fingerprint rows, err := db.Query(`select id from certificates where sha256_fingerprint = $1`, fp) if rows != nil { defer rows.Close() } if err != nil { log.Fatal(err) } var ids []uint64 var smallestid uint64 = 18446744073709551615 for rows.Next() { var id uint64 err = rows.Scan(&id) if err != nil { log.Fatal(err) } if id < smallestid { smallestid = id } ids = append(ids, id) } log.Printf("Found %d certificates with fingerprint %s, smallest id is %d", len(ids), fp, smallestid) for _, id := range ids { if id == smallestid { continue } log.Println("reattaching all trust from", id, "to", smallestid) // reattach all trust to the smallest id instead of the current id _, err = db.Exec(`update trust set cert_id = $1 where cert_id = $2`, smallestid, id) if err != nil { log.Fatal(err) } _, err = db.Exec(`update trust set issuer_id = $1 where issuer_id = $2`, smallestid, id) if err != nil { log.Fatal(err) } log.Println("reattaching all scans from", id, "to", smallestid) // reattach all trust to the smallest id instead of the current id _, err = db.Exec(`update scans set cert_id = $1 where cert_id = $2`, smallestid, id) if err != nil { log.Fatal(err) } log.Println("deleting certificate", id) // remove the duplicate certificate _, err = db.Exec(`delete from certificates where id = $1`, id) if err != nil { log.Fatal(err) } } } }
func main() { var observatory = flag.String("observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory") flag.Parse() db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // batch side: do 100 certs at a time limit := 100 batch := 0 var donedomains []string for { fmt.Printf("\nProcessing batch %d to %d\n", batch*limit, batch*limit+limit) rows, err := db.Query(` SELECT domains FROM certificates INNER JOIN trust ON (trust.cert_id=certificates.id) WHERE is_ca='false' AND trusted_mozilla='true' ORDER BY certificates.id ASC LIMIT $1 OFFSET $2`, limit, batch*limit) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var domains string err = rows.Scan(&domains) if err != nil { fmt.Println("error while retrieving domains:", err) continue } for _, domain := range strings.Split(domains, ",") { domain = strings.TrimSpace(domain) if domain == "" { continue } if domain[0] == '*' { domain = "www" + domain[1:] } if contains(donedomains, domain) { continue } resp, err := http.Post(*observatory+"/api/v1/scan?target="+domain, "application/json", nil) if err != nil { panic(err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { panic(err) } var scan scan err = json.Unmarshal(body, &scan) if err != nil { panic(err) } fmt.Printf("Started scan %d on %s - %s/api/v1/results?id=%d\n", scan.ID, domain, *observatory, scan.ID) donedomains = append(donedomains, domain) time.Sleep(500 * time.Millisecond) } } if i == 0 { fmt.Println("done!") break } batch++ } }
func main() { log := logger.GetLogger() router := NewRouter() var cfgFile string var debug bool flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/api.cfg", "Input file csv format") flag.BoolVar(&debug, "debug", false, "Set debug logging") flag.Parse() if debug { logger.SetLevelToDebug() } conf, err := config.Load(cfgFile) if err != nil { log.Fatal("Failed to load configuration: %v", err) } if !conf.General.Enable && os.Getenv("TLSOBS_API_ENABLE") != "on" { log.Fatal("API is disabled in configuration") } dbtls := "disable" if conf.General.PostgresUseTLS { dbtls = "verify-full" } db, err := pg.RegisterConnection( conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, dbtls) defer db.Close() if err != nil { log.Fatal(err) } db.SetMaxOpenConns(runtime.NumCPU() * 27) db.SetMaxIdleConns(2) // simple DB watchdog, crashes the process if connection dies go func() { for { var one uint err = db.QueryRow("SELECT 1").Scan(&one) if err != nil { log.Fatal("Database connection failed:", err) } if one != 1 { log.Fatal("Apparently the database doesn't know the meaning of one anymore. Crashing.") } time.Sleep(10 * time.Second) } }() scanRefreshRate = float64(conf.General.ScanRefreshRate) // wait for clients err = http.ListenAndServe(":8083", Adapt(router, AddDB(db))) log.Fatal(err) }
func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // batch side: do 100 certs at a time limit := 100 batch := 0 for { fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit) rows, err := db.Query(`SELECT id, raw_cert FROM certificates WHERE key_alg IS NULL ORDER BY id ASC LIMIT $1`, limit) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var raw string var id int64 err = rows.Scan(&id, &raw) if err != nil { fmt.Println("error while parsing cert", id, ":", err) continue } certdata, err := base64.StdEncoding.DecodeString(raw) if err != nil { fmt.Println("error decoding base64 of cert", id, ":", err) continue } c, err := x509.ParseCertificate(certdata) if err != nil { fmt.Println("error while x509 parsing cert", id, ":", err) continue } key, err := getPublicKeyInfo(c) if err != nil { fmt.Println("error while parsing public key info for cert", id, ":", err) continue } keydata, err := json.Marshal(key) if err != nil { fmt.Println("error while marshalling key info of cert", id, " : ", err) continue } pkp_sha256 := certificate.PKPSHA256Hash(c) _, err = db.Exec(`UPDATE certificates SET key=$1, key_alg=$2, pkp_sha256=$3 WHERE id=$4`, keydata, key.Alg, pkp_sha256, id) if err != nil { fmt.Println("error while updating cert", id, "in database:", err) } } if i == 0 { fmt.Println("done!") break } //offset += limit batch++ } }
func main() { var ( cfgFile, cipherscan string debug bool ) flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Configuration file") flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location") flag.BoolVar(&debug, "debug", false, "Set debug logging") flag.Parse() if debug { logger.SetLevelToDebug() } conf, err := config.Load(cfgFile) if err != nil { log.Fatal(fmt.Sprintf("Failed to load configuration: %v", err)) } if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" { log.Fatal("Scanner is disabled in configuration") } _, err = os.Stat(cipherscan) if err != nil { log.WithFields(logrus.Fields{ "error": err.Error(), }).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.") } // increase the n runtime.GOMAXPROCS(conf.General.MaxProc) dbtls := "disable" if conf.General.PostgresUseTLS { dbtls = "verify-full" } db, err = pg.RegisterConnection( conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, dbtls) defer db.Close() if err != nil { log.WithFields(logrus.Fields{ "error": err.Error(), }).Fatal("Failed to connect to database") } db.SetMaxOpenConns(conf.General.MaxProc) db.SetMaxIdleConns(10) // simple DB watchdog, crashes the process if connection dies go func() { for { var one uint err = db.QueryRow("SELECT 1").Scan(&one) if err != nil { log.Fatal("Database connection failed:", err) } if one != 1 { log.Fatal("Apparently the database doesn't know the meaning of one anymore. Crashing.") } time.Sleep(10 * time.Second) } }() incomingScans := db.RegisterScanListener( conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable") Setup(conf) activeScanners := 0 for scanID := range incomingScans { // wait until we have an available scanner for { if activeScanners >= conf.General.MaxProc { time.Sleep(time.Second) } else { break } } go func() { activeScanners++ scan(scanID, cipherscan) activeScanners-- }() } }
func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // batch side: do 100 certs at a time limit := 100 batch := 0 for { fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit) rows, err := db.Query(`SELECT id, raw_cert FROM certificates WHERE issuer IS NULL AND subject IS NULL AND domains IS NULL ORDER BY id ASC LIMIT $1`, limit) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var raw string var id int64 err = rows.Scan(&id, &raw) if err != nil { fmt.Println("error while parsing cert", id, ":", err) continue } certdata, err := base64.StdEncoding.DecodeString(raw) if err != nil { fmt.Println("error decoding base64 of cert", id, ":", err) continue } c, err := x509.ParseCertificate(certdata) if err != nil { fmt.Println("error while x509 parsing cert", id, ":", err) continue } issuer := certificate.Issuer{Country: c.Issuer.Country, CommonName: c.Issuer.CommonName, OrgUnit: c.Issuer.OrganizationalUnit, Organisation: c.Issuer.Organization} issuerjs, err := json.Marshal(issuer) if err != nil { fmt.Println("error while marshalling issuer of cert", id, " : ", err) continue } subject := certificate.Subject{Country: c.Subject.Country, CommonName: c.Subject.CommonName, OrgUnit: c.Subject.OrganizationalUnit, Organisation: c.Subject.Organization} subjectjs, err := json.Marshal(subject) if err != nil { fmt.Println("error while marshalling subject of cert", id, " : ", err) continue } domainstr := "" if !c.IsCA { domainfound := false for _, d := range c.DNSNames { if d == c.Subject.CommonName { domainfound = true } } var domains []string if !domainfound { domains = append(c.DNSNames, c.Subject.CommonName) } else { domains = c.DNSNames } domainstr = strings.Join(domains, ",") fmt.Printf("%d,", id) } _, err = db.Exec(`UPDATE certificates SET issuer=$1, subject=$2, domains=$3 WHERE id=$4`, issuerjs, subjectjs, domainstr, id) if err != nil { fmt.Println("error while updating cert", id, "in database:", err) } } if i == 0 { fmt.Println("done!") break } //offset += limit batch++ } }
func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } batch := 0 for { fmt.Printf("\nProcessing batch %d to %d: ", batch, batch+100) rows, err := db.Query(`SELECT id, raw_cert FROM certificates WHERE x509_certificatepolicies IS NULL OR x509_certificatepolicies='null' OR permitted_names IS NULL OR permitted_names='null' ORDER BY id ASC LIMIT 100`) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var raw string var id int64 err = rows.Scan(&id, &raw) if err != nil { fmt.Println("error while parsing cert", id, ":", err) continue } certdata, err := base64.StdEncoding.DecodeString(raw) if err != nil { fmt.Println("error decoding base64 of cert", id, ":", err) continue } c, err := x509.ParseCertificate(certdata) if err != nil { fmt.Println("error while x509 parsing cert", id, ":", err) continue } var valInfo certificate.ValidationInfo cert := certificate.CertToStored(c, "", "", "", "", &valInfo) policies, err := json.Marshal(cert.X509v3Extensions.PolicyIdentifiers) if err != nil { log.Printf("error while marshalling policies for cert %d: %v", id, err) continue } permittednames, err := json.Marshal(cert.X509v3Extensions.PermittedNames) if err != nil { log.Printf("error while marshalling permitted names for cert %d: %v", id, err) continue } log.Printf("id=%d, subject=%s, policies=%s, permittednames=%s", id, cert.Subject.String(), policies, permittednames) _, err = db.Exec(`UPDATE certificates SET x509_certificatepolicies=$1, permitted_names=$2, is_name_constrained=$3 WHERE id=$4`, policies, permittednames, cert.X509v3Extensions.IsNameConstrained, id) if err != nil { fmt.Println("error while updating cert", id, "in database:", err) } fmt.Printf(".") } if i == 0 { fmt.Println("done!") break } //offset += limit batch += 100 } }
func main() { db, err := database.RegisterConnection( os.Getenv("TLSOBS_POSTGRESDB"), os.Getenv("TLSOBS_POSTGRESUSER"), os.Getenv("TLSOBS_POSTGRESPASS"), os.Getenv("TLSOBS_POSTGRES"), "require") defer db.Close() if err != nil { panic(err) } // batch side: do 100 certs at a time limit := 100 offset := 0 for { fmt.Println("Processing batch", offset, "to", offset+limit) rows, err := db.Query(`SELECT id, raw_cert FROM certificates WHERE not_valid_after < NOW() AND not_valid_after > NOW() - INTERVAL '4 days' AND not_valid_before > not_valid_after - INTERVAL '5 minutes' LIMIT $1 OFFSET $2`, limit, offset) if rows != nil { defer rows.Close() } if err != nil { panic(fmt.Errorf("Error while retrieving certs: '%v'", err)) } i := 0 for rows.Next() { i++ var raw string var id int64 err = rows.Scan(&id, &raw) if err != nil { fmt.Println("error while parsing cert", id, ":", err) continue } certdata, err := base64.StdEncoding.DecodeString(raw) if err != nil { fmt.Println("error decoding base64 of cert", id, ":", err) continue } c, err := x509.ParseCertificate(certdata) if err != nil { fmt.Println("error while x509 parsing cert", id, ":", err) continue } //fmt.Println("cert", id, "has validity", c.NotBefore, c.NotAfter, "updating in database") _, err = db.Exec(`UPDATE certificates SET not_valid_before=$1, not_valid_after=$2 WHERE id=$3`, c.NotBefore, c.NotAfter, id) if err != nil { fmt.Println("error while updating cert", id, "in database:", err) } } if i == 0 { fmt.Println("done!") break } offset += limit } }