func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// batch side: do 100 certs at a time
limit := 100
batch := 0
for {
fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE sha256_subject_spki IS NULL
ORDER BY id ASC LIMIT $1`, limit)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
_, err = db.Exec(`UPDATE certificates SET sha256_subject_spki=$1 WHERE id=$2`,
certificate.SHA256SubjectSPKI(c), id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
}
if i == 0 {
fmt.Println("done!")
break
}
//offset += limit
batch++
}
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// get all the sha256 fingerprints of dup certificates
rows, err := db.Query(`select sha256_fingerprint from certificates
group by sha256_fingerprint
having count(sha256_fingerprint) > 1`)
if rows != nil {
defer rows.Close()
}
if err != nil {
log.Fatal(err)
}
for rows.Next() {
var fp string
err = rows.Scan(&fp)
if err != nil {
log.Fatal(err)
}
// get all the ids of certificates with this fingerprint
rows, err := db.Query(`select id from certificates where sha256_fingerprint = $1`, fp)
if rows != nil {
defer rows.Close()
}
if err != nil {
log.Fatal(err)
}
var ids []uint64
var smallestid uint64 = 18446744073709551615
for rows.Next() {
var id uint64
err = rows.Scan(&id)
if err != nil {
log.Fatal(err)
}
if id < smallestid {
smallestid = id
}
ids = append(ids, id)
}
log.Printf("Found %d certificates with fingerprint %s, smallest id is %d", len(ids), fp, smallestid)
for _, id := range ids {
if id == smallestid {
continue
}
log.Println("reattaching all trust from", id, "to", smallestid)
// reattach all trust to the smallest id instead of the current id
_, err = db.Exec(`update trust set cert_id = $1 where cert_id = $2`, smallestid, id)
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(`update trust set issuer_id = $1 where issuer_id = $2`, smallestid, id)
if err != nil {
log.Fatal(err)
}
log.Println("reattaching all scans from", id, "to", smallestid)
// reattach all trust to the smallest id instead of the current id
_, err = db.Exec(`update scans set cert_id = $1 where cert_id = $2`, smallestid, id)
if err != nil {
log.Fatal(err)
}
log.Println("deleting certificate", id)
// remove the duplicate certificate
_, err = db.Exec(`delete from certificates where id = $1`, id)
if err != nil {
log.Fatal(err)
}
}
}
}
func main() {
var observatory = flag.String("observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory")
flag.Parse()
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// batch side: do 100 certs at a time
limit := 100
batch := 0
var donedomains []string
for {
fmt.Printf("\nProcessing batch %d to %d\n", batch*limit, batch*limit+limit)
rows, err := db.Query(` SELECT domains
FROM certificates INNER JOIN trust ON (trust.cert_id=certificates.id)
WHERE is_ca='false' AND trusted_mozilla='true'
ORDER BY certificates.id ASC LIMIT $1 OFFSET $2`, limit, batch*limit)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var domains string
err = rows.Scan(&domains)
if err != nil {
fmt.Println("error while retrieving domains:", err)
continue
}
for _, domain := range strings.Split(domains, ",") {
domain = strings.TrimSpace(domain)
if domain == "" {
continue
}
if domain[0] == '*' {
domain = "www" + domain[1:]
}
if contains(donedomains, domain) {
continue
}
resp, err := http.Post(*observatory+"/api/v1/scan?target="+domain, "application/json", nil)
if err != nil {
panic(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
var scan scan
err = json.Unmarshal(body, &scan)
if err != nil {
panic(err)
}
fmt.Printf("Started scan %d on %s - %s/api/v1/results?id=%d\n", scan.ID, domain, *observatory, scan.ID)
donedomains = append(donedomains, domain)
time.Sleep(500 * time.Millisecond)
}
}
if i == 0 {
fmt.Println("done!")
break
}
batch++
}
}
func main() {
log := logger.GetLogger()
router := NewRouter()
var cfgFile string
var debug bool
flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/api.cfg", "Input file csv format")
flag.BoolVar(&debug, "debug", false, "Set debug logging")
flag.Parse()
if debug {
logger.SetLevelToDebug()
}
conf, err := config.Load(cfgFile)
if err != nil {
log.Fatal("Failed to load configuration: %v", err)
}
if !conf.General.Enable && os.Getenv("TLSOBS_API_ENABLE") != "on" {
log.Fatal("API is disabled in configuration")
}
dbtls := "disable"
if conf.General.PostgresUseTLS {
dbtls = "verify-full"
}
db, err := pg.RegisterConnection(
conf.General.PostgresDB,
conf.General.PostgresUser,
conf.General.PostgresPass,
conf.General.Postgres,
dbtls)
defer db.Close()
if err != nil {
log.Fatal(err)
}
db.SetMaxOpenConns(runtime.NumCPU() * 27)
db.SetMaxIdleConns(2)
// simple DB watchdog, crashes the process if connection dies
go func() {
for {
var one uint
err = db.QueryRow("SELECT 1").Scan(&one)
if err != nil {
log.Fatal("Database connection failed:", err)
}
if one != 1 {
log.Fatal("Apparently the database doesn't know the meaning of one anymore. Crashing.")
}
time.Sleep(10 * time.Second)
}
}()
scanRefreshRate = float64(conf.General.ScanRefreshRate)
// wait for clients
err = http.ListenAndServe(":8083", Adapt(router, AddDB(db)))
log.Fatal(err)
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// batch side: do 100 certs at a time
limit := 100
batch := 0
for {
fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE key_alg IS NULL
ORDER BY id ASC LIMIT $1`, limit)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
key, err := getPublicKeyInfo(c)
if err != nil {
fmt.Println("error while parsing public key info for cert", id, ":", err)
continue
}
keydata, err := json.Marshal(key)
if err != nil {
fmt.Println("error while marshalling key info of cert", id, " : ", err)
continue
}
pkp_sha256 := certificate.PKPSHA256Hash(c)
_, err = db.Exec(`UPDATE certificates SET key=$1, key_alg=$2, pkp_sha256=$3 WHERE id=$4`,
keydata, key.Alg, pkp_sha256, id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
}
if i == 0 {
fmt.Println("done!")
break
}
//offset += limit
batch++
}
}
func main() {
var (
cfgFile, cipherscan string
debug bool
)
flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/scanner.cfg", "Configuration file")
flag.StringVar(&cipherscan, "b", "/opt/cipherscan/cipherscan", "Cipherscan binary location")
flag.BoolVar(&debug, "debug", false, "Set debug logging")
flag.Parse()
if debug {
logger.SetLevelToDebug()
}
conf, err := config.Load(cfgFile)
if err != nil {
log.Fatal(fmt.Sprintf("Failed to load configuration: %v", err))
}
if !conf.General.Enable && os.Getenv("TLSOBS_SCANNER_ENABLE") != "on" {
log.Fatal("Scanner is disabled in configuration")
}
_, err = os.Stat(cipherscan)
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
}
// increase the n
runtime.GOMAXPROCS(conf.General.MaxProc)
dbtls := "disable"
if conf.General.PostgresUseTLS {
dbtls = "verify-full"
}
db, err = pg.RegisterConnection(
conf.General.PostgresDB,
conf.General.PostgresUser,
conf.General.PostgresPass,
conf.General.Postgres,
dbtls)
defer db.Close()
if err != nil {
log.WithFields(logrus.Fields{
"error": err.Error(),
}).Fatal("Failed to connect to database")
}
db.SetMaxOpenConns(conf.General.MaxProc)
db.SetMaxIdleConns(10)
// simple DB watchdog, crashes the process if connection dies
go func() {
for {
var one uint
err = db.QueryRow("SELECT 1").Scan(&one)
if err != nil {
log.Fatal("Database connection failed:", err)
}
if one != 1 {
log.Fatal("Apparently the database doesn't know the meaning of one anymore. Crashing.")
}
time.Sleep(10 * time.Second)
}
}()
incomingScans := db.RegisterScanListener(
conf.General.PostgresDB,
conf.General.PostgresUser,
conf.General.PostgresPass,
conf.General.Postgres,
"disable")
Setup(conf)
activeScanners := 0
for scanID := range incomingScans {
// wait until we have an available scanner
for {
if activeScanners >= conf.General.MaxProc {
time.Sleep(time.Second)
} else {
break
}
}
go func() {
activeScanners++
scan(scanID, cipherscan)
activeScanners--
}()
}
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// batch side: do 100 certs at a time
limit := 100
batch := 0
for {
fmt.Printf("\nProcessing batch %d to %d: ", batch*limit, batch*limit+limit)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE issuer IS NULL AND subject IS NULL AND domains IS NULL
ORDER BY id ASC LIMIT $1`, limit)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
issuer := certificate.Issuer{Country: c.Issuer.Country, CommonName: c.Issuer.CommonName, OrgUnit: c.Issuer.OrganizationalUnit, Organisation: c.Issuer.Organization}
issuerjs, err := json.Marshal(issuer)
if err != nil {
fmt.Println("error while marshalling issuer of cert", id, " : ", err)
continue
}
subject := certificate.Subject{Country: c.Subject.Country, CommonName: c.Subject.CommonName, OrgUnit: c.Subject.OrganizationalUnit, Organisation: c.Subject.Organization}
subjectjs, err := json.Marshal(subject)
if err != nil {
fmt.Println("error while marshalling subject of cert", id, " : ", err)
continue
}
domainstr := ""
if !c.IsCA {
domainfound := false
for _, d := range c.DNSNames {
if d == c.Subject.CommonName {
domainfound = true
}
}
var domains []string
if !domainfound {
domains = append(c.DNSNames, c.Subject.CommonName)
} else {
domains = c.DNSNames
}
domainstr = strings.Join(domains, ",")
fmt.Printf("%d,", id)
}
_, err = db.Exec(`UPDATE certificates SET issuer=$1, subject=$2, domains=$3 WHERE id=$4`,
issuerjs, subjectjs, domainstr, id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
}
if i == 0 {
fmt.Println("done!")
break
}
//offset += limit
batch++
}
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
batch := 0
for {
fmt.Printf("\nProcessing batch %d to %d: ", batch, batch+100)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE x509_certificatepolicies IS NULL OR x509_certificatepolicies='null'
OR permitted_names IS NULL OR permitted_names='null'
ORDER BY id ASC
LIMIT 100`)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
var valInfo certificate.ValidationInfo
cert := certificate.CertToStored(c, "", "", "", "", &valInfo)
policies, err := json.Marshal(cert.X509v3Extensions.PolicyIdentifiers)
if err != nil {
log.Printf("error while marshalling policies for cert %d: %v", id, err)
continue
}
permittednames, err := json.Marshal(cert.X509v3Extensions.PermittedNames)
if err != nil {
log.Printf("error while marshalling permitted names for cert %d: %v", id, err)
continue
}
log.Printf("id=%d, subject=%s, policies=%s, permittednames=%s", id, cert.Subject.String(), policies, permittednames)
_, err = db.Exec(`UPDATE certificates
SET x509_certificatepolicies=$1,
permitted_names=$2,
is_name_constrained=$3
WHERE id=$4`,
policies,
permittednames,
cert.X509v3Extensions.IsNameConstrained,
id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
fmt.Printf(".")
}
if i == 0 {
fmt.Println("done!")
break
}
//offset += limit
batch += 100
}
}
func main() {
db, err := database.RegisterConnection(
os.Getenv("TLSOBS_POSTGRESDB"),
os.Getenv("TLSOBS_POSTGRESUSER"),
os.Getenv("TLSOBS_POSTGRESPASS"),
os.Getenv("TLSOBS_POSTGRES"),
"require")
defer db.Close()
if err != nil {
panic(err)
}
// batch side: do 100 certs at a time
limit := 100
offset := 0
for {
fmt.Println("Processing batch", offset, "to", offset+limit)
rows, err := db.Query(`SELECT id, raw_cert
FROM certificates
WHERE not_valid_after < NOW() AND not_valid_after > NOW() - INTERVAL '4 days'
AND not_valid_before > not_valid_after - INTERVAL '5 minutes'
LIMIT $1 OFFSET $2`, limit, offset)
if rows != nil {
defer rows.Close()
}
if err != nil {
panic(fmt.Errorf("Error while retrieving certs: '%v'", err))
}
i := 0
for rows.Next() {
i++
var raw string
var id int64
err = rows.Scan(&id, &raw)
if err != nil {
fmt.Println("error while parsing cert", id, ":", err)
continue
}
certdata, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
fmt.Println("error decoding base64 of cert", id, ":", err)
continue
}
c, err := x509.ParseCertificate(certdata)
if err != nil {
fmt.Println("error while x509 parsing cert", id, ":", err)
continue
}
//fmt.Println("cert", id, "has validity", c.NotBefore, c.NotAfter, "updating in database")
_, err = db.Exec(`UPDATE certificates SET not_valid_before=$1, not_valid_after=$2 WHERE id=$3`,
c.NotBefore, c.NotAfter, id)
if err != nil {
fmt.Println("error while updating cert", id, "in database:", err)
}
}
if i == 0 {
fmt.Println("done!")
break
}
offset += limit
}
}