func main() {
log.SetFlags(0)
log.SetPrefix("")
dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect)
if err != nil {
log.Printf("unable to parse %s: %s", *config.DbConnect, err)
}
if len(dbConnectStr) == 0 {
config.Usage()
os.Exit(2)
}
db, err := sql.Open("mysql", dbConnectStr)
if err != nil {
log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err)
}
if err = db.Ping(); err != nil {
log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err)
}
dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
dbMap := &gorp.DbMap{Db: db, Dialect: dialect}
entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap,
Verbose: *config.Verbose, KnownIssuers: make(map[string]int)}
err = entriesDb.InitTables()
if err != nil {
log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err)
}
// Get a Firefox Telemetry client
client, err := firefoxtelemetry.NewClient()
if err != nil {
log.Fatalf("unable to open Telemetry Client: %s", err)
}
// Find the versions and dates we can query
versionList, err := client.GetVersions("release")
if err != nil {
log.Fatalf("unable to get telemetry versions: %s", err)
}
// log.Printf("%+v", versionList)
// Filter on the oldest date we want
oldestDate := time.Now().AddDate(0, 0, -1**config.HistoricalDays)
versionDateMap := make(map[string][]time.Time)
for _, obj := range versionList {
timeObj, err := time.Parse(firefoxtelemetry.TelemetryDateFormat, obj.Date)
if err != nil {
// log.Fatalf("unable to parse date in GetVersions: %s", err)
continue
}
if timeObj.Before(oldestDate) {
continue
}
versionDateMap[obj.Version] = append(versionDateMap[obj.Version], timeObj)
}
type TelemetryResult struct {
LoadsTLS int
LoadsTotal int
}
dateDataMap := make(map[time.Time]TelemetryResult)
for versionNumber, dateList := range versionDateMap {
// Obtain data from Firefox Telemetry
data, err := client.GetAggregates("HTTP_PAGELOAD_IS_SSL", "release", dateList, versionNumber)
if err != nil {
log.Printf("unable to get telemetry: %s", err)
continue
}
// log.Printf("VERSION %s DATES: [%s] Result:\n%+v", versionNumber, dateList, data)
for _, d := range data.Data {
dateObj, err := time.Parse(firefoxtelemetry.TelemetryDateFormat, d.Date)
if err != nil {
log.Fatalf("unable to parse date in GetAggregates: %s", err)
}
entry, exists := dateDataMap[dateObj]
if !exists {
entry = TelemetryResult{}
}
entry.LoadsTLS += d.Histogram[1]
entry.LoadsTotal += d.Histogram[0] + d.Histogram[1]
dateDataMap[dateObj] = entry
}
}
for dateObj, result := range dateDataMap {
if *config.Verbose {
percentTLS := (float64)(result.LoadsTLS) / (float64)(result.LoadsTotal)
log.Printf("%s: (%d / %d) = %f percent was TLS\n", dateObj, result.LoadsTLS, result.LoadsTotal, percentTLS)
}
err = entriesDb.InsertOrUpdatePageloadIsTLS(dateObj, result.LoadsTLS, result.LoadsTotal)
if err != nil {
log.Fatalf("unable to update DB: %s", err)
}
}
}
func main() {
log.SetFlags(0)
log.SetPrefix("")
dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect)
if err != nil {
log.Printf("unable to parse %s: %s", *config.DbConnect, err)
}
if len(dbConnectStr) == 0 || len(*config.GeoipDbPath) == 0 {
config.Usage()
os.Exit(2)
}
db, err := sql.Open("mysql", dbConnectStr)
if err != nil {
log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err)
}
if err = db.Ping(); err != nil {
log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err)
}
dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
dbMap := &gorp.DbMap{Db: db, Dialect: dialect}
entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap,
Verbose: *config.Verbose, KnownIssuers: make(map[string]int)}
err = entriesDb.InitTables()
if err != nil {
log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err)
}
geoDB, err := geoip2.Open(*config.GeoipDbPath)
if err != nil {
log.Fatalf("unable to prepare GeoIP DB. geoipDbPath=%s: %s", *config.GeoipDbPath, err)
}
defer geoDB.Close()
netscan := &NetScan{
wg: new(sync.WaitGroup),
db: entriesDb,
geodb: geoDB,
}
if *config.Limit == 0 {
// Didn't include a mandatory action, so print usage and exit.
log.Fatalf("You must set a limit")
}
oldestAllowed := time.Now().AddDate(-1, 0, 0)
var entries []ResolutionEntry
_, err = dbMap.Select(&entries,
`SELECT q.nameID, f.name FROM netscanqueue AS q
NATURAL JOIN fqdn AS f
LIMIT :limit`,
map[string]interface{}{
"oldestAllowed": oldestAllowed,
"limit": *config.Limit,
})
if err != nil {
log.Fatalf("unable to execute SQL: %s", err)
}
err = netscan.processEntries(entries)
if err != nil {
log.Fatalf("error while running importer: %s", err)
}
netscan.wg.Wait()
os.Exit(0)
}
func main() {
log.SetFlags(0)
log.SetPrefix("")
dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect)
if err != nil {
log.Printf("unable to parse %s: %s", *config.DbConnect, err)
}
if len(dbConnectStr) == 0 || (config.CensysPath == nil && config.LogUrl == nil) {
config.Usage()
os.Exit(2)
}
db, err := sql.Open("mysql", dbConnectStr)
if err != nil {
log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err)
}
if err = db.Ping(); err != nil {
log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err)
}
var certFolderDB *utils.FolderDatabase
if config.CertPath != nil && len(*config.CertPath) > 0 {
certFolderDB, err = utils.NewFolderDatabase(*config.CertPath, 0444, *config.CertsPerFolder)
if err != nil {
log.Fatalf("unable to open Certificate Path: %s: %s", config.CertPath, err)
}
}
dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
dbMap := &gorp.DbMap{Db: db, Dialect: dialect}
entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap,
Verbose: *config.Verbose, FullCerts: certFolderDB,
KnownIssuers: make(map[string]int)}
err = entriesDb.InitTables()
if err != nil {
log.Fatalf("unable to prepare SQL: %s: %s", dbConnectStr, err)
}
if config.LogUrl != nil && len(*config.LogUrl) > 5 {
ctLogUrl, err := url.Parse(*config.LogUrl)
if err != nil {
log.Fatalf("unable to set Certificate Log: %s", err)
}
ctLog := client.New(*config.LogUrl, nil)
log.Printf("Starting download from log %s, fullCerts=%t\n", ctLogUrl, (certFolderDB != nil))
err = downloadLog(ctLogUrl, ctLog, entriesDb)
if err != nil {
log.Fatalf("error while updating CT entries: %s", err)
}
os.Exit(0)
}
var importer censysdata.Importer
if config.CensysUrl != nil && len(*config.CensysUrl) > 5 {
urlImporter, err := censysdata.OpenURL(*config.CensysUrl)
if err != nil {
log.Fatalf("unable to open Censys URL: %s", err)
}
importer = urlImporter
} else if config.CensysPath != nil && len(*config.CensysPath) > 5 {
fileImporter, err := censysdata.OpenFile(*config.CensysPath)
if err != nil {
log.Fatalf("unable to open Censys file: %s", err)
}
importer = fileImporter
defer fileImporter.Close()
} else if *config.CensysStdin {
stdinImporter, err := censysdata.OpenFileHandle(os.Stdin)
if err != nil {
log.Fatalf("unable to open stdin: %s", err)
}
importer = stdinImporter
defer stdinImporter.Close()
}
if importer != nil {
log.Printf("Starting Censys Import, using %s, fullCerts=%t\n", importer.String(), (certFolderDB != nil))
wg := new(sync.WaitGroup)
err = processImporter(importer, entriesDb, wg)
if err != nil {
log.Fatalf("error while running importer: %s", err)
}
wg.Wait()
os.Exit(0)
}
// Didn't include a mandatory action, so print usage and exit.
config.Usage()
os.Exit(2)
}
