func main() { log.SetFlags(0) log.SetPrefix("") dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect) if err != nil { log.Printf("unable to parse %s: %s", *config.DbConnect, err) } if len(dbConnectStr) == 0 { config.Usage() os.Exit(2) } db, err := sql.Open("mysql", dbConnectStr) if err != nil { log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err) } if err = db.Ping(); err != nil { log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err) } dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"} dbMap := &gorp.DbMap{Db: db, Dialect: dialect} entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap, Verbose: *config.Verbose, KnownIssuers: make(map[string]int)} err = entriesDb.InitTables() if err != nil { log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err) } // Get a Firefox Telemetry client client, err := firefoxtelemetry.NewClient() if err != nil { log.Fatalf("unable to open Telemetry Client: %s", err) } // Find the versions and dates we can query versionList, err := client.GetVersions("release") if err != nil { log.Fatalf("unable to get telemetry versions: %s", err) } // log.Printf("%+v", versionList) // Filter on the oldest date we want oldestDate := time.Now().AddDate(0, 0, -1**config.HistoricalDays) versionDateMap := make(map[string][]time.Time) for _, obj := range versionList { timeObj, err := time.Parse(firefoxtelemetry.TelemetryDateFormat, obj.Date) if err != nil { // log.Fatalf("unable to parse date in GetVersions: %s", err) continue } if timeObj.Before(oldestDate) { continue } versionDateMap[obj.Version] = append(versionDateMap[obj.Version], timeObj) } type TelemetryResult struct { LoadsTLS int LoadsTotal int } dateDataMap := make(map[time.Time]TelemetryResult) for versionNumber, dateList := range versionDateMap { // Obtain data from Firefox Telemetry data, err := client.GetAggregates("HTTP_PAGELOAD_IS_SSL", "release", dateList, versionNumber) if err != nil { log.Printf("unable to get telemetry: %s", err) continue } // log.Printf("VERSION %s DATES: [%s] Result:\n%+v", versionNumber, dateList, data) for _, d := range data.Data { dateObj, err := time.Parse(firefoxtelemetry.TelemetryDateFormat, d.Date) if err != nil { log.Fatalf("unable to parse date in GetAggregates: %s", err) } entry, exists := dateDataMap[dateObj] if !exists { entry = TelemetryResult{} } entry.LoadsTLS += d.Histogram[1] entry.LoadsTotal += d.Histogram[0] + d.Histogram[1] dateDataMap[dateObj] = entry } } for dateObj, result := range dateDataMap { if *config.Verbose { percentTLS := (float64)(result.LoadsTLS) / (float64)(result.LoadsTotal) log.Printf("%s: (%d / %d) = %f percent was TLS\n", dateObj, result.LoadsTLS, result.LoadsTotal, percentTLS) } err = entriesDb.InsertOrUpdatePageloadIsTLS(dateObj, result.LoadsTLS, result.LoadsTotal) if err != nil { log.Fatalf("unable to update DB: %s", err) } } }
func main() { log.SetFlags(0) log.SetPrefix("") dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect) if err != nil { log.Printf("unable to parse %s: %s", *config.DbConnect, err) } if len(dbConnectStr) == 0 || len(*config.GeoipDbPath) == 0 { config.Usage() os.Exit(2) } db, err := sql.Open("mysql", dbConnectStr) if err != nil { log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err) } if err = db.Ping(); err != nil { log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err) } dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"} dbMap := &gorp.DbMap{Db: db, Dialect: dialect} entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap, Verbose: *config.Verbose, KnownIssuers: make(map[string]int)} err = entriesDb.InitTables() if err != nil { log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err) } geoDB, err := geoip2.Open(*config.GeoipDbPath) if err != nil { log.Fatalf("unable to prepare GeoIP DB. geoipDbPath=%s: %s", *config.GeoipDbPath, err) } defer geoDB.Close() netscan := &NetScan{ wg: new(sync.WaitGroup), db: entriesDb, geodb: geoDB, } if *config.Limit == 0 { // Didn't include a mandatory action, so print usage and exit. log.Fatalf("You must set a limit") } oldestAllowed := time.Now().AddDate(-1, 0, 0) var entries []ResolutionEntry _, err = dbMap.Select(&entries, `SELECT q.nameID, f.name FROM netscanqueue AS q NATURAL JOIN fqdn AS f LIMIT :limit`, map[string]interface{}{ "oldestAllowed": oldestAllowed, "limit": *config.Limit, }) if err != nil { log.Fatalf("unable to execute SQL: %s", err) } err = netscan.processEntries(entries) if err != nil { log.Fatalf("error while running importer: %s", err) } netscan.wg.Wait() os.Exit(0) }
func main() { log.SetFlags(0) log.SetPrefix("") dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect) if err != nil { log.Printf("unable to parse %s: %s", *config.DbConnect, err) } if len(dbConnectStr) == 0 || (config.CensysPath == nil && config.LogUrl == nil) { config.Usage() os.Exit(2) } db, err := sql.Open("mysql", dbConnectStr) if err != nil { log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err) } if err = db.Ping(); err != nil { log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err) } var certFolderDB *utils.FolderDatabase if config.CertPath != nil && len(*config.CertPath) > 0 { certFolderDB, err = utils.NewFolderDatabase(*config.CertPath, 0444, *config.CertsPerFolder) if err != nil { log.Fatalf("unable to open Certificate Path: %s: %s", config.CertPath, err) } } dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"} dbMap := &gorp.DbMap{Db: db, Dialect: dialect} entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap, Verbose: *config.Verbose, FullCerts: certFolderDB, KnownIssuers: make(map[string]int)} err = entriesDb.InitTables() if err != nil { log.Fatalf("unable to prepare SQL: %s: %s", dbConnectStr, err) } if config.LogUrl != nil && len(*config.LogUrl) > 5 { ctLogUrl, err := url.Parse(*config.LogUrl) if err != nil { log.Fatalf("unable to set Certificate Log: %s", err) } ctLog := client.New(*config.LogUrl, nil) log.Printf("Starting download from log %s, fullCerts=%t\n", ctLogUrl, (certFolderDB != nil)) err = downloadLog(ctLogUrl, ctLog, entriesDb) if err != nil { log.Fatalf("error while updating CT entries: %s", err) } os.Exit(0) } var importer censysdata.Importer if config.CensysUrl != nil && len(*config.CensysUrl) > 5 { urlImporter, err := censysdata.OpenURL(*config.CensysUrl) if err != nil { log.Fatalf("unable to open Censys URL: %s", err) } importer = urlImporter } else if config.CensysPath != nil && len(*config.CensysPath) > 5 { fileImporter, err := censysdata.OpenFile(*config.CensysPath) if err != nil { log.Fatalf("unable to open Censys file: %s", err) } importer = fileImporter defer fileImporter.Close() } else if *config.CensysStdin { stdinImporter, err := censysdata.OpenFileHandle(os.Stdin) if err != nil { log.Fatalf("unable to open stdin: %s", err) } importer = stdinImporter defer stdinImporter.Close() } if importer != nil { log.Printf("Starting Censys Import, using %s, fullCerts=%t\n", importer.String(), (certFolderDB != nil)) wg := new(sync.WaitGroup) err = processImporter(importer, entriesDb, wg) if err != nil { log.Fatalf("error while running importer: %s", err) } wg.Wait() os.Exit(0) } // Didn't include a mandatory action, so print usage and exit. config.Usage() os.Exit(2) }