func addCerts(csvFilename string, dbMap *gorp.DbMap, stats metrics.Statter, statsRate float32) {
file, err := os.Open(csvFilename)
cmd.FailOnError(err, "Could not open the file for reading")
csvReader := csv.NewReader(file)
for {
record, err := csvReader.Read()
if err == io.EOF {
break
} else if err != nil {
fmt.Println("Error:", err)
return
}
notAfter, err := time.Parse(datestampFormat, record[3])
spkiBytes, err := hex.DecodeString(record[4])
certDER, err := hex.DecodeString(record[7])
externalCert := core.ExternalCert{
SHA1: record[0],
Issuer: record[1],
Subject: record[2],
NotAfter: notAfter,
SPKI: spkiBytes,
Valid: record[5] == "1",
EV: record[6] == "1",
CertDER: certDER,
}
importStart := time.Now()
err = dbMap.Insert(&externalCert)
stats.TimingDuration("ExistingCert.Certs.ImportLatency", time.Since(importStart), statsRate)
stats.Inc("ExistingCert.Certs.Imported", 1, statsRate)
}
}
func addIdentifiers(csvFilename string, dbMap *gorp.DbMap, stats metrics.Statter, statsRate float32) {
file, err := os.Open(csvFilename)
cmd.FailOnError(err, "Could not open the file for reading")
csvReader := csv.NewReader(file)
for {
record, err := csvReader.Read()
if err == io.EOF {
break
} else if err != nil {
fmt.Println("Error:", err)
return
}
identifierData := core.IdentifierData{
ReversedName: record[1],
CertSHA1: record[0],
}
importStart := time.Now()
err = dbMap.Insert(&identifierData)
stats.TimingDuration("ExistingCert.Domains.ImportLatency", time.Since(importStart), statsRate)
stats.Inc("ExistingCert.Domains.Imported", 1, statsRate)
}
}
func removeInvalidCerts(csvFilename string, dbMap *gorp.DbMap, stats metrics.Statter, statsRate float32) {
file, err := os.Open(csvFilename)
cmd.FailOnError(err, "Could not open the file for reading")
csvReader := csv.NewReader(file)
for {
record, err := csvReader.Read()
if err == io.EOF {
break
} else if err != nil {
fmt.Println("Error:", err)
return
}
identifierData := core.IdentifierData{
CertSHA1: record[0],
}
externalCert := core.ExternalCert{
SHA1: record[0],
}
deleteStart := time.Now()
_, err = dbMap.Delete(&identifierData)
stats.TimingDuration("ExistingCert.Domains.DeleteLatency", time.Since(deleteStart), statsRate)
_, err = dbMap.Delete(&externalCert)
stats.TimingDuration("ExistingCert.Certs.DeleteLatency", time.Since(deleteStart), statsRate)
stats.Inc("ExistingCert.Removed", 1, statsRate)
}
}
// ProfileCmd runs forever, sending Go runtime statistics to StatsD.
func ProfileCmd(profileName string, stats metrics.Statter) {
var memoryStats runtime.MemStats
prevNumGC := int64(0)
c := time.Tick(1 * time.Second)
for range c {
runtime.ReadMemStats(&memoryStats)
// Gather goroutine count
stats.Gauge(fmt.Sprintf("%s.Gostats.Goroutines", profileName), int64(runtime.NumGoroutine()), 1.0)
// Gather various heap metrics
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Alloc", profileName), int64(memoryStats.HeapAlloc), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Objects", profileName), int64(memoryStats.HeapObjects), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Idle", profileName), int64(memoryStats.HeapIdle), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.InUse", profileName), int64(memoryStats.HeapInuse), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Released", profileName), int64(memoryStats.HeapReleased), 1.0)
// Gather various GC related metrics
if memoryStats.NumGC > 0 {
totalRecentGC := uint64(0)
realBufSize := uint32(256)
if memoryStats.NumGC < 256 {
realBufSize = memoryStats.NumGC
}
for _, pause := range memoryStats.PauseNs {
totalRecentGC += pause
}
gcPauseAvg := totalRecentGC / uint64(realBufSize)
lastGC := memoryStats.PauseNs[(memoryStats.NumGC+255)%256]
stats.Timing(fmt.Sprintf("%s.Gostats.Gc.PauseAvg", profileName), int64(gcPauseAvg), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.LastPause", profileName), int64(lastGC), 1.0)
}
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.NextAt", profileName), int64(memoryStats.NextGC), 1.0)
// Send both a counter and a gauge here we can much more easily observe
// the GC rate (versus the raw number of GCs) in graphing tools that don't
// like deltas
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.Count", profileName), int64(memoryStats.NumGC), 1.0)
gcInc := int64(memoryStats.NumGC) - prevNumGC
stats.Inc(fmt.Sprintf("%s.Gostats.Gc.Rate", profileName), gcInc, 1.0)
prevNumGC += gcInc
}
}