// ClientSetup loads various TLS certificates and creates a
// gRPC TransportCredentials that presents the client certificate
// and validates the certificate presented by the server is for a
// specific hostname and issued by the provided issuer certificate
// thens dials and returns a grpc.ClientConn to the remote service.
func ClientSetup(c *cmd.GRPCClientConfig, stats metrics.Scope) (*grpc.ClientConn, error) {
if len(c.ServerAddresses) == 0 {
return nil, fmt.Errorf("boulder/grpc: ServerAddresses is empty")
}
if stats == nil {
return nil, errNilScope
}
serverIssuerBytes, err := ioutil.ReadFile(c.ServerIssuerPath)
if err != nil {
return nil, err
}
rootCAs := x509.NewCertPool()
if ok := rootCAs.AppendCertsFromPEM(serverIssuerBytes); !ok {
return nil, fmt.Errorf("Failed to parse server issues from '%s'", c.ServerIssuerPath)
}
clientCert, err := tls.LoadX509KeyPair(c.ClientCertificatePath, c.ClientKeyPath)
if err != nil {
return nil, err
}
grpc_prometheus.EnableHandlingTimeHistogram()
ci := clientInterceptor{stats.NewScope("gRPCClient"), clock.Default(), c.Timeout.Duration}
creds := bcreds.NewClientCredentials(rootCAs, []tls.Certificate{clientCert})
return grpc.Dial(
"", // Since our staticResolver provides addresses we don't need to pass an address here
grpc.WithTransportCredentials(creds),
grpc.WithBalancer(grpc.RoundRobin(newStaticResolver(c.ServerAddresses))),
grpc.WithUnaryInterceptor(ci.intercept),
)
}
// NewAmqpRPCServer creates a new RPC server for the given queue and will begin
// consuming requests from the queue. To start the server you must call Start().
func NewAmqpRPCServer(
amqpConf *cmd.AMQPConfig,
maxConcurrentRPCServerRequests int64,
stats metrics.Scope,
log blog.Logger,
) (*AmqpRPCServer, error) {
stats = stats.NewScope("RPC")
reconnectBase := amqpConf.ReconnectTimeouts.Base.Duration
if reconnectBase == 0 {
reconnectBase = 20 * time.Millisecond
}
reconnectMax := amqpConf.ReconnectTimeouts.Max.Duration
if reconnectMax == 0 {
reconnectMax = time.Minute
}
return &AmqpRPCServer{
serverQueue: amqpConf.ServiceQueue,
connection: newAMQPConnector(amqpConf.ServiceQueue, reconnectBase, reconnectMax),
log: log,
dispatchTable: make(map[string]messageHandler),
maxConcurrentRPCServerRequests: maxConcurrentRPCServerRequests,
clk: clock.Default(),
stats: stats,
}, nil
}
// NewCachePurgeClient constructs a new CachePurgeClient
func NewCachePurgeClient(
endpoint,
clientToken,
clientSecret,
accessToken string,
retries int,
retryBackoff time.Duration,
log blog.Logger,
stats metrics.Scope,
) (*CachePurgeClient, error) {
stats = stats.NewScope("CCU")
if strings.HasSuffix(endpoint, "/") {
endpoint = endpoint[:len(endpoint)-1]
}
apiURL, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
return &CachePurgeClient{
client: new(http.Client),
apiEndpoint: endpoint,
apiHost: apiURL.Host,
apiScheme: strings.ToLower(apiURL.Scheme),
clientToken: clientToken,
clientSecret: clientSecret,
accessToken: accessToken,
retries: retries,
retryBackoff: retryBackoff,
log: log,
stats: stats,
clk: clock.Default(),
}, nil
}
func ReportDbConnCount(dbMap *gorp.DbMap, statter metrics.Scope) {
db := dbMap.Db
for {
statter.Gauge("OpenConnections", int64(db.Stats().OpenConnections))
time.Sleep(1 * time.Second)
}
}
// NewDNSResolverImpl constructs a new DNS resolver object that utilizes the
// provided list of DNS servers for resolution.
func NewDNSResolverImpl(
readTimeout time.Duration,
servers []string,
caaSERVFAILExceptions map[string]bool,
stats metrics.Scope,
clk clock.Clock,
maxTries int,
) *DNSResolverImpl {
// TODO(jmhodges): make constructor use an Option func pattern
dnsClient := new(dns.Client)
// Set timeout for underlying net.Conn
dnsClient.ReadTimeout = readTimeout
dnsClient.Net = "tcp"
return &DNSResolverImpl{
dnsClient: dnsClient,
servers: servers,
allowRestrictedAddresses: false,
caaSERVFAILExceptions: caaSERVFAILExceptions,
maxTries: maxTries,
clk: clk,
stats: stats,
txtStats: stats.NewScope("TXT"),
aStats: stats.NewScope("A"),
aaaaStats: stats.NewScope("AAAA"),
caaStats: stats.NewScope("CAA"),
mxStats: stats.NewScope("MX"),
}
}
// NewNonceService constructs a NonceService with defaults
func NewNonceService(scope metrics.Scope) (*NonceService, error) {
scope = scope.NewScope("NonceService")
key := make([]byte, 16)
if _, err := rand.Read(key); err != nil {
return nil, err
}
c, err := aes.NewCipher(key)
if err != nil {
panic("Failure in NewCipher: " + err.Error())
}
gcm, err := cipher.NewGCM(c)
if err != nil {
panic("Failure in NewGCM: " + err.Error())
}
return &NonceService{
earliest: 0,
latest: 0,
used: make(map[int64]bool, MaxUsed),
gcm: gcm,
maxUsed: MaxUsed,
stats: scope,
}, nil
}
// New constructs a Mailer to represent an account on a particular mail
// transfer agent.
func New(
server,
port,
username,
password string,
from mail.Address,
logger blog.Logger,
stats metrics.Scope,
reconnectBase time.Duration,
reconnectMax time.Duration) *MailerImpl {
return &MailerImpl{
dialer: &dialerImpl{
username: username,
password: password,
server: server,
port: port,
},
log: logger,
from: from,
clk: clock.Default(),
csprgSource: realSource{},
stats: stats.NewScope("Mailer"),
reconnectBase: reconnectBase,
reconnectMax: reconnectMax,
}
}
// NewRegistrationAuthorityImpl constructs a new RA object.
func NewRegistrationAuthorityImpl(
clk clock.Clock,
logger blog.Logger,
stats metrics.Scope,
maxContactsPerReg int,
keyPolicy goodkey.KeyPolicy,
maxNames int,
forceCNFromSAN bool,
reuseValidAuthz bool,
authorizationLifetime time.Duration,
pendingAuthorizationLifetime time.Duration,
pubc core.Publisher,
) *RegistrationAuthorityImpl {
ra := &RegistrationAuthorityImpl{
stats: stats,
clk: clk,
log: logger,
authorizationLifetime: authorizationLifetime,
pendingAuthorizationLifetime: pendingAuthorizationLifetime,
rlPolicies: ratelimit.New(),
tiMu: new(sync.RWMutex),
maxContactsPerReg: maxContactsPerReg,
keyPolicy: keyPolicy,
maxNames: maxNames,
forceCNFromSAN: forceCNFromSAN,
reuseValidAuthz: reuseValidAuthz,
regByIPStats: stats.NewScope("RA", "RateLimit", "RegistrationsByIP"),
pendAuthByRegIDStats: stats.NewScope("RA", "RateLimit", "PendingAuthorizationsByRegID"),
certsForDomainStats: stats.NewScope("RA", "RateLimit", "CertificatesForDomain"),
totalCertsStats: stats.NewScope("RA", "RateLimit", "TotalCertificates"),
publisher: pubc,
}
return ra
}
// NewServer loads various TLS certificates and creates a
// gRPC Server that verifies the client certificate was
// issued by the provided issuer certificate and presents a
// a server TLS certificate.
func NewServer(c *cmd.GRPCServerConfig, stats metrics.Scope) (*grpc.Server, net.Listener, error) {
if stats == nil {
return nil, nil, errNilScope
}
cert, err := tls.LoadX509KeyPair(c.ServerCertificatePath, c.ServerKeyPath)
if err != nil {
return nil, nil, err
}
clientIssuerBytes, err := ioutil.ReadFile(c.ClientIssuerPath)
if err != nil {
return nil, nil, err
}
clientCAs := x509.NewCertPool()
if ok := clientCAs.AppendCertsFromPEM(clientIssuerBytes); !ok {
return nil, nil, errors.New("Failed to parse client issuer certificates")
}
servTLSConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: clientCAs,
}
acceptedSANs := make(map[string]struct{})
for _, name := range c.ClientNames {
acceptedSANs[name] = struct{}{}
}
creds, err := bcreds.NewServerCredentials(servTLSConfig, acceptedSANs)
if err != nil {
return nil, nil, err
}
l, err := net.Listen("tcp", c.Address)
if err != nil {
return nil, nil, err
}
grpc_prometheus.EnableHandlingTimeHistogram()
si := &serverInterceptor{stats.NewScope("gRPCServer"), clock.Default()}
return grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(si.intercept)), l, nil
}
// exchangeOne performs a single DNS exchange with a randomly chosen server
// out of the server list, returning the response, time, and error (if any).
// This method sets the DNSSEC OK bit on the message to true before sending
// it to the resolver in case validation isn't the resolvers default behaviour.
func (dnsResolver *DNSResolverImpl) exchangeOne(hostname string, qtype uint16, msgStats metrics.Scope) (rsp *dns.Msg, err error) {
m := new(dns.Msg)
// Set question type
m.SetQuestion(dns.Fqdn(hostname), qtype)
// Set DNSSEC OK bit for resolver
m.SetEdns0(4096, true)
if len(dnsResolver.Servers) < 1 {
err = fmt.Errorf("Not configured with at least one DNS Server")
return
}
dnsResolver.stats.Inc("Rate", 1)
// Randomly pick a server
chosenServer := dnsResolver.Servers[rand.Intn(len(dnsResolver.Servers))]
msg, rtt, err := dnsResolver.DNSClient.Exchange(m, chosenServer)
msgStats.TimingDuration("RTT", rtt)
if err == nil {
msgStats.Inc("Successes", 1)
} else {
msgStats.Inc("Errors", 1)
}
return msg, err
}
// NewDNSResolverImpl constructs a new DNS resolver object that utilizes the
// provided list of DNS servers for resolution.
func NewDNSResolverImpl(readTimeout time.Duration, servers []string, stats metrics.Scope) *DNSResolverImpl {
dnsClient := new(dns.Client)
// Set timeout for underlying net.Conn
dnsClient.ReadTimeout = readTimeout
dnsClient.Net = "tcp"
return &DNSResolverImpl{
DNSClient: dnsClient,
Servers: servers,
allowRestrictedAddresses: false,
stats: stats,
txtStats: stats.NewScope("TXT"),
aStats: stats.NewScope("A"),
caaStats: stats.NewScope("CAA"),
mxStats: stats.NewScope("MX"),
}
}
func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus, stats metrics.Scope) error {
// Use the semaphore pattern from
// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
// outstanding requests. The number of outstanding requests equals the
// capacity of the channel.
sem := make(chan int, updater.parallelGenerateOCSPRequests)
wait := func() {
sem <- 1 // Block until there's capacity.
}
done := func() {
<-sem // Indicate there's more capacity.
}
work := func(status core.CertificateStatus) {
defer done()
meta, err := updater.generateResponse(ctx, status)
if err != nil {
updater.log.AuditErr(fmt.Sprintf("Failed to generate OCSP response: %s", err))
stats.Inc("Errors.ResponseGeneration", 1)
return
}
updater.stats.Inc("GeneratedResponses", 1)
err = updater.storeResponse(meta)
if err != nil {
updater.log.AuditErr(fmt.Sprintf("Failed to store OCSP response: %s", err))
stats.Inc("Errors.StoreResponse", 1)
return
}
stats.Inc("StoredResponses", 1)
}
for _, status := range statuses {
wait()
go work(status)
}
// Block until the channel reaches its full capacity again, indicating each
// goroutine has completed.
for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
wait()
}
return nil
}
// NewAmqpRPCClient constructs an RPC client using AMQP
func NewAmqpRPCClient(
clientQueuePrefix string,
amqpConf *cmd.AMQPConfig,
rpcConf *cmd.RPCServerConfig,
stats metrics.Scope,
) (rpc *AmqpRPCCLient, err error) {
stats = stats.NewScope("RPC")
hostname, err := os.Hostname()
if err != nil {
return nil, err
}
randID := make([]byte, 3)
_, err = rand.Read(randID)
if err != nil {
return nil, err
}
clientQueue := fmt.Sprintf("%s.%s.%x", clientQueuePrefix, hostname, randID)
reconnectBase := amqpConf.ReconnectTimeouts.Base.Duration
if reconnectBase == 0 {
reconnectBase = 20 * time.Millisecond
}
reconnectMax := amqpConf.ReconnectTimeouts.Max.Duration
if reconnectMax == 0 {
reconnectMax = time.Minute
}
timeout := rpcConf.RPCTimeout.Duration
if timeout == 0 {
timeout = 10 * time.Second
}
rpc = &AmqpRPCCLient{
serverQueue: rpcConf.Server,
clientQueue: clientQueue,
connection: newAMQPConnector(clientQueue, reconnectBase, reconnectMax),
pending: make(map[string]chan []byte),
timeout: timeout,
log: blog.Get(),
stats: stats,
}
err = rpc.connection.connect(amqpConf)
if err != nil {
return nil, err
}
go func() {
for {
select {
case msg, ok := <-rpc.connection.messages():
if ok {
corrID := msg.CorrelationId
rpc.mu.RLock()
responseChan, present := rpc.pending[corrID]
rpc.mu.RUnlock()
if !present {
// occurs when a request is timed out and the arrives
// afterwards
stats.Inc("AfterTimeoutResponseArrivals."+clientQueuePrefix, 1)
continue
}
responseChan <- msg.Body
rpc.mu.Lock()
delete(rpc.pending, corrID)
rpc.mu.Unlock()
} else {
// chan has been closed by rpc.connection.Cancel
rpc.log.Info(fmt.Sprintf(" [!] Client reply channel closed: %s", rpc.clientQueue))
continue
}
case err = <-rpc.connection.closeChannel():
rpc.log.Info(fmt.Sprintf(" [!] Client reply channel closed : %s", rpc.clientQueue))
rpc.connection.reconnect(amqpConf, rpc.log)
}
}
}()
return rpc, err
}
// This is somewhat gross but can be pared down a bit once the publisher and this
// are fully smooshed together
func newUpdater(
stats metrics.Scope,
clk clock.Clock,
dbMap ocspDB,
ca core.CertificateAuthority,
pub core.Publisher,
sac core.StorageAuthority,
config cmd.OCSPUpdaterConfig,
logConfigs []cmd.LogDescription,
issuerPath string,
log blog.Logger,
) (*OCSPUpdater, error) {
if config.NewCertificateBatchSize == 0 ||
config.OldOCSPBatchSize == 0 ||
config.MissingSCTBatchSize == 0 {
return nil, fmt.Errorf("Loop batch sizes must be non-zero")
}
if config.NewCertificateWindow.Duration == 0 ||
config.OldOCSPWindow.Duration == 0 ||
config.MissingSCTWindow.Duration == 0 {
return nil, fmt.Errorf("Loop window sizes must be non-zero")
}
if config.OCSPStaleMaxAge.Duration == 0 {
// Default to 30 days
config.OCSPStaleMaxAge = cmd.ConfigDuration{Duration: time.Hour * 24 * 30}
}
if config.ParallelGenerateOCSPRequests == 0 {
// Default to 1
config.ParallelGenerateOCSPRequests = 1
}
logs := make([]*ctLog, len(logConfigs))
for i, logConfig := range logConfigs {
l, err := newLog(logConfig)
if err != nil {
return nil, err
}
logs[i] = l
}
updater := OCSPUpdater{
stats: stats,
clk: clk,
dbMap: dbMap,
cac: ca,
log: log,
sac: sac,
pubc: pub,
logs: logs,
ocspMinTimeToExpiry: config.OCSPMinTimeToExpiry.Duration,
ocspStaleMaxAge: config.OCSPStaleMaxAge.Duration,
oldestIssuedSCT: config.OldestIssuedSCT.Duration,
parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
}
// Setup loops
updater.loops = []*looper{
{
clk: clk,
stats: stats.NewScope("NewCertificates"),
batchSize: config.NewCertificateBatchSize,
tickDur: config.NewCertificateWindow.Duration,
tickFunc: updater.newCertificateTick,
name: "NewCertificates",
failureBackoffFactor: config.SignFailureBackoffFactor,
failureBackoffMax: config.SignFailureBackoffMax.Duration,
},
{
clk: clk,
stats: stats.NewScope("OldOCSPResponses"),
batchSize: config.OldOCSPBatchSize,
tickDur: config.OldOCSPWindow.Duration,
tickFunc: updater.oldOCSPResponsesTick,
name: "OldOCSPResponses",
failureBackoffFactor: config.SignFailureBackoffFactor,
failureBackoffMax: config.SignFailureBackoffMax.Duration,
},
// The missing SCT loop doesn't need to know about failureBackoffFactor or
// failureBackoffMax as it doesn't make any calls to the CA
{
clk: clk,
stats: stats.NewScope("MissingSCTReceipts"),
batchSize: config.MissingSCTBatchSize,
tickDur: config.MissingSCTWindow.Duration,
tickFunc: updater.missingReceiptsTick,
name: "MissingSCTReceipts",
},
}
if config.RevokedCertificateBatchSize != 0 &&
config.RevokedCertificateWindow.Duration != 0 {
updater.loops = append(updater.loops, &looper{
clk: clk,
stats: stats,
batchSize: config.RevokedCertificateBatchSize,
tickDur: config.RevokedCertificateWindow.Duration,
tickFunc: updater.revokedCertificatesTick,
name: "RevokedCertificates",
failureBackoffFactor: config.SignFailureBackoffFactor,
failureBackoffMax: config.SignFailureBackoffMax.Duration,
})
}
// TODO(#1050): Remove this gate and the nil ccu checks below
if config.AkamaiBaseURL != "" {
issuer, err := core.LoadCert(issuerPath)
ccu, err := akamai.NewCachePurgeClient(
config.AkamaiBaseURL,
config.AkamaiClientToken,
config.AkamaiClientSecret,
config.AkamaiAccessToken,
config.AkamaiPurgeRetries,
config.AkamaiPurgeRetryBackoff.Duration,
log,
stats,
)
if err != nil {
return nil, err
}
updater.ccu = ccu
updater.issuer = issuer
}
return &updater, nil
}
// exchangeOne performs a single DNS exchange with a randomly chosen server
// out of the server list, returning the response, time, and error (if any).
// This method sets the DNSSEC OK bit on the message to true before sending
// it to the resolver in case validation isn't the resolvers default behaviour.
func (dnsResolver *DNSResolverImpl) exchangeOne(ctx context.Context, hostname string, qtype uint16, msgStats metrics.Scope) (*dns.Msg, error) {
m := new(dns.Msg)
// Set question type
m.SetQuestion(dns.Fqdn(hostname), qtype)
// Set DNSSEC OK bit for resolver
m.SetEdns0(4096, true)
if len(dnsResolver.servers) < 1 {
return nil, fmt.Errorf("Not configured with at least one DNS Server")
}
dnsResolver.stats.Inc("Rate", 1)
// Randomly pick a server
chosenServer := dnsResolver.servers[rand.Intn(len(dnsResolver.servers))]
client := dnsResolver.dnsClient
tries := 1
start := dnsResolver.clk.Now()
msgStats.Inc("Calls", 1)
defer func() {
msgStats.TimingDuration("Latency", dnsResolver.clk.Now().Sub(start))
}()
for {
msgStats.Inc("Tries", 1)
ch := make(chan dnsResp, 1)
go func() {
rsp, rtt, err := client.Exchange(m, chosenServer)
msgStats.TimingDuration("SingleTryLatency", rtt)
ch <- dnsResp{m: rsp, err: err}
}()
select {
case <-ctx.Done():
msgStats.Inc("Cancels", 1)
msgStats.Inc("Errors", 1)
return nil, ctx.Err()
case r := <-ch:
if r.err != nil {
msgStats.Inc("Errors", 1)
operr, ok := r.err.(*net.OpError)
isRetryable := ok && operr.Temporary()
hasRetriesLeft := tries < dnsResolver.maxTries
if isRetryable && hasRetriesLeft {
tries++
continue
} else if isRetryable && !hasRetriesLeft {
msgStats.Inc("RanOutOfTries", 1)
}
} else {
msgStats.Inc("Successes", 1)
}
return r.m, r.err
}
}
}
// ProfileCmd runs forever, sending Go runtime statistics to StatsD.
func ProfileCmd(stats metrics.Scope) {
stats = stats.NewScope("Gostats")
var memoryStats runtime.MemStats
prevNumGC := int64(0)
c := time.Tick(1 * time.Second)
for range c {
runtime.ReadMemStats(&memoryStats)
// Gather goroutine count
stats.Gauge("Goroutines", int64(runtime.NumGoroutine()))
// Gather various heap metrics
stats.Gauge("Heap.Alloc", int64(memoryStats.HeapAlloc))
stats.Gauge("Heap.Objects", int64(memoryStats.HeapObjects))
stats.Gauge("Heap.Idle", int64(memoryStats.HeapIdle))
stats.Gauge("Heap.InUse", int64(memoryStats.HeapInuse))
stats.Gauge("Heap.Released", int64(memoryStats.HeapReleased))
// Gather various GC related metrics
if memoryStats.NumGC > 0 {
totalRecentGC := uint64(0)
realBufSize := uint32(256)
if memoryStats.NumGC < 256 {
realBufSize = memoryStats.NumGC
}
for _, pause := range memoryStats.PauseNs {
totalRecentGC += pause
}
gcPauseAvg := totalRecentGC / uint64(realBufSize)
lastGC := memoryStats.PauseNs[(memoryStats.NumGC+255)%256]
stats.Timing("Gc.PauseAvg", int64(gcPauseAvg))
stats.Gauge("Gc.LastPause", int64(lastGC))
}
stats.Gauge("Gc.NextAt", int64(memoryStats.NextGC))
// Send both a counter and a gauge here we can much more easily observe
// the GC rate (versus the raw number of GCs) in graphing tools that don't
// like deltas
stats.Gauge("Gc.Count", int64(memoryStats.NumGC))
gcInc := int64(memoryStats.NumGC) - prevNumGC
stats.Inc("Gc.Rate", gcInc)
prevNumGC += gcInc
}
}