// exchangeOne performs a single DNS exchange with a randomly chosen server
// out of the server list, returning the response, time, and error (if any).
// This method sets the DNSSEC OK bit on the message to true before sending
// it to the resolver in case validation isn't the resolvers default behaviour.
func (dnsResolver *DNSResolverImpl) exchangeOne(hostname string, qtype uint16, msgStats metrics.Scope) (rsp *dns.Msg, err error) {
m := new(dns.Msg)
// Set question type
m.SetQuestion(dns.Fqdn(hostname), qtype)
// Set DNSSEC OK bit for resolver
m.SetEdns0(4096, true)
if len(dnsResolver.Servers) < 1 {
err = fmt.Errorf("Not configured with at least one DNS Server")
return
}
dnsResolver.stats.Inc("Rate", 1)
// Randomly pick a server
chosenServer := dnsResolver.Servers[rand.Intn(len(dnsResolver.Servers))]
msg, rtt, err := dnsResolver.DNSClient.Exchange(m, chosenServer)
msgStats.TimingDuration("RTT", rtt)
if err == nil {
msgStats.Inc("Successes", 1)
} else {
msgStats.Inc("Errors", 1)
}
return msg, err
}
// ProfileCmd runs forever, sending Go runtime statistics to StatsD.
func ProfileCmd(stats metrics.Scope) {
stats = stats.NewScope("Gostats")
var memoryStats runtime.MemStats
prevNumGC := int64(0)
c := time.Tick(1 * time.Second)
for range c {
runtime.ReadMemStats(&memoryStats)
// Gather goroutine count
stats.Gauge("Goroutines", int64(runtime.NumGoroutine()))
// Gather various heap metrics
stats.Gauge("Heap.Alloc", int64(memoryStats.HeapAlloc))
stats.Gauge("Heap.Objects", int64(memoryStats.HeapObjects))
stats.Gauge("Heap.Idle", int64(memoryStats.HeapIdle))
stats.Gauge("Heap.InUse", int64(memoryStats.HeapInuse))
stats.Gauge("Heap.Released", int64(memoryStats.HeapReleased))
// Gather various GC related metrics
if memoryStats.NumGC > 0 {
totalRecentGC := uint64(0)
realBufSize := uint32(256)
if memoryStats.NumGC < 256 {
realBufSize = memoryStats.NumGC
}
for _, pause := range memoryStats.PauseNs {
totalRecentGC += pause
}
gcPauseAvg := totalRecentGC / uint64(realBufSize)
lastGC := memoryStats.PauseNs[(memoryStats.NumGC+255)%256]
stats.Timing("Gc.PauseAvg", int64(gcPauseAvg))
stats.Gauge("Gc.LastPause", int64(lastGC))
}
stats.Gauge("Gc.NextAt", int64(memoryStats.NextGC))
// Send both a counter and a gauge here we can much more easily observe
// the GC rate (versus the raw number of GCs) in graphing tools that don't
// like deltas
stats.Gauge("Gc.Count", int64(memoryStats.NumGC))
gcInc := int64(memoryStats.NumGC) - prevNumGC
stats.Inc("Gc.Rate", gcInc)
prevNumGC += gcInc
}
}
func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, statuses []core.CertificateStatus, stats metrics.Scope) error {
// Use the semaphore pattern from
// https://github.com/golang/go/wiki/BoundingResourceUse to send a number of
// GenerateOCSP / storeResponse requests in parallel, while limiting the total number of
// outstanding requests. The number of outstanding requests equals the
// capacity of the channel.
sem := make(chan int, updater.parallelGenerateOCSPRequests)
wait := func() {
sem <- 1 // Block until there's capacity.
}
done := func() {
<-sem // Indicate there's more capacity.
}
work := func(status core.CertificateStatus) {
defer done()
meta, err := updater.generateResponse(ctx, status)
if err != nil {
updater.log.AuditErr(fmt.Sprintf("Failed to generate OCSP response: %s", err))
stats.Inc("Errors.ResponseGeneration", 1)
return
}
updater.stats.Inc("GeneratedResponses", 1)
err = updater.storeResponse(meta)
if err != nil {
updater.log.AuditErr(fmt.Sprintf("Failed to store OCSP response: %s", err))
stats.Inc("Errors.StoreResponse", 1)
return
}
stats.Inc("StoredResponses", 1)
}
for _, status := range statuses {
wait()
go work(status)
}
// Block until the channel reaches its full capacity again, indicating each
// goroutine has completed.
for i := 0; i < updater.parallelGenerateOCSPRequests; i++ {
wait()
}
return nil
}
// NewAmqpRPCClient constructs an RPC client using AMQP
func NewAmqpRPCClient(
clientQueuePrefix string,
amqpConf *cmd.AMQPConfig,
rpcConf *cmd.RPCServerConfig,
stats metrics.Scope,
) (rpc *AmqpRPCCLient, err error) {
stats = stats.NewScope("RPC")
hostname, err := os.Hostname()
if err != nil {
return nil, err
}
randID := make([]byte, 3)
_, err = rand.Read(randID)
if err != nil {
return nil, err
}
clientQueue := fmt.Sprintf("%s.%s.%x", clientQueuePrefix, hostname, randID)
reconnectBase := amqpConf.ReconnectTimeouts.Base.Duration
if reconnectBase == 0 {
reconnectBase = 20 * time.Millisecond
}
reconnectMax := amqpConf.ReconnectTimeouts.Max.Duration
if reconnectMax == 0 {
reconnectMax = time.Minute
}
timeout := rpcConf.RPCTimeout.Duration
if timeout == 0 {
timeout = 10 * time.Second
}
rpc = &AmqpRPCCLient{
serverQueue: rpcConf.Server,
clientQueue: clientQueue,
connection: newAMQPConnector(clientQueue, reconnectBase, reconnectMax),
pending: make(map[string]chan []byte),
timeout: timeout,
log: blog.Get(),
stats: stats,
}
err = rpc.connection.connect(amqpConf)
if err != nil {
return nil, err
}
go func() {
for {
select {
case msg, ok := <-rpc.connection.messages():
if ok {
corrID := msg.CorrelationId
rpc.mu.RLock()
responseChan, present := rpc.pending[corrID]
rpc.mu.RUnlock()
if !present {
// occurs when a request is timed out and the arrives
// afterwards
stats.Inc("AfterTimeoutResponseArrivals."+clientQueuePrefix, 1)
continue
}
responseChan <- msg.Body
rpc.mu.Lock()
delete(rpc.pending, corrID)
rpc.mu.Unlock()
} else {
// chan has been closed by rpc.connection.Cancel
rpc.log.Info(fmt.Sprintf(" [!] Client reply channel closed: %s", rpc.clientQueue))
continue
}
case err = <-rpc.connection.closeChannel():
rpc.log.Info(fmt.Sprintf(" [!] Client reply channel closed : %s", rpc.clientQueue))
rpc.connection.reconnect(amqpConf, rpc.log)
}
}
}()
return rpc, err
}
// exchangeOne performs a single DNS exchange with a randomly chosen server
// out of the server list, returning the response, time, and error (if any).
// This method sets the DNSSEC OK bit on the message to true before sending
// it to the resolver in case validation isn't the resolvers default behaviour.
func (dnsResolver *DNSResolverImpl) exchangeOne(ctx context.Context, hostname string, qtype uint16, msgStats metrics.Scope) (*dns.Msg, error) {
m := new(dns.Msg)
// Set question type
m.SetQuestion(dns.Fqdn(hostname), qtype)
// Set DNSSEC OK bit for resolver
m.SetEdns0(4096, true)
if len(dnsResolver.servers) < 1 {
return nil, fmt.Errorf("Not configured with at least one DNS Server")
}
dnsResolver.stats.Inc("Rate", 1)
// Randomly pick a server
chosenServer := dnsResolver.servers[rand.Intn(len(dnsResolver.servers))]
client := dnsResolver.dnsClient
tries := 1
start := dnsResolver.clk.Now()
msgStats.Inc("Calls", 1)
defer func() {
msgStats.TimingDuration("Latency", dnsResolver.clk.Now().Sub(start))
}()
for {
msgStats.Inc("Tries", 1)
ch := make(chan dnsResp, 1)
go func() {
rsp, rtt, err := client.Exchange(m, chosenServer)
msgStats.TimingDuration("SingleTryLatency", rtt)
ch <- dnsResp{m: rsp, err: err}
}()
select {
case <-ctx.Done():
msgStats.Inc("Cancels", 1)
msgStats.Inc("Errors", 1)
return nil, ctx.Err()
case r := <-ch:
if r.err != nil {
msgStats.Inc("Errors", 1)
operr, ok := r.err.(*net.OpError)
isRetryable := ok && operr.Temporary()
hasRetriesLeft := tries < dnsResolver.maxTries
if isRetryable && hasRetriesLeft {
tries++
continue
} else if isRetryable && !hasRetriesLeft {
msgStats.Inc("RanOutOfTries", 1)
}
} else {
msgStats.Inc("Successes", 1)
}
return r.m, r.err
}
}
}
