// LoadCertificateForDomains loads certificates from ACME for given domains
func (a *ACME) LoadCertificateForDomains(domains []string) {
domains = fun.Map(types.CanonicalDomain, domains).([]string)
safe.Go(func() {
operation := func() error {
if a.client == nil {
return fmt.Errorf("ACME client still not built")
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Error getting ACME client: %v, retrying in %s", err, time)
}
ebo := backoff.NewExponentialBackOff()
ebo.MaxElapsedTime = 30 * time.Second
err := backoff.RetryNotify(operation, ebo, notify)
if err != nil {
log.Errorf("Error getting ACME client: %v", err)
return
}
account := a.store.Get().(*Account)
var domain Domain
if len(domains) == 0 {
// no domain
return
} else if len(domains) > 1 {
domain = Domain{Main: domains[0], SANs: domains[1:]}
} else {
domain = Domain{Main: domains[0]}
}
if _, exists := account.DomainsCertificate.exists(domain); exists {
// domain already exists
return
}
certificate, err := a.getDomainsCertificates(domains)
if err != nil {
log.Errorf("Error getting ACME certificates %+v : %v", domains, err)
return
}
log.Debugf("Got certificate for domains %+v", domains)
transaction, object, err := a.store.Begin()
if err != nil {
log.Errorf("Error creating transaction %+v : %v", domains, err)
return
}
account = object.(*Account)
_, err = account.DomainsCertificate.addCertificateForDomains(certificate, domain)
if err != nil {
log.Errorf("Error adding ACME certificates %+v : %v", domains, err)
return
}
if err = transaction.Commit(account); err != nil {
log.Errorf("Error Saving ACME account %+v: %v", account, err)
return
}
})
}
func (server *Server) startProviders() {
// start providers
for _, provider := range server.providers {
jsonConf, _ := json.Marshal(provider)
log.Infof("Starting provider %v %s", reflect.TypeOf(provider), jsonConf)
currentProvider := provider
safe.Go(func() {
err := currentProvider.Provide(server.configurationChan, server.routinesPool, server.globalConfiguration.Constraints)
if err != nil {
log.Errorf("Error starting provider %s", err)
}
})
}
}
func (server *Server) listenProviders(stop chan bool) {
lastReceivedConfiguration := safe.New(time.Unix(0, 0))
lastConfigs := cmap.New()
for {
select {
case <-stop:
return
case configMsg, ok := <-server.configurationChan:
if !ok {
return
}
server.defaultConfigurationValues(configMsg.Configuration)
currentConfigurations := server.currentConfigurations.Get().(configs)
jsonConf, _ := json.Marshal(configMsg.Configuration)
log.Debugf("Configuration received from provider %s: %s", configMsg.ProviderName, string(jsonConf))
if configMsg.Configuration == nil || configMsg.Configuration.Backends == nil && configMsg.Configuration.Frontends == nil {
log.Infof("Skipping empty Configuration for provider %s", configMsg.ProviderName)
} else if reflect.DeepEqual(currentConfigurations[configMsg.ProviderName], configMsg.Configuration) {
log.Infof("Skipping same configuration for provider %s", configMsg.ProviderName)
} else {
lastConfigs.Set(configMsg.ProviderName, &configMsg)
lastReceivedConfigurationValue := lastReceivedConfiguration.Get().(time.Time)
if time.Now().After(lastReceivedConfigurationValue.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Last %s config received more than %s, OK", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration.String())
// last config received more than n s ago
server.configurationValidatedChan <- configMsg
} else {
log.Debugf("Last %s config received less than %s, waiting...", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration.String())
safe.Go(func() {
<-time.After(server.globalConfiguration.ProvidersThrottleDuration)
lastReceivedConfigurationValue := lastReceivedConfiguration.Get().(time.Time)
if time.Now().After(lastReceivedConfigurationValue.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Waited for %s config, OK", configMsg.ProviderName)
if lastConfig, ok := lastConfigs.Get(configMsg.ProviderName); ok {
server.configurationValidatedChan <- *lastConfig.(*types.ConfigMessage)
}
}
})
}
lastReceivedConfiguration.Set(time.Now())
}
}
}
}
func TestKvWatchTree(t *testing.T) {
returnedChans := make(chan chan []*store.KVPair)
provider := &KvMock{
Kv{
kvclient: &Mock{
WatchTreeMethod: func() <-chan []*store.KVPair {
c := make(chan []*store.KVPair, 10)
returnedChans <- c
return c
},
},
},
}
configChan := make(chan types.ConfigMessage)
safe.Go(func() {
provider.watchKv(configChan, "prefix", make(chan bool, 1))
})
select {
case c1 := <-returnedChans:
c1 <- []*store.KVPair{}
<-configChan
close(c1) // WatchTree chans can close due to error
case <-time.After(1 * time.Second):
t.Fatalf("Failed to create a new WatchTree chan")
}
select {
case c2 := <-returnedChans:
c2 <- []*store.KVPair{}
<-configChan
case <-time.After(1 * time.Second):
t.Fatalf("Failed to create a new WatchTree chan")
}
select {
case _ = <-configChan:
t.Fatalf("configChan should be empty")
default:
}
}
func (provider *ConsulCatalog) watchServices(stopCh <-chan struct{}) <-chan map[string][]string {
watchCh := make(chan map[string][]string)
catalog := provider.client.Catalog()
safe.Go(func() {
defer close(watchCh)
opts := &api.QueryOptions{WaitTime: DefaultWatchWaitTime}
for {
select {
case <-stopCh:
return
default:
}
data, meta, err := catalog.Services(opts)
if err != nil {
log.WithError(err).Errorf("Failed to list services")
return
}
// If LastIndex didn't change then it means `Get` returned
// because of the WaitTime and the key didn't changed.
if opts.WaitIndex == meta.LastIndex {
continue
}
opts.WaitIndex = meta.LastIndex
if data != nil {
watchCh <- data
}
}
})
return watchCh
}
func (server *Server) listenProviders(stop chan bool) {
lastReceivedConfiguration := safe.New(time.Unix(0, 0))
lastConfigs := cmap.New()
for {
select {
case <-stop:
return
case configMsg, ok := <-server.configurationChan:
if !ok {
return
}
jsonConf, _ := json.Marshal(configMsg.Configuration)
log.Debugf("Configuration received from provider %s: %s", configMsg.ProviderName, string(jsonConf))
lastConfigs.Set(configMsg.ProviderName, &configMsg)
lastReceivedConfigurationValue := lastReceivedConfiguration.Get().(time.Time)
if time.Now().After(lastReceivedConfigurationValue.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Last %s config received more than %s, OK", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration)
// last config received more than n s ago
server.configurationValidatedChan <- configMsg
} else {
log.Debugf("Last %s config received less than %s, waiting...", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration)
safe.Go(func() {
<-time.After(server.globalConfiguration.ProvidersThrottleDuration)
lastReceivedConfigurationValue := lastReceivedConfiguration.Get().(time.Time)
if time.Now().After(lastReceivedConfigurationValue.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Waited for %s config, OK", configMsg.ProviderName)
if lastConfig, ok := lastConfigs.Get(configMsg.ProviderName); ok {
server.configurationValidatedChan <- *lastConfig.(*types.ConfigMessage)
}
}
})
}
lastReceivedConfiguration.Set(time.Now())
}
}
}
// CreateConfig creates a tls.config from using ACME configuration
func (a *ACME) CreateConfig(tlsConfig *tls.Config, CheckOnDemandDomain func(domain string) bool) error {
acme.Logger = fmtlog.New(ioutil.Discard, "", 0)
if len(a.StorageFile) == 0 {
return errors.New("Empty StorageFile, please provide a filename for certs storage")
}
log.Debugf("Generating default certificate...")
if len(tlsConfig.Certificates) == 0 {
// no certificates in TLS config, so we add a default one
cert, err := generateDefaultCertificate()
if err != nil {
return err
}
tlsConfig.Certificates = append(tlsConfig.Certificates, *cert)
}
var account *Account
var needRegister bool
// if certificates in storage, load them
if fileInfo, err := os.Stat(a.StorageFile); err == nil && fileInfo.Size() != 0 {
log.Infof("Loading ACME certificates...")
// load account
account, err = a.loadAccount(a)
if err != nil {
return err
}
} else {
log.Infof("Generating ACME Account...")
// Create a user. New accounts need an email and private key to start
privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return err
}
account = &Account{
Email: a.Email,
PrivateKey: x509.MarshalPKCS1PrivateKey(privateKey),
}
account.DomainsCertificate = DomainsCertificates{Certs: []*DomainsCertificate{}, lock: &sync.RWMutex{}}
needRegister = true
}
client, err := a.buildACMEClient(account)
if err != nil {
return err
}
client.ExcludeChallenges([]acme.Challenge{acme.HTTP01, acme.DNS01})
wrapperChallengeProvider := newWrapperChallengeProvider()
client.SetChallengeProvider(acme.TLSSNI01, wrapperChallengeProvider)
if needRegister {
// New users will need to register; be sure to save it
reg, err := client.Register()
if err != nil {
return err
}
account.Registration = reg
}
// The client has a URL to the current Let's Encrypt Subscriber
// Agreement. The user will need to agree to it.
err = client.AgreeToTOS()
if err != nil {
return err
}
safe.Go(func() {
a.retrieveCertificates(client, account)
})
tlsConfig.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if challengeCert, ok := wrapperChallengeProvider.getCertificate(clientHello.ServerName); ok {
return challengeCert, nil
}
if domainCert, ok := account.DomainsCertificate.getCertificateForDomain(clientHello.ServerName); ok {
return domainCert.tlsCert, nil
}
if a.OnDemand {
if CheckOnDemandDomain != nil && !CheckOnDemandDomain(clientHello.ServerName) {
return nil, nil
}
return a.loadCertificateOnDemand(client, account, clientHello)
}
return nil, nil
}
ticker := time.NewTicker(24 * time.Hour)
safe.Go(func() {
for {
select {
case <-ticker.C:
if err := a.renewCertificates(client, account); err != nil {
log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
}
}
}
})
return nil
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool) error {
// TODO register this routine in pool, and watch for stop channel
safe.Go(func() {
operation := func() error {
var err error
dockerClient, err := provider.createClient()
if err != nil {
log.Errorf("Failed to create a client for docker, error: %s", err)
return err
}
version, err := dockerClient.ServerVersion(context.Background())
log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion)
containers, err := listContainers(dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
return err
}
configuration := provider.loadDockerConfig(containers)
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
if provider.Watch {
ctx, cancel := context.WithCancel(context.Background())
f := filters.NewArgs()
f.Add("type", "container")
options := dockertypes.EventsOptions{
Filters: f,
}
eventHandler := events.NewHandler(events.ByAction)
startStopHandle := func(m eventtypes.Message) {
log.Debugf("Docker event received %+v", m)
containers, err := listContainers(dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
// Call cancel to get out of the monitor
cancel()
}
configuration := provider.loadDockerConfig(containers)
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
}
}
eventHandler.Handle("start", startStopHandle)
eventHandler.Handle("die", startStopHandle)
errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler)
pool.Go(func(stop chan bool) {
for {
select {
case <-stop:
cancel()
return
}
}
})
if err := <-errChan; err != nil {
return err
}
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to docker server %+v", err)
}
})
return nil
}
func run(traefikConfiguration *TraefikConfiguration) {
fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)
// load global configuration
globalConfiguration := traefikConfiguration.GlobalConfiguration
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = globalConfiguration.MaxIdleConnsPerHost
if globalConfiguration.InsecureSkipVerify {
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
loggerMiddleware := middlewares.NewLogger(globalConfiguration.AccessLogsFile)
defer loggerMiddleware.Close()
if globalConfiguration.File != nil && len(globalConfiguration.File.Filename) == 0 {
// no filename, setting to global config file
if len(traefikConfiguration.ConfigFile) != 0 {
globalConfiguration.File.Filename = traefikConfiguration.ConfigFile
} else {
log.Errorln("Error using file configuration backend, no filename defined")
}
}
if len(globalConfiguration.EntryPoints) == 0 {
globalConfiguration.EntryPoints = map[string]*EntryPoint{"http": {Address: ":80"}}
globalConfiguration.DefaultEntryPoints = []string{"http"}
}
if globalConfiguration.Debug {
globalConfiguration.LogLevel = "DEBUG"
}
// logging
level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
if err != nil {
log.Error("Error getting level", err)
}
log.SetLevel(level)
if len(globalConfiguration.TraefikLogsFile) > 0 {
fi, err := os.OpenFile(globalConfiguration.TraefikLogsFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
defer func() {
if err := fi.Close(); err != nil {
log.Error("Error closing file", err)
}
}()
if err != nil {
log.Error("Error opening file", err)
} else {
log.SetOutput(fi)
log.SetFormatter(&logrus.TextFormatter{DisableColors: true, FullTimestamp: true, DisableSorting: true})
}
} else {
log.SetFormatter(&logrus.TextFormatter{FullTimestamp: true, DisableSorting: true})
}
jsonConf, _ := json.Marshal(globalConfiguration)
log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)
if globalConfiguration.CheckNewVersion {
ticker := time.NewTicker(24 * time.Hour)
safe.Go(func() {
version.CheckNewVersion()
for {
select {
case <-ticker.C:
version.CheckNewVersion()
}
}
})
}
if len(traefikConfiguration.ConfigFile) != 0 {
log.Infof("Using TOML configuration file %s", traefikConfiguration.ConfigFile)
}
log.Debugf("Global configuration loaded %s", string(jsonConf))
server := NewServer(globalConfiguration)
server.Start()
defer server.Close()
sent, err := daemon.SdNotify("READY=1")
if !sent && err != nil {
log.Error("Fail to notify", err)
}
server.Wait()
log.Info("Shutting down")
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
provider.Constraints = append(provider.Constraints, constraints...)
// TODO register this routine in pool, and watch for stop channel
safe.Go(func() {
operation := func() error {
var err error
dockerClient, err := provider.createClient()
if err != nil {
log.Errorf("Failed to create a client for docker, error: %s", err)
return err
}
ctx := context.Background()
version, err := dockerClient.ServerVersion(ctx)
log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion)
var dockerDataList []dockerData
if provider.SwarmMode {
dockerDataList, err = listServices(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list services for docker swarm mode, error %s", err)
return err
}
} else {
dockerDataList, err = listContainers(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
return err
}
}
configuration := provider.loadDockerConfig(dockerDataList)
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
if provider.Watch {
ctx, cancel := context.WithCancel(ctx)
if provider.SwarmMode {
// TODO: This need to be change. Linked to Swarm events docker/docker#23827
ticker := time.NewTicker(SwarmDefaultWatchTime)
pool.Go(func(stop chan bool) {
for {
select {
case <-ticker.C:
services, err := listServices(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list services for docker, error %s", err)
return
}
configuration := provider.loadDockerConfig(services)
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
}
case <-stop:
ticker.Stop()
cancel()
return
}
}
})
} else {
pool.Go(func(stop chan bool) {
for {
select {
case <-stop:
cancel()
return
}
}
})
f := filters.NewArgs()
f.Add("type", "container")
options := dockertypes.EventsOptions{
Filters: f,
}
eventHandler := events.NewHandler(events.ByAction)
startStopHandle := func(m eventtypes.Message) {
log.Debugf("Docker event received %+v", m)
containers, err := listContainers(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
// Call cancel to get out of the monitor
cancel()
return
}
configuration := provider.loadDockerConfig(containers)
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
}
}
eventHandler.Handle("start", startStopHandle)
eventHandler.Handle("die", startStopHandle)
eventHandler.Handle("health_status: healthy", startStopHandle)
eventHandler.Handle("health_status: unhealthy", startStopHandle)
eventHandler.Handle("health_status: starting", startStopHandle)
errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler)
if err := <-errChan; err != nil {
return err
}
}
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to docker server %+v", err)
}
})
return nil
}
// CreateLocalConfig creates a tls.config using local ACME configuration
func (a *ACME) CreateLocalConfig(tlsConfig *tls.Config, checkOnDemandDomain func(domain string) bool) error {
err := a.init()
if err != nil {
return err
}
if len(a.Storage) == 0 {
return errors.New("Empty Store, please provide a filename for certs storage")
}
a.checkOnDemandDomain = checkOnDemandDomain
tlsConfig.Certificates = append(tlsConfig.Certificates, *a.defaultCertificate)
tlsConfig.GetCertificate = a.getCertificate
localStore := NewLocalStore(a.Storage)
a.store = localStore
a.challengeProvider = &challengeProvider{store: a.store}
var needRegister bool
var account *Account
if fileInfo, fileErr := os.Stat(a.Storage); fileErr == nil && fileInfo.Size() != 0 {
log.Infof("Loading ACME Account...")
// load account
object, err := localStore.Load()
if err != nil {
return err
}
account = object.(*Account)
} else {
log.Infof("Generating ACME Account...")
account, err = NewAccount(a.Email)
if err != nil {
return err
}
needRegister = true
}
a.client, err = a.buildACMEClient(account)
if err != nil {
return err
}
if needRegister {
// New users will need to register; be sure to save it
log.Infof("Register...")
reg, err := a.client.Register()
if err != nil {
return err
}
account.Registration = reg
}
// The client has a URL to the current Let's Encrypt Subscriber
// Agreement. The user will need to agree to it.
log.Debugf("AgreeToTOS...")
err = a.client.AgreeToTOS()
if err != nil {
// Let's Encrypt Subscriber Agreement renew ?
reg, err := a.client.QueryRegistration()
if err != nil {
return err
}
account.Registration = reg
err = a.client.AgreeToTOS()
if err != nil {
log.Errorf("Error sending ACME agreement to TOS: %+v: %s", account, err.Error())
}
}
// save account
transaction, _, err := a.store.Begin()
if err != nil {
return err
}
err = transaction.Commit(account)
if err != nil {
return err
}
safe.Go(func() {
a.retrieveCertificates()
if err := a.renewCertificates(); err != nil {
log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
}
})
ticker := time.NewTicker(24 * time.Hour)
safe.Go(func() {
for range ticker.C {
if err := a.renewCertificates(); err != nil {
log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
}
}
})
return nil
}
// CreateClusterConfig creates a tls.config using ACME configuration in cluster mode
func (a *ACME) CreateClusterConfig(leadership *cluster.Leadership, tlsConfig *tls.Config, checkOnDemandDomain func(domain string) bool) error {
err := a.init()
if err != nil {
return err
}
if len(a.Storage) == 0 {
return errors.New("Empty Store, please provide a key for certs storage")
}
a.checkOnDemandDomain = checkOnDemandDomain
tlsConfig.Certificates = append(tlsConfig.Certificates, *a.defaultCertificate)
tlsConfig.GetCertificate = a.getCertificate
listener := func(object cluster.Object) error {
account := object.(*Account)
account.Init()
if !leadership.IsLeader() {
a.client, err = a.buildACMEClient(account)
if err != nil {
log.Errorf("Error building ACME client %+v: %s", object, err.Error())
}
}
return nil
}
datastore, err := cluster.NewDataStore(
leadership.Pool.Ctx(),
staert.KvSource{
Store: leadership.Store,
Prefix: a.Storage,
},
&Account{},
listener)
if err != nil {
return err
}
a.store = datastore
a.challengeProvider = &challengeProvider{store: a.store}
ticker := time.NewTicker(24 * time.Hour)
leadership.Pool.AddGoCtx(func(ctx context.Context) {
log.Infof("Starting ACME renew job...")
defer log.Infof("Stopped ACME renew job...")
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := a.renewCertificates(); err != nil {
log.Errorf("Error renewing ACME certificate: %s", err.Error())
}
}
}
})
leadership.AddListener(func(elected bool) error {
if elected {
object, err := a.store.Load()
if err != nil {
return err
}
transaction, object, err := a.store.Begin()
if err != nil {
return err
}
account := object.(*Account)
account.Init()
var needRegister bool
if account == nil || len(account.Email) == 0 {
account, err = NewAccount(a.Email)
if err != nil {
return err
}
needRegister = true
}
if err != nil {
return err
}
a.client, err = a.buildACMEClient(account)
if err != nil {
return err
}
if needRegister {
// New users will need to register; be sure to save it
log.Debugf("Register...")
reg, err := a.client.Register()
if err != nil {
return err
}
account.Registration = reg
}
// The client has a URL to the current Let's Encrypt Subscriber
// Agreement. The user will need to agree to it.
log.Debugf("AgreeToTOS...")
err = a.client.AgreeToTOS()
if err != nil {
// Let's Encrypt Subscriber Agreement renew ?
reg, err := a.client.QueryRegistration()
if err != nil {
return err
}
account.Registration = reg
err = a.client.AgreeToTOS()
if err != nil {
log.Errorf("Error sending ACME agreement to TOS: %+v: %s", account, err.Error())
}
}
err = transaction.Commit(account)
if err != nil {
return err
}
safe.Go(func() {
a.retrieveCertificates()
if err := a.renewCertificates(); err != nil {
log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
}
})
}
return nil
})
return nil
}