// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
config := api.DefaultConfig()
config.Address = provider.Endpoint
client, err := api.NewClient(config)
if err != nil {
return err
}
provider.client = client
provider.Constraints = append(provider.Constraints, constraints...)
pool.Go(func(stop chan bool) {
notify := func(err error, time time.Duration) {
log.Errorf("Consul connection error %+v, retrying in %s", err, time)
}
operation := func() error {
return provider.watch(configurationChan, stop)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to consul server %+v", err)
}
})
return err
}
func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
provider.Constraints = append(provider.Constraints, constraints...)
operation := func() error {
if _, err := provider.kvclient.Exists("qmslkjdfmqlskdjfmqlksjazçueznbvbwzlkajzebvkwjdcqmlsfj"); err != nil {
return fmt.Errorf("Failed to test KV store connection: %v", err)
}
if provider.Watch {
pool.Go(func(stop chan bool) {
err := provider.watchKv(configurationChan, provider.Prefix, stop)
if err != nil {
log.Errorf("Cannot watch KV store: %v", err)
}
})
}
configuration := provider.loadConfig()
configurationChan <- types.ConfigMessage{
ProviderName: string(provider.storeType),
Configuration: configuration,
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}
return nil
}
// listenForNodeChanges listens for changes to node status using change feeds.
// This function will block until the query fails
func (c *Cluster) listenForNodeChanges() error {
// Start listening to changes from a random active node
node, hpr, err := c.GetNextNode()
if err != nil {
return err
}
q, err := newQuery(
DB("rethinkdb").Table("server_status").Changes(),
map[string]interface{}{},
c.opts,
)
if err != nil {
return fmt.Errorf("Error building query: %s", err)
}
cursor, err := node.Query(q)
if err != nil {
hpr.Mark(err)
return err
}
// Keep reading node status updates from changefeed
var result struct {
NewVal nodeStatus `gorethink:"new_val"`
OldVal nodeStatus `gorethink:"old_val"`
}
for cursor.Next(&result) {
addr := fmt.Sprintf("%s:%d", result.NewVal.Network.Hostname, result.NewVal.Network.ReqlPort)
addr = strings.ToLower(addr)
switch result.NewVal.Status {
case "connected":
// Connect to node using exponential backoff (give up after waiting 5s)
// to give the node time to start-up.
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = time.Second * 5
backoff.Retry(func() error {
node, err := c.connectNodeWithStatus(result.NewVal)
if err == nil {
if !c.nodeExists(node) {
c.addNode(node)
Log.WithFields(logrus.Fields{
"id": node.ID,
"host": node.Host.String(),
}).Debug("Connected to node")
}
}
return err
}, b)
}
}
err = cursor.Err()
hpr.Mark(err)
return err
}
func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) error {
operation := func() error {
events, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{}))
if err != nil {
return fmt.Errorf("Failed to KV WatchTree: %v", err)
}
for {
select {
case <-stop:
return nil
case _, ok := <-events:
if !ok {
return errors.New("watchtree channel closed")
}
configuration := provider.loadConfig()
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: string(provider.storeType),
Configuration: configuration,
}
}
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
return fmt.Errorf("Cannot connect to KV server: %v", err)
}
return nil
}
func newConfiguredStorage(newLog spec.Log, storageType, storagePrefix, storageAddr string) (spec.Storage, error) {
var newStorage spec.Storage
var err error
switch storageType {
case "redis":
newStorageConfig := redis.DefaultStorageConfigWithAddr(storageAddr)
newStorageConfig.BackoffFactory = func() spec.Backoff {
return backoff.NewExponentialBackOff()
}
newStorageConfig.Instrumentation, err = newPrometheusInstrumentation([]string{"Feature", "Storage", "Redis"})
if err != nil {
return nil, maskAny(err)
}
newStorageConfig.Log = newLog
newStorageConfig.Prefix = storagePrefix
newStorage, err = redis.NewStorage(newStorageConfig)
if err != nil {
return nil, maskAny(err)
}
case "memory":
newStorage, err = memory.NewStorage(memory.DefaultStorageConfig())
if err != nil {
return nil, maskAny(err)
}
default:
return nil, maskAnyf(invalidStorageFlagError, "%s", storageType)
}
return newStorage, nil
}
// NewBreakerWithOptions creates a base breaker with a specified backoff, clock and TripFunc
func NewBreakerWithOptions(options *Options) *Breaker {
if options == nil {
options = &Options{}
}
if options.Clock == nil {
options.Clock = clock.New()
}
if options.BackOff == nil {
b := backoff.NewExponentialBackOff()
b.InitialInterval = defaultInitialBackOffInterval
b.MaxElapsedTime = defaultBackoffMaxElapsedTime
b.Clock = options.Clock
b.Reset()
options.BackOff = b
}
if options.WindowTime == 0 {
options.WindowTime = DefaultWindowTime
}
if options.WindowBuckets == 0 {
options.WindowBuckets = DefaultWindowBuckets
}
return &Breaker{
BackOff: options.BackOff,
Clock: options.Clock,
ShouldTrip: options.ShouldTrip,
nextBackOff: options.BackOff.NextBackOff(),
counts: newWindow(options.WindowTime, options.WindowBuckets),
}
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
k8sClient, err := provider.newK8sClient()
if err != nil {
return err
}
provider.Constraints = append(provider.Constraints, constraints...)
pool.Go(func(stop chan bool) {
operation := func() error {
for {
stopWatch := make(chan struct{}, 1)
defer close(stopWatch)
log.Debugf("Using label selector: '%s'", provider.LabelSelector)
eventsChan, err := k8sClient.WatchAll(provider.LabelSelector, stopWatch)
if err != nil {
log.Errorf("Error watching kubernetes events: %v", err)
timer := time.NewTimer(1 * time.Second)
select {
case <-timer.C:
return err
case <-stop:
return nil
}
}
for {
select {
case <-stop:
return nil
case event := <-eventsChan:
log.Debugf("Received event from kubernetes %+v", event)
templateObjects, err := provider.loadIngresses(k8sClient)
if err != nil {
return err
}
if reflect.DeepEqual(provider.lastConfiguration.Get(), templateObjects) {
log.Debugf("Skipping event from kubernetes %+v", event)
} else {
provider.lastConfiguration.Set(templateObjects)
configurationChan <- types.ConfigMessage{
ProviderName: "kubernetes",
Configuration: provider.loadConfig(*templateObjects),
}
}
}
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to Kubernetes server %+v", err)
}
})
return nil
}
// LoadCertificateForDomains loads certificates from ACME for given domains
func (a *ACME) LoadCertificateForDomains(domains []string) {
domains = fun.Map(types.CanonicalDomain, domains).([]string)
safe.Go(func() {
operation := func() error {
if a.client == nil {
return fmt.Errorf("ACME client still not built")
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Error getting ACME client: %v, retrying in %s", err, time)
}
ebo := backoff.NewExponentialBackOff()
ebo.MaxElapsedTime = 30 * time.Second
err := backoff.RetryNotify(operation, ebo, notify)
if err != nil {
log.Errorf("Error getting ACME client: %v", err)
return
}
account := a.store.Get().(*Account)
var domain Domain
if len(domains) == 0 {
// no domain
return
} else if len(domains) > 1 {
domain = Domain{Main: domains[0], SANs: domains[1:]}
} else {
domain = Domain{Main: domains[0]}
}
if _, exists := account.DomainsCertificate.exists(domain); exists {
// domain already exists
return
}
certificate, err := a.getDomainsCertificates(domains)
if err != nil {
log.Errorf("Error getting ACME certificates %+v : %v", domains, err)
return
}
log.Debugf("Got certificate for domains %+v", domains)
transaction, object, err := a.store.Begin()
if err != nil {
log.Errorf("Error creating transaction %+v : %v", domains, err)
return
}
account = object.(*Account)
_, err = account.DomainsCertificate.addCertificateForDomains(certificate, domain)
if err != nil {
log.Errorf("Error adding ACME certificates %+v : %v", domains, err)
return
}
if err = transaction.Commit(account); err != nil {
log.Errorf("Error Saving ACME account %+v: %v", account, err)
return
}
})
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Eureka) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, _ types.Constraints) error {
operation := func() error {
configuration, err := provider.buildConfiguration()
if err != nil {
log.Errorf("Failed to build configuration for Eureka, error: %s", err)
return err
}
configurationChan <- types.ConfigMessage{
ProviderName: "eureka",
Configuration: configuration,
}
var delay time.Duration
if len(provider.Delay) > 0 {
var err error
delay, err = time.ParseDuration(provider.Delay)
if err != nil {
log.Errorf("Failed to parse delay for Eureka, error: %s", err)
return err
}
} else {
delay = time.Second * 30
}
ticker := time.NewTicker(delay)
go func() {
for t := range ticker.C {
log.Debug("Refreshing Eureka " + t.String())
configuration, err := provider.buildConfiguration()
if err != nil {
log.Errorf("Failed to refresh Eureka configuration, error: %s", err)
return
}
configurationChan <- types.ConfigMessage{
ProviderName: "eureka",
Configuration: configuration,
}
}
}()
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Eureka connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to Eureka server %+v", err)
return err
}
return nil
}
// Begin creates a transaction with the KV store.
func (d *Datastore) Begin() (Transaction, Object, error) {
id := uuid.NewV4().String()
log.Debugf("Transaction %s begins", id)
remoteLock, err := d.kv.NewLock(d.lockKey, &store.LockOptions{TTL: 20 * time.Second, Value: []byte(id)})
if err != nil {
return nil, nil, err
}
stopCh := make(chan struct{})
ctx, cancel := context.WithCancel(d.ctx)
var errLock error
go func() {
_, errLock = remoteLock.Lock(stopCh)
cancel()
}()
select {
case <-ctx.Done():
if errLock != nil {
return nil, nil, errLock
}
case <-d.ctx.Done():
stopCh <- struct{}{}
return nil, nil, d.ctx.Err()
}
// we got the lock! Now make sure we are synced with KV store
operation := func() error {
meta := d.get()
if meta.Lock != id {
return fmt.Errorf("Object lock value: expected %s, got %s", id, meta.Lock)
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Datastore sync error: %v, retrying in %s", err, time)
err = d.reload()
if err != nil {
log.Errorf("Error reloading: %+v", err)
}
}
ebo := backoff.NewExponentialBackOff()
ebo.MaxElapsedTime = 60 * time.Second
err = backoff.RetryNotify(operation, ebo, notify)
if err != nil {
return nil, nil, fmt.Errorf("Datastore cannot sync: %v", err)
}
// we synced with KV store, we can now return Setter
return &datastoreTransaction{
Datastore: d,
remoteLock: remoteLock,
id: id,
}, d.meta.object, nil
}
func newRandomService() (servicespec.Random, error) {
newConfig := random.DefaultConfig()
newConfig.BackoffFactory = func() spec.Backoff {
return backoff.NewExponentialBackOff()
}
newService, err := random.New(newConfig)
if err != nil {
return nil, maskAny(err)
}
return newService, nil
}
func connectSlackRTM(h *helios.Engine, s *SlackService) {
err := backoff.Retry(func() error {
h.Debug("Connecting to slack rtm")
err := runSlackRTM(h, s)
if err != nil {
h.Warn("Failed to start slack rtm. Retrying.", "error", err.Error())
}
return err
}, backoff.NewExponentialBackOff())
if err != nil {
s.Messages <- helios.NewError("Slack service error: %v", err)
}
}
// TryRequest try operation timeout, and retry backoff
func TryRequest(url string, timeout time.Duration, condition Condition) error {
exponentialBackOff := backoff.NewExponentialBackOff()
exponentialBackOff.MaxElapsedTime = timeout
var res *http.Response
err := backoff.Retry(func() error {
var err error
res, err = http.Get(url)
if err != nil {
return err
}
return condition(res)
}, exponentialBackOff)
return err
}
// Participate tries to be a leader
func (l *Leadership) Participate(pool *safe.Pool) {
pool.GoCtx(func(ctx context.Context) {
log.Debugf("Node %s running for election", l.Cluster.Node)
defer log.Debugf("Node %s no more running for election", l.Cluster.Node)
backOff := backoff.NewExponentialBackOff()
operation := func() error {
return l.run(ctx, l.candidate)
}
notify := func(err error, time time.Duration) {
log.Errorf("Leadership election error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backOff, notify)
if err != nil {
log.Errorf("Cannot elect leadership %+v", err)
}
})
}
// NewStorage creates a new configured memory storage object. Therefore it
// manages an in-memory redis instance which can be shut down using the
// configured closer. This is used for local development.
func NewStorage(config StorageConfig) (spec.Storage, error) {
addrChan := make(chan string, 1)
closer := make(chan struct{}, 1)
redisAddr := ""
go func() {
s, err := miniredis.Run()
if err != nil {
panic(err)
}
addrChan <- s.Addr()
<-closer
s.Close()
}()
select {
case <-time.After(1 * time.Second):
panic("starting miniredis timed out")
case addr := <-addrChan:
redisAddr = addr
}
newRedisStorageConfig := redis.DefaultStorageConfigWithAddr(redisAddr)
newRedisStorageConfig.BackoffFactory = func() spec.Backoff {
return backoff.NewExponentialBackOff()
}
newRedisStorage, err := redis.NewStorage(newRedisStorageConfig)
if err != nil {
return nil, maskAny(err)
}
newStorage := &storage{
StorageConfig: config,
Closer: closer,
ID: id.MustNewID(),
RedisStorage: newRedisStorage,
ShutdownOnce: sync.Once{},
Type: ObjectType,
}
return newStorage, nil
}
func (d *Datastore) watchChanges() error {
stopCh := make(chan struct{})
kvCh, err := d.kv.Watch(d.lockKey, stopCh)
if err != nil {
return err
}
go func() {
ctx, cancel := context.WithCancel(d.ctx)
operation := func() error {
for {
select {
case <-ctx.Done():
stopCh <- struct{}{}
return nil
case _, ok := <-kvCh:
if !ok {
cancel()
return err
}
err = d.reload()
if err != nil {
return err
}
// log.Debugf("Datastore object change received: %+v", d.meta)
if d.listener != nil {
err := d.listener(d.meta.object)
if err != nil {
log.Errorf("Error calling datastore listener: %s", err)
}
}
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Error in watch datastore: %v", err)
}
}()
return nil
}
// discover attempts to find new nodes in the cluster using the current nodes
func (c *Cluster) discover() {
// Keep retrying with exponential backoff.
b := backoff.NewExponentialBackOff()
// Never finish retrying (max interval is still 60s)
b.MaxElapsedTime = 0
// Keep trying to discover new nodes
for {
backoff.RetryNotify(func() error {
// If no hosts try seeding nodes
if len(c.GetNodes()) == 0 {
c.connectNodes(c.getSeeds())
}
return c.listenForNodeChanges()
}, b, func(err error, wait time.Duration) {
Log.Debugf("Error discovering hosts %s, waiting: %s", err, wait)
})
}
}
func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate, exists bool) {
log.Debugf("Challenge GetCertificate %s", domain)
if !strings.HasSuffix(domain, ".acme.invalid") {
return nil, false
}
c.lock.RLock()
defer c.lock.RUnlock()
account := c.store.Get().(*Account)
if account.ChallengeCerts == nil {
return nil, false
}
account.Init()
var result *tls.Certificate
operation := func() error {
for _, cert := range account.ChallengeCerts {
for _, dns := range cert.certificate.Leaf.DNSNames {
if domain == dns {
result = cert.certificate
return nil
}
}
}
return fmt.Errorf("Cannot find challenge cert for domain %s", domain)
}
notify := func(err error, time time.Duration) {
log.Errorf("Error getting cert: %v, retrying in %s", err, time)
}
ebo := backoff.NewExponentialBackOff()
ebo.MaxElapsedTime = 60 * time.Second
err := backoff.RetryNotify(operation, ebo, notify)
if err != nil {
log.Errorf("Error getting cert: %v", err)
return nil, false
}
return result, true
}
func TestJobBackOff(t *testing.T) {
var (
testInitialInterval = 500 * time.Millisecond
testRandomizationFactor = 0.1
testMultiplier = 2.0
testMaxInterval = 5 * time.Second
testMinJobInterval = 1 * time.Second
)
exp := NewBackOff(backoff.NewExponentialBackOff())
exp.InitialInterval = testInitialInterval
exp.RandomizationFactor = testRandomizationFactor
exp.Multiplier = testMultiplier
exp.MaxInterval = testMaxInterval
exp.MinJobInterval = testMinJobInterval
exp.Reset()
var expectedResults = []time.Duration{500, 500, 500, 1000, 2000, 4000, 5000, 5000, 500, 1000, 2000, 4000, 5000, 5000}
for i, d := range expectedResults {
expectedResults[i] = d * time.Millisecond
}
for i, expected := range expectedResults {
// Assert that the next backoff falls in the expected range.
var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected))
var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected))
if i < 3 || i == 8 {
time.Sleep(2 * time.Second)
}
var actualInterval = exp.NextBackOff()
if !(minInterval <= actualInterval && actualInterval <= maxInterval) {
t.Error("error")
}
// assertEquals(t, expected, exp.currentInterval)
}
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
provider.Constraints = append(provider.Constraints, constraints...)
// TODO register this routine in pool, and watch for stop channel
safe.Go(func() {
operation := func() error {
var err error
dockerClient, err := provider.createClient()
if err != nil {
log.Errorf("Failed to create a client for docker, error: %s", err)
return err
}
ctx := context.Background()
version, err := dockerClient.ServerVersion(ctx)
log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion)
var dockerDataList []dockerData
if provider.SwarmMode {
dockerDataList, err = listServices(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list services for docker swarm mode, error %s", err)
return err
}
} else {
dockerDataList, err = listContainers(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
return err
}
}
configuration := provider.loadDockerConfig(dockerDataList)
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
if provider.Watch {
ctx, cancel := context.WithCancel(ctx)
if provider.SwarmMode {
// TODO: This need to be change. Linked to Swarm events docker/docker#23827
ticker := time.NewTicker(SwarmDefaultWatchTime)
pool.Go(func(stop chan bool) {
for {
select {
case <-ticker.C:
services, err := listServices(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list services for docker, error %s", err)
return
}
configuration := provider.loadDockerConfig(services)
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
}
case <-stop:
ticker.Stop()
cancel()
return
}
}
})
} else {
pool.Go(func(stop chan bool) {
for {
select {
case <-stop:
cancel()
return
}
}
})
f := filters.NewArgs()
f.Add("type", "container")
options := dockertypes.EventsOptions{
Filters: f,
}
eventHandler := events.NewHandler(events.ByAction)
startStopHandle := func(m eventtypes.Message) {
log.Debugf("Docker event received %+v", m)
containers, err := listContainers(ctx, dockerClient)
if err != nil {
log.Errorf("Failed to list containers for docker, error %s", err)
// Call cancel to get out of the monitor
cancel()
return
}
configuration := provider.loadDockerConfig(containers)
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "docker",
Configuration: configuration,
}
}
}
eventHandler.Handle("start", startStopHandle)
eventHandler.Handle("die", startStopHandle)
eventHandler.Handle("health_status: healthy", startStopHandle)
eventHandler.Handle("health_status: unhealthy", startStopHandle)
eventHandler.Handle("health_status: starting", startStopHandle)
errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler)
if err := <-errChan; err != nil {
return err
}
}
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to docker server %+v", err)
}
})
return nil
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
provider.Constraints = append(provider.Constraints, constraints...)
operation := func() error {
config := marathon.NewDefaultConfig()
config.URL = provider.Endpoint
config.EventsTransport = marathon.EventsTransportSSE
if provider.Basic != nil {
config.HTTPBasicAuthUser = provider.Basic.HTTPBasicAuthUser
config.HTTPBasicPassword = provider.Basic.HTTPBasicPassword
}
if len(provider.DCOSToken) > 0 {
config.DCOSToken = provider.DCOSToken
}
TLSConfig, err := provider.TLS.CreateTLSConfig()
if err != nil {
return err
}
config.HTTPClient = &http.Client{
Transport: &http.Transport{
DialContext: (&net.Dialer{
KeepAlive: provider.KeepAlive * time.Second,
Timeout: time.Second * provider.DialerTimeout,
}).DialContext,
TLSClientConfig: TLSConfig,
},
}
client, err := marathon.NewClient(config)
if err != nil {
log.Errorf("Failed to create a client for marathon, error: %s", err)
return err
}
provider.marathonClient = client
update := make(marathon.EventsChannel, 5)
if provider.Watch {
if err := client.AddEventsListener(update, marathon.EventIDApplications); err != nil {
log.Errorf("Failed to register for events, %s", err)
return err
}
pool.Go(func(stop chan bool) {
defer close(update)
for {
select {
case <-stop:
return
case event := <-update:
log.Debug("Marathon event receveived", event)
configuration := provider.loadMarathonConfig()
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "marathon",
Configuration: configuration,
}
}
}
}
})
}
configuration := provider.loadMarathonConfig()
configurationChan <- types.ConfigMessage{
ProviderName: "marathon",
Configuration: configuration,
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("Marathon connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to Marathon server %+v", err)
}
return nil
}
func (p DeployMarathon) Install(data manifest.Manifest) error {
marathonApi, err := MarathonClient(data.GetString("marathon-address"))
if err != nil {
return err
}
fullName := data.GetString("app-name")
bs, bf, bmax, grace := 5.0, 2.0, 120.0, 30.0
app := &marathon.Application{
BackoffSeconds: &bs,
BackoffFactor: &bf,
MaxLaunchDelaySeconds: &bmax,
TaskKillGracePeriodSeconds: &grace,
UpgradeStrategy: &marathon.UpgradeStrategy{
MinimumHealthCapacity: 0.5, // rolling update
MaximumOverCapacity: 0.0,
},
}
portArgs := ""
if port := data.GetString("listen-port"); port != "" {
portArgs = "--port " + port
}
app.Name(fullName)
app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd")))
app.Count(data.GetInt("instances"))
app.Memory(float64(data.GetInt("mem")))
if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil {
app.CPU(cpu)
}
if cluster := data.GetString("cluster"); cluster != "" {
cs := strings.SplitN(cluster, ":", 2)
app.AddConstraint(cs[0], "CLUSTER", cs[1])
app.AddLabel(cs[0], cs[1])
}
for _, cons := range data.GetArray("constraints") {
if consArr, ok := cons.Unwrap().([]interface{}); ok {
consStrings := make([]string, len(consArr))
for i, c := range consArr {
consStrings[i] = fmt.Sprintf("%s", c)
}
app.AddConstraint(consStrings...)
}
}
for _, port := range data.GetArray("ports") {
app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0)))
}
app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app
for k, v := range data.GetMap("environment") {
app.AddEnv(k, fmt.Sprintf("%s", v.Unwrap()))
}
app.AddUris(data.GetString("package-uri"))
// todo: в манифесте задавать массив healthchecks, их использовтаь в марафоне и консул-супервизоре
// todo: открыть сетевой доступ от марафона до мезос-агентов, чтобы марафон мог хелсчеки посылать
//if portArgs != "" {
// health := marathon.NewDefaultHealthCheck()
// health.Protocol = "TCP"
// health.IntervalSeconds = 5
// *health.PortIndex = 0
// app.AddHealthCheck(*health)
//}
if _, err := marathonApi.UpdateApplication(app, false); err != nil {
color.Yellow("marathon <- %s", app)
return err
}
color.Green("marathon <- %s", app)
consulApi, err := utils.ConsulClient(data.GetString("consul-address"))
if err != nil {
return err
}
if err := utils.RegisterPluginData("deploy.marathon", data.GetString("app-name"), data.String(), data.GetString("consul-address")); err != nil {
return err
}
return backoff.Retry(func() error {
services, _, err := consulApi.Health().Service(fullName, "", true, nil)
if err != nil {
log.Println(color.RedString("Error in check health in consul: %v", err))
return err
}
if len(services) == 0 {
log.Printf("Service `%s` not started yet! Retry...", fullName)
return fmt.Errorf("Service `%s` not started!", fullName)
}
log.Println(color.GreenString("Service `%s` successfully started!", fullName))
return nil
}, backoff.NewExponentialBackOff())
}
func (p ReleaseHttp) Run(data manifest.Manifest) error {
if !data.Has("routes") {
log.Println("No routes configured for release.")
return nil
}
consul, err := utils.ConsulClient(data.GetString("consul-address"))
if err != nil {
return err
}
fullName := data.GetString("full-name")
// check current service is alive and healthy
if err := backoff.Retry(func() error {
services, _, err := consul.Health().Service(fullName, "", true, nil)
if err != nil {
log.Println(color.RedString("Error in check health in consul: %v", err))
return err
}
if len(services) == 0 {
log.Printf("Service `%s` not started yet! Retry...", fullName)
return fmt.Errorf("Service `%s` not started!", fullName)
} else {
log.Printf("Service `%s` started with %v instances.", fullName, len(services))
return nil
}
}, backoff.NewExponentialBackOff()); err != nil {
return err
}
routeVars := make(map[string]string, 0)
if data.Has("route") {
if err := json.Unmarshal([]byte(data.GetString("route")), &routeVars); err != nil {
log.Println(color.RedString("Error parse route json: %v, %s", err, data.GetString("route")))
return err
}
}
// collect routes
routes := make([]map[string]string, 0)
for _, route := range data.GetArray("routes") {
if !route.Has("host") {
log.Printf("Not found 'host': %s, skip...", route.String())
continue
}
fields := make(map[string]string)
for k, v := range route.Unwrap().(map[string]interface{}) {
fields[k] = fmt.Sprintf("%v", v)
}
routes = append(routes, utils.MergeMaps(fields, routeVars))
}
if len(routes) == 0 {
log.Println("No routes configured for release.")
return nil
}
routesJson, err := json.MarshalIndent(routes, "", " ")
if err != nil {
return err
}
// write routes to consul kv
if err := utils.PutConsulKv(consul, "services/routes/"+fullName, string(routesJson)); err != nil {
return err
}
log.Println(color.GreenString("Service `%s` released with routes: %s", fullName, string(routesJson)))
// find old services with the same routes
existsRoutes, err := utils.ListConsulKv(consul, "services/routes/"+data.GetString("name-prefix"), nil)
if err != nil {
return err
}
for _, existsRoute := range existsRoutes {
if existsRoute.Key != fmt.Sprintf("services/routes/%s", fullName) { // skip current service
oldRoutes := make([]map[string]string, 0)
if err := json.Unmarshal(existsRoute.Value, &oldRoutes); err != nil {
return err
}
OuterLoop:
for _, route := range routes {
for _, oldRoute := range oldRoutes {
if utils.MapsEqual(route, oldRoute) {
outdated := strings.TrimPrefix(existsRoute.Key, "services/routes/")
log.Println(color.GreenString("Found %s with the same routes %v. Remove it!", outdated, string(existsRoute.Value)))
if err := utils.DelConsulKv(consul, existsRoute.Key); err != nil {
return err
}
if err := utils.MarkAsOutdated(consul, outdated, 10*time.Minute); err != nil {
return err
}
break OuterLoop
}
}
}
}
}
return nil
}
// Try try operation timeout, and retry backoff
func Try(timeout time.Duration, operation func() error) error {
exponentialBackOff := backoff.NewExponentialBackOff()
exponentialBackOff.MaxElapsedTime = timeout
err := backoff.Retry(operation, exponentialBackOff)
return err
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
operation := func() error {
// initialize logging
logging.SetupLogs()
log.Debugf("%s", provider.IPSources)
var zk string
var masters []string
if strings.HasPrefix(provider.Endpoint, "zk://") {
zk = provider.Endpoint
} else {
masters = strings.Split(provider.Endpoint, ",")
}
errch := make(chan error)
changed := detectMasters(zk, masters)
reload := time.NewTicker(time.Second * time.Duration(provider.RefreshSeconds))
zkTimeout := time.Second * time.Duration(provider.ZkDetectionTimeout)
timeout := time.AfterFunc(zkTimeout, func() {
if zkTimeout > 0 {
errch <- fmt.Errorf("master detection timed out after %s", zkTimeout)
}
})
defer reload.Stop()
defer util.HandleCrash()
if !provider.Watch {
reload.Stop()
timeout.Stop()
}
for {
select {
case <-reload.C:
configuration := provider.loadMesosConfig()
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "mesos",
Configuration: configuration,
}
}
case masters := <-changed:
if len(masters) == 0 || masters[0] == "" {
// no leader
timeout.Reset(zkTimeout)
} else {
timeout.Stop()
}
log.Debugf("new masters detected: %v", masters)
provider.Masters = masters
configuration := provider.loadMesosConfig()
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: "mesos",
Configuration: configuration,
}
}
case err := <-errch:
log.Errorf("%s", err)
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("mesos connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Cannot connect to mesos server %+v", err)
}
return nil
}