// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { config := api.DefaultConfig() config.Address = provider.Endpoint client, err := api.NewClient(config) if err != nil { return err } provider.client = client provider.Constraints = append(provider.Constraints, constraints...) pool.Go(func(stop chan bool) { notify := func(err error, time time.Duration) { log.Errorf("Consul connection error %+v, retrying in %s", err, time) } operation := func() error { return provider.watch(configurationChan, stop) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to consul server %+v", err) } }) return err }
func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { provider.Constraints = append(provider.Constraints, constraints...) operation := func() error { if _, err := provider.kvclient.Exists("qmslkjdfmqlskdjfmqlksjazçueznbvbwzlkajzebvkwjdcqmlsfj"); err != nil { return fmt.Errorf("Failed to test KV store connection: %v", err) } if provider.Watch { pool.Go(func(stop chan bool) { err := provider.watchKv(configurationChan, provider.Prefix, stop) if err != nil { log.Errorf("Cannot watch KV store: %v", err) } }) } configuration := provider.loadConfig() configurationChan <- types.ConfigMessage{ ProviderName: string(provider.storeType), Configuration: configuration, } return nil } notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } return nil }
// listenForNodeChanges listens for changes to node status using change feeds. // This function will block until the query fails func (c *Cluster) listenForNodeChanges() error { // Start listening to changes from a random active node node, hpr, err := c.GetNextNode() if err != nil { return err } q, err := newQuery( DB("rethinkdb").Table("server_status").Changes(), map[string]interface{}{}, c.opts, ) if err != nil { return fmt.Errorf("Error building query: %s", err) } cursor, err := node.Query(q) if err != nil { hpr.Mark(err) return err } // Keep reading node status updates from changefeed var result struct { NewVal nodeStatus `gorethink:"new_val"` OldVal nodeStatus `gorethink:"old_val"` } for cursor.Next(&result) { addr := fmt.Sprintf("%s:%d", result.NewVal.Network.Hostname, result.NewVal.Network.ReqlPort) addr = strings.ToLower(addr) switch result.NewVal.Status { case "connected": // Connect to node using exponential backoff (give up after waiting 5s) // to give the node time to start-up. b := backoff.NewExponentialBackOff() b.MaxElapsedTime = time.Second * 5 backoff.Retry(func() error { node, err := c.connectNodeWithStatus(result.NewVal) if err == nil { if !c.nodeExists(node) { c.addNode(node) Log.WithFields(logrus.Fields{ "id": node.ID, "host": node.Host.String(), }).Debug("Connected to node") } } return err }, b) } } err = cursor.Err() hpr.Mark(err) return err }
func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) error { operation := func() error { events, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{})) if err != nil { return fmt.Errorf("Failed to KV WatchTree: %v", err) } for { select { case <-stop: return nil case _, ok := <-events: if !ok { return errors.New("watchtree channel closed") } configuration := provider.loadConfig() if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: string(provider.storeType), Configuration: configuration, } } } } } notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } return nil }
func newConfiguredStorage(newLog spec.Log, storageType, storagePrefix, storageAddr string) (spec.Storage, error) { var newStorage spec.Storage var err error switch storageType { case "redis": newStorageConfig := redis.DefaultStorageConfigWithAddr(storageAddr) newStorageConfig.BackoffFactory = func() spec.Backoff { return backoff.NewExponentialBackOff() } newStorageConfig.Instrumentation, err = newPrometheusInstrumentation([]string{"Feature", "Storage", "Redis"}) if err != nil { return nil, maskAny(err) } newStorageConfig.Log = newLog newStorageConfig.Prefix = storagePrefix newStorage, err = redis.NewStorage(newStorageConfig) if err != nil { return nil, maskAny(err) } case "memory": newStorage, err = memory.NewStorage(memory.DefaultStorageConfig()) if err != nil { return nil, maskAny(err) } default: return nil, maskAnyf(invalidStorageFlagError, "%s", storageType) } return newStorage, nil }
// NewBreakerWithOptions creates a base breaker with a specified backoff, clock and TripFunc func NewBreakerWithOptions(options *Options) *Breaker { if options == nil { options = &Options{} } if options.Clock == nil { options.Clock = clock.New() } if options.BackOff == nil { b := backoff.NewExponentialBackOff() b.InitialInterval = defaultInitialBackOffInterval b.MaxElapsedTime = defaultBackoffMaxElapsedTime b.Clock = options.Clock b.Reset() options.BackOff = b } if options.WindowTime == 0 { options.WindowTime = DefaultWindowTime } if options.WindowBuckets == 0 { options.WindowBuckets = DefaultWindowBuckets } return &Breaker{ BackOff: options.BackOff, Clock: options.Clock, ShouldTrip: options.ShouldTrip, nextBackOff: options.BackOff.NextBackOff(), counts: newWindow(options.WindowTime, options.WindowBuckets), } }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { k8sClient, err := provider.newK8sClient() if err != nil { return err } provider.Constraints = append(provider.Constraints, constraints...) pool.Go(func(stop chan bool) { operation := func() error { for { stopWatch := make(chan struct{}, 1) defer close(stopWatch) log.Debugf("Using label selector: '%s'", provider.LabelSelector) eventsChan, err := k8sClient.WatchAll(provider.LabelSelector, stopWatch) if err != nil { log.Errorf("Error watching kubernetes events: %v", err) timer := time.NewTimer(1 * time.Second) select { case <-timer.C: return err case <-stop: return nil } } for { select { case <-stop: return nil case event := <-eventsChan: log.Debugf("Received event from kubernetes %+v", event) templateObjects, err := provider.loadIngresses(k8sClient) if err != nil { return err } if reflect.DeepEqual(provider.lastConfiguration.Get(), templateObjects) { log.Debugf("Skipping event from kubernetes %+v", event) } else { provider.lastConfiguration.Set(templateObjects) configurationChan <- types.ConfigMessage{ ProviderName: "kubernetes", Configuration: provider.loadConfig(*templateObjects), } } } } } } notify := func(err error, time time.Duration) { log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Kubernetes server %+v", err) } }) return nil }
// LoadCertificateForDomains loads certificates from ACME for given domains func (a *ACME) LoadCertificateForDomains(domains []string) { domains = fun.Map(types.CanonicalDomain, domains).([]string) safe.Go(func() { operation := func() error { if a.client == nil { return fmt.Errorf("ACME client still not built") } return nil } notify := func(err error, time time.Duration) { log.Errorf("Error getting ACME client: %v, retrying in %s", err, time) } ebo := backoff.NewExponentialBackOff() ebo.MaxElapsedTime = 30 * time.Second err := backoff.RetryNotify(operation, ebo, notify) if err != nil { log.Errorf("Error getting ACME client: %v", err) return } account := a.store.Get().(*Account) var domain Domain if len(domains) == 0 { // no domain return } else if len(domains) > 1 { domain = Domain{Main: domains[0], SANs: domains[1:]} } else { domain = Domain{Main: domains[0]} } if _, exists := account.DomainsCertificate.exists(domain); exists { // domain already exists return } certificate, err := a.getDomainsCertificates(domains) if err != nil { log.Errorf("Error getting ACME certificates %+v : %v", domains, err) return } log.Debugf("Got certificate for domains %+v", domains) transaction, object, err := a.store.Begin() if err != nil { log.Errorf("Error creating transaction %+v : %v", domains, err) return } account = object.(*Account) _, err = account.DomainsCertificate.addCertificateForDomains(certificate, domain) if err != nil { log.Errorf("Error adding ACME certificates %+v : %v", domains, err) return } if err = transaction.Commit(account); err != nil { log.Errorf("Error Saving ACME account %+v: %v", account, err) return } }) }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Eureka) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, _ types.Constraints) error { operation := func() error { configuration, err := provider.buildConfiguration() if err != nil { log.Errorf("Failed to build configuration for Eureka, error: %s", err) return err } configurationChan <- types.ConfigMessage{ ProviderName: "eureka", Configuration: configuration, } var delay time.Duration if len(provider.Delay) > 0 { var err error delay, err = time.ParseDuration(provider.Delay) if err != nil { log.Errorf("Failed to parse delay for Eureka, error: %s", err) return err } } else { delay = time.Second * 30 } ticker := time.NewTicker(delay) go func() { for t := range ticker.C { log.Debug("Refreshing Eureka " + t.String()) configuration, err := provider.buildConfiguration() if err != nil { log.Errorf("Failed to refresh Eureka configuration, error: %s", err) return } configurationChan <- types.ConfigMessage{ ProviderName: "eureka", Configuration: configuration, } } }() return nil } notify := func(err error, time time.Duration) { log.Errorf("Eureka connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Eureka server %+v", err) return err } return nil }
// Begin creates a transaction with the KV store. func (d *Datastore) Begin() (Transaction, Object, error) { id := uuid.NewV4().String() log.Debugf("Transaction %s begins", id) remoteLock, err := d.kv.NewLock(d.lockKey, &store.LockOptions{TTL: 20 * time.Second, Value: []byte(id)}) if err != nil { return nil, nil, err } stopCh := make(chan struct{}) ctx, cancel := context.WithCancel(d.ctx) var errLock error go func() { _, errLock = remoteLock.Lock(stopCh) cancel() }() select { case <-ctx.Done(): if errLock != nil { return nil, nil, errLock } case <-d.ctx.Done(): stopCh <- struct{}{} return nil, nil, d.ctx.Err() } // we got the lock! Now make sure we are synced with KV store operation := func() error { meta := d.get() if meta.Lock != id { return fmt.Errorf("Object lock value: expected %s, got %s", id, meta.Lock) } return nil } notify := func(err error, time time.Duration) { log.Errorf("Datastore sync error: %v, retrying in %s", err, time) err = d.reload() if err != nil { log.Errorf("Error reloading: %+v", err) } } ebo := backoff.NewExponentialBackOff() ebo.MaxElapsedTime = 60 * time.Second err = backoff.RetryNotify(operation, ebo, notify) if err != nil { return nil, nil, fmt.Errorf("Datastore cannot sync: %v", err) } // we synced with KV store, we can now return Setter return &datastoreTransaction{ Datastore: d, remoteLock: remoteLock, id: id, }, d.meta.object, nil }
func newRandomService() (servicespec.Random, error) { newConfig := random.DefaultConfig() newConfig.BackoffFactory = func() spec.Backoff { return backoff.NewExponentialBackOff() } newService, err := random.New(newConfig) if err != nil { return nil, maskAny(err) } return newService, nil }
func connectSlackRTM(h *helios.Engine, s *SlackService) { err := backoff.Retry(func() error { h.Debug("Connecting to slack rtm") err := runSlackRTM(h, s) if err != nil { h.Warn("Failed to start slack rtm. Retrying.", "error", err.Error()) } return err }, backoff.NewExponentialBackOff()) if err != nil { s.Messages <- helios.NewError("Slack service error: %v", err) } }
// TryRequest try operation timeout, and retry backoff func TryRequest(url string, timeout time.Duration, condition Condition) error { exponentialBackOff := backoff.NewExponentialBackOff() exponentialBackOff.MaxElapsedTime = timeout var res *http.Response err := backoff.Retry(func() error { var err error res, err = http.Get(url) if err != nil { return err } return condition(res) }, exponentialBackOff) return err }
// Participate tries to be a leader func (l *Leadership) Participate(pool *safe.Pool) { pool.GoCtx(func(ctx context.Context) { log.Debugf("Node %s running for election", l.Cluster.Node) defer log.Debugf("Node %s no more running for election", l.Cluster.Node) backOff := backoff.NewExponentialBackOff() operation := func() error { return l.run(ctx, l.candidate) } notify := func(err error, time time.Duration) { log.Errorf("Leadership election error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, backOff, notify) if err != nil { log.Errorf("Cannot elect leadership %+v", err) } }) }
// NewStorage creates a new configured memory storage object. Therefore it // manages an in-memory redis instance which can be shut down using the // configured closer. This is used for local development. func NewStorage(config StorageConfig) (spec.Storage, error) { addrChan := make(chan string, 1) closer := make(chan struct{}, 1) redisAddr := "" go func() { s, err := miniredis.Run() if err != nil { panic(err) } addrChan <- s.Addr() <-closer s.Close() }() select { case <-time.After(1 * time.Second): panic("starting miniredis timed out") case addr := <-addrChan: redisAddr = addr } newRedisStorageConfig := redis.DefaultStorageConfigWithAddr(redisAddr) newRedisStorageConfig.BackoffFactory = func() spec.Backoff { return backoff.NewExponentialBackOff() } newRedisStorage, err := redis.NewStorage(newRedisStorageConfig) if err != nil { return nil, maskAny(err) } newStorage := &storage{ StorageConfig: config, Closer: closer, ID: id.MustNewID(), RedisStorage: newRedisStorage, ShutdownOnce: sync.Once{}, Type: ObjectType, } return newStorage, nil }
func (d *Datastore) watchChanges() error { stopCh := make(chan struct{}) kvCh, err := d.kv.Watch(d.lockKey, stopCh) if err != nil { return err } go func() { ctx, cancel := context.WithCancel(d.ctx) operation := func() error { for { select { case <-ctx.Done(): stopCh <- struct{}{} return nil case _, ok := <-kvCh: if !ok { cancel() return err } err = d.reload() if err != nil { return err } // log.Debugf("Datastore object change received: %+v", d.meta) if d.listener != nil { err := d.listener(d.meta.object) if err != nil { log.Errorf("Error calling datastore listener: %s", err) } } } } } notify := func(err error, time time.Duration) { log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Error in watch datastore: %v", err) } }() return nil }
// discover attempts to find new nodes in the cluster using the current nodes func (c *Cluster) discover() { // Keep retrying with exponential backoff. b := backoff.NewExponentialBackOff() // Never finish retrying (max interval is still 60s) b.MaxElapsedTime = 0 // Keep trying to discover new nodes for { backoff.RetryNotify(func() error { // If no hosts try seeding nodes if len(c.GetNodes()) == 0 { c.connectNodes(c.getSeeds()) } return c.listenForNodeChanges() }, b, func(err error, wait time.Duration) { Log.Debugf("Error discovering hosts %s, waiting: %s", err, wait) }) } }
func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate, exists bool) { log.Debugf("Challenge GetCertificate %s", domain) if !strings.HasSuffix(domain, ".acme.invalid") { return nil, false } c.lock.RLock() defer c.lock.RUnlock() account := c.store.Get().(*Account) if account.ChallengeCerts == nil { return nil, false } account.Init() var result *tls.Certificate operation := func() error { for _, cert := range account.ChallengeCerts { for _, dns := range cert.certificate.Leaf.DNSNames { if domain == dns { result = cert.certificate return nil } } } return fmt.Errorf("Cannot find challenge cert for domain %s", domain) } notify := func(err error, time time.Duration) { log.Errorf("Error getting cert: %v, retrying in %s", err, time) } ebo := backoff.NewExponentialBackOff() ebo.MaxElapsedTime = 60 * time.Second err := backoff.RetryNotify(operation, ebo, notify) if err != nil { log.Errorf("Error getting cert: %v", err) return nil, false } return result, true }
func TestJobBackOff(t *testing.T) { var ( testInitialInterval = 500 * time.Millisecond testRandomizationFactor = 0.1 testMultiplier = 2.0 testMaxInterval = 5 * time.Second testMinJobInterval = 1 * time.Second ) exp := NewBackOff(backoff.NewExponentialBackOff()) exp.InitialInterval = testInitialInterval exp.RandomizationFactor = testRandomizationFactor exp.Multiplier = testMultiplier exp.MaxInterval = testMaxInterval exp.MinJobInterval = testMinJobInterval exp.Reset() var expectedResults = []time.Duration{500, 500, 500, 1000, 2000, 4000, 5000, 5000, 500, 1000, 2000, 4000, 5000, 5000} for i, d := range expectedResults { expectedResults[i] = d * time.Millisecond } for i, expected := range expectedResults { // Assert that the next backoff falls in the expected range. var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected)) var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected)) if i < 3 || i == 8 { time.Sleep(2 * time.Second) } var actualInterval = exp.NextBackOff() if !(minInterval <= actualInterval && actualInterval <= maxInterval) { t.Error("error") } // assertEquals(t, expected, exp.currentInterval) } }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { provider.Constraints = append(provider.Constraints, constraints...) // TODO register this routine in pool, and watch for stop channel safe.Go(func() { operation := func() error { var err error dockerClient, err := provider.createClient() if err != nil { log.Errorf("Failed to create a client for docker, error: %s", err) return err } ctx := context.Background() version, err := dockerClient.ServerVersion(ctx) log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion) var dockerDataList []dockerData if provider.SwarmMode { dockerDataList, err = listServices(ctx, dockerClient) if err != nil { log.Errorf("Failed to list services for docker swarm mode, error %s", err) return err } } else { dockerDataList, err = listContainers(ctx, dockerClient) if err != nil { log.Errorf("Failed to list containers for docker, error %s", err) return err } } configuration := provider.loadDockerConfig(dockerDataList) configurationChan <- types.ConfigMessage{ ProviderName: "docker", Configuration: configuration, } if provider.Watch { ctx, cancel := context.WithCancel(ctx) if provider.SwarmMode { // TODO: This need to be change. Linked to Swarm events docker/docker#23827 ticker := time.NewTicker(SwarmDefaultWatchTime) pool.Go(func(stop chan bool) { for { select { case <-ticker.C: services, err := listServices(ctx, dockerClient) if err != nil { log.Errorf("Failed to list services for docker, error %s", err) return } configuration := provider.loadDockerConfig(services) if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: "docker", Configuration: configuration, } } case <-stop: ticker.Stop() cancel() return } } }) } else { pool.Go(func(stop chan bool) { for { select { case <-stop: cancel() return } } }) f := filters.NewArgs() f.Add("type", "container") options := dockertypes.EventsOptions{ Filters: f, } eventHandler := events.NewHandler(events.ByAction) startStopHandle := func(m eventtypes.Message) { log.Debugf("Docker event received %+v", m) containers, err := listContainers(ctx, dockerClient) if err != nil { log.Errorf("Failed to list containers for docker, error %s", err) // Call cancel to get out of the monitor cancel() return } configuration := provider.loadDockerConfig(containers) if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: "docker", Configuration: configuration, } } } eventHandler.Handle("start", startStopHandle) eventHandler.Handle("die", startStopHandle) eventHandler.Handle("health_status: healthy", startStopHandle) eventHandler.Handle("health_status: unhealthy", startStopHandle) eventHandler.Handle("health_status: starting", startStopHandle) errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler) if err := <-errChan; err != nil { return err } } } return nil } notify := func(err error, time time.Duration) { log.Errorf("Docker connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to docker server %+v", err) } }) return nil }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { provider.Constraints = append(provider.Constraints, constraints...) operation := func() error { config := marathon.NewDefaultConfig() config.URL = provider.Endpoint config.EventsTransport = marathon.EventsTransportSSE if provider.Basic != nil { config.HTTPBasicAuthUser = provider.Basic.HTTPBasicAuthUser config.HTTPBasicPassword = provider.Basic.HTTPBasicPassword } if len(provider.DCOSToken) > 0 { config.DCOSToken = provider.DCOSToken } TLSConfig, err := provider.TLS.CreateTLSConfig() if err != nil { return err } config.HTTPClient = &http.Client{ Transport: &http.Transport{ DialContext: (&net.Dialer{ KeepAlive: provider.KeepAlive * time.Second, Timeout: time.Second * provider.DialerTimeout, }).DialContext, TLSClientConfig: TLSConfig, }, } client, err := marathon.NewClient(config) if err != nil { log.Errorf("Failed to create a client for marathon, error: %s", err) return err } provider.marathonClient = client update := make(marathon.EventsChannel, 5) if provider.Watch { if err := client.AddEventsListener(update, marathon.EventIDApplications); err != nil { log.Errorf("Failed to register for events, %s", err) return err } pool.Go(func(stop chan bool) { defer close(update) for { select { case <-stop: return case event := <-update: log.Debug("Marathon event receveived", event) configuration := provider.loadMarathonConfig() if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: "marathon", Configuration: configuration, } } } } }) } configuration := provider.loadMarathonConfig() configurationChan <- types.ConfigMessage{ ProviderName: "marathon", Configuration: configuration, } return nil } notify := func(err error, time time.Duration) { log.Errorf("Marathon connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Marathon server %+v", err) } return nil }
func (p DeployMarathon) Install(data manifest.Manifest) error { marathonApi, err := MarathonClient(data.GetString("marathon-address")) if err != nil { return err } fullName := data.GetString("app-name") bs, bf, bmax, grace := 5.0, 2.0, 120.0, 30.0 app := &marathon.Application{ BackoffSeconds: &bs, BackoffFactor: &bf, MaxLaunchDelaySeconds: &bmax, TaskKillGracePeriodSeconds: &grace, UpgradeStrategy: &marathon.UpgradeStrategy{ MinimumHealthCapacity: 0.5, // rolling update MaximumOverCapacity: 0.0, }, } portArgs := "" if port := data.GetString("listen-port"); port != "" { portArgs = "--port " + port } app.Name(fullName) app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd"))) app.Count(data.GetInt("instances")) app.Memory(float64(data.GetInt("mem"))) if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil { app.CPU(cpu) } if cluster := data.GetString("cluster"); cluster != "" { cs := strings.SplitN(cluster, ":", 2) app.AddConstraint(cs[0], "CLUSTER", cs[1]) app.AddLabel(cs[0], cs[1]) } for _, cons := range data.GetArray("constraints") { if consArr, ok := cons.Unwrap().([]interface{}); ok { consStrings := make([]string, len(consArr)) for i, c := range consArr { consStrings[i] = fmt.Sprintf("%s", c) } app.AddConstraint(consStrings...) } } for _, port := range data.GetArray("ports") { app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0))) } app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app for k, v := range data.GetMap("environment") { app.AddEnv(k, fmt.Sprintf("%s", v.Unwrap())) } app.AddUris(data.GetString("package-uri")) // todo: в манифесте задавать массив healthchecks, их использовтаь в марафоне и консул-супервизоре // todo: открыть сетевой доступ от марафона до мезос-агентов, чтобы марафон мог хелсчеки посылать //if portArgs != "" { // health := marathon.NewDefaultHealthCheck() // health.Protocol = "TCP" // health.IntervalSeconds = 5 // *health.PortIndex = 0 // app.AddHealthCheck(*health) //} if _, err := marathonApi.UpdateApplication(app, false); err != nil { color.Yellow("marathon <- %s", app) return err } color.Green("marathon <- %s", app) consulApi, err := utils.ConsulClient(data.GetString("consul-address")) if err != nil { return err } if err := utils.RegisterPluginData("deploy.marathon", data.GetString("app-name"), data.String(), data.GetString("consul-address")); err != nil { return err } return backoff.Retry(func() error { services, _, err := consulApi.Health().Service(fullName, "", true, nil) if err != nil { log.Println(color.RedString("Error in check health in consul: %v", err)) return err } if len(services) == 0 { log.Printf("Service `%s` not started yet! Retry...", fullName) return fmt.Errorf("Service `%s` not started!", fullName) } log.Println(color.GreenString("Service `%s` successfully started!", fullName)) return nil }, backoff.NewExponentialBackOff()) }
func (p ReleaseHttp) Run(data manifest.Manifest) error { if !data.Has("routes") { log.Println("No routes configured for release.") return nil } consul, err := utils.ConsulClient(data.GetString("consul-address")) if err != nil { return err } fullName := data.GetString("full-name") // check current service is alive and healthy if err := backoff.Retry(func() error { services, _, err := consul.Health().Service(fullName, "", true, nil) if err != nil { log.Println(color.RedString("Error in check health in consul: %v", err)) return err } if len(services) == 0 { log.Printf("Service `%s` not started yet! Retry...", fullName) return fmt.Errorf("Service `%s` not started!", fullName) } else { log.Printf("Service `%s` started with %v instances.", fullName, len(services)) return nil } }, backoff.NewExponentialBackOff()); err != nil { return err } routeVars := make(map[string]string, 0) if data.Has("route") { if err := json.Unmarshal([]byte(data.GetString("route")), &routeVars); err != nil { log.Println(color.RedString("Error parse route json: %v, %s", err, data.GetString("route"))) return err } } // collect routes routes := make([]map[string]string, 0) for _, route := range data.GetArray("routes") { if !route.Has("host") { log.Printf("Not found 'host': %s, skip...", route.String()) continue } fields := make(map[string]string) for k, v := range route.Unwrap().(map[string]interface{}) { fields[k] = fmt.Sprintf("%v", v) } routes = append(routes, utils.MergeMaps(fields, routeVars)) } if len(routes) == 0 { log.Println("No routes configured for release.") return nil } routesJson, err := json.MarshalIndent(routes, "", " ") if err != nil { return err } // write routes to consul kv if err := utils.PutConsulKv(consul, "services/routes/"+fullName, string(routesJson)); err != nil { return err } log.Println(color.GreenString("Service `%s` released with routes: %s", fullName, string(routesJson))) // find old services with the same routes existsRoutes, err := utils.ListConsulKv(consul, "services/routes/"+data.GetString("name-prefix"), nil) if err != nil { return err } for _, existsRoute := range existsRoutes { if existsRoute.Key != fmt.Sprintf("services/routes/%s", fullName) { // skip current service oldRoutes := make([]map[string]string, 0) if err := json.Unmarshal(existsRoute.Value, &oldRoutes); err != nil { return err } OuterLoop: for _, route := range routes { for _, oldRoute := range oldRoutes { if utils.MapsEqual(route, oldRoute) { outdated := strings.TrimPrefix(existsRoute.Key, "services/routes/") log.Println(color.GreenString("Found %s with the same routes %v. Remove it!", outdated, string(existsRoute.Value))) if err := utils.DelConsulKv(consul, existsRoute.Key); err != nil { return err } if err := utils.MarkAsOutdated(consul, outdated, 10*time.Minute); err != nil { return err } break OuterLoop } } } } } return nil }
// Try try operation timeout, and retry backoff func Try(timeout time.Duration, operation func() error) error { exponentialBackOff := backoff.NewExponentialBackOff() exponentialBackOff.MaxElapsedTime = timeout err := backoff.Retry(operation, exponentialBackOff) return err }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { operation := func() error { // initialize logging logging.SetupLogs() log.Debugf("%s", provider.IPSources) var zk string var masters []string if strings.HasPrefix(provider.Endpoint, "zk://") { zk = provider.Endpoint } else { masters = strings.Split(provider.Endpoint, ",") } errch := make(chan error) changed := detectMasters(zk, masters) reload := time.NewTicker(time.Second * time.Duration(provider.RefreshSeconds)) zkTimeout := time.Second * time.Duration(provider.ZkDetectionTimeout) timeout := time.AfterFunc(zkTimeout, func() { if zkTimeout > 0 { errch <- fmt.Errorf("master detection timed out after %s", zkTimeout) } }) defer reload.Stop() defer util.HandleCrash() if !provider.Watch { reload.Stop() timeout.Stop() } for { select { case <-reload.C: configuration := provider.loadMesosConfig() if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: "mesos", Configuration: configuration, } } case masters := <-changed: if len(masters) == 0 || masters[0] == "" { // no leader timeout.Reset(zkTimeout) } else { timeout.Stop() } log.Debugf("new masters detected: %v", masters) provider.Masters = masters configuration := provider.loadMesosConfig() if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: "mesos", Configuration: configuration, } } case err := <-errch: log.Errorf("%s", err) } } } notify := func(err error, time time.Duration) { log.Errorf("mesos connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to mesos server %+v", err) } return nil }