Ejemplo n.º 1
0
func (s *storage) Set(key, value string) error {
	s.Log.WithTags(spec.Tags{C: nil, L: "D", O: s, V: 13}, "call Set")

	action := func() error {
		conn := s.Pool.Get()
		defer conn.Close()

		reply, err := redis.String(conn.Do("SET", s.withPrefix(key), value))
		if err != nil {
			return maskAny(err)
		}

		if reply != "OK" {
			return maskAnyf(queryExecutionFailedError, "SET not executed correctly")
		}

		return nil
	}

	err := backoff.Retry(s.Instrumentation.WrapFunc("Set", action), s.BackoffFactory())
	if err != nil {
		return maskAny(err)
	}

	return nil
}
Ejemplo n.º 2
0
// listenForNodeChanges listens for changes to node status using change feeds.
// This function will block until the query fails
func (c *Cluster) listenForNodeChanges() error {
	// Start listening to changes from a random active node
	node, hpr, err := c.GetNextNode()
	if err != nil {
		return err
	}

	q, err := newQuery(
		DB("rethinkdb").Table("server_status").Changes(),
		map[string]interface{}{},
		c.opts,
	)
	if err != nil {
		return fmt.Errorf("Error building query: %s", err)
	}

	cursor, err := node.Query(q)
	if err != nil {
		hpr.Mark(err)
		return err
	}

	// Keep reading node status updates from changefeed
	var result struct {
		NewVal nodeStatus `gorethink:"new_val"`
		OldVal nodeStatus `gorethink:"old_val"`
	}
	for cursor.Next(&result) {
		addr := fmt.Sprintf("%s:%d", result.NewVal.Network.Hostname, result.NewVal.Network.ReqlPort)
		addr = strings.ToLower(addr)

		switch result.NewVal.Status {
		case "connected":
			// Connect to node using exponential backoff (give up after waiting 5s)
			// to give the node time to start-up.
			b := backoff.NewExponentialBackOff()
			b.MaxElapsedTime = time.Second * 5

			backoff.Retry(func() error {
				node, err := c.connectNodeWithStatus(result.NewVal)
				if err == nil {
					if !c.nodeExists(node) {
						c.addNode(node)

						Log.WithFields(logrus.Fields{
							"id":   node.ID,
							"host": node.Host.String(),
						}).Debug("Connected to node")
					}
				}

				return err
			}, b)
		}
	}

	err = cursor.Err()
	hpr.Mark(err)
	return err
}
Ejemplo n.º 3
0
func (c *Client) executeRequest(req *http.Request) (*Response, error) {

	var parsedError error
	var parsedResponse *Response

	// Setup our retryable operation.
	retry := func() error {

		// Execute the actual request.
		response, err := c.goClient.Do(req)
		if err != nil {
			parsedError = err
			return nil
		}

		// Defer closing response body if it exists.
		if response != nil {
			defer response.Body.Close()
		}

		// Parse our response into a gohttp.Response object.
		parsedResponse, parsedError = NewResponse(response)
		if parsedError != nil {
			return nil
		}

		// Retry status code checks.
		//
		// If the client encounters a retryable status code, we return
		// an error to signal a retry should occur
		for _, code := range c.RetryableStatusCodes {
			if code == parsedResponse.Code {
				return errors.New("Encountered retryable status code.")
			}
		}
		return nil
	}

	// Rate Limiter - If we are using a rate limiter, we enter here.
	//
	// This will block until the rate limiter is satisfied.
	if c.rateLimiter != nil {
		err := c.rateLimiter.Enter()
		if err != nil {
			return nil, err
		}
	}

	// Execute the retryable operation
	err := backoff.Retry(retry, c.Backoff)
	if err != nil {
		return nil, err
	}

	return parsedResponse, parsedError
}
Ejemplo n.º 4
0
func connectSlackRTM(h *helios.Engine, s *SlackService) {
	err := backoff.Retry(func() error {
		h.Debug("Connecting to slack rtm")
		err := runSlackRTM(h, s)
		if err != nil {
			h.Warn("Failed to start slack rtm. Retrying.", "error", err.Error())
		}
		return err
	}, backoff.NewExponentialBackOff())
	if err != nil {
		s.Messages <- helios.NewError("Slack service error: %v", err)
	}
}
Ejemplo n.º 5
0
// TryRequest try operation timeout, and retry backoff
func TryRequest(url string, timeout time.Duration, condition Condition) error {
	exponentialBackOff := backoff.NewExponentialBackOff()
	exponentialBackOff.MaxElapsedTime = timeout
	var res *http.Response
	err := backoff.Retry(func() error {
		var err error
		res, err = http.Get(url)
		if err != nil {
			return err
		}
		return condition(res)
	}, exponentialBackOff)
	return err
}
Ejemplo n.º 6
0
func SupervisorCommand() cli.Command {
	return cli.Command{
		Name:            "supervisor",
		SkipFlagParsing: true,
		Action: func(c *cli.Context) error {
			var cmd *exec.Cmd = nil

			go func() {
				backoff.Retry(func() error {
					log.Println(color.GreenString("supervisor: Starting %v", c.Args()))

					cmd = exec.Command(c.Args().First(), c.Args().Tail()...)
					cmd.Stdout = os.Stdout
					cmd.Stderr = os.Stderr

					if err := cmd.Start(); err != nil {
						log.Println(color.RedString("supervisor: Error on process staring %v", err))
						return err
					}

					err := cmd.Wait()
					if err != nil {
						log.Println(color.RedString("supervisor: Command exit with error: %v", err))
					} else {
						log.Println(color.YellowString("supervisor: Command completed."))
						os.Exit(0)
					}
					return err
				}, backoff.NewConstantBackOff(time.Second*3))
			}()

			// Handle shutdown signals and kill child process
			ch := make(chan os.Signal)
			signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
			log.Println("supervisor: receive signal", <-ch)

			if cmd != nil {
				log.Println("supervisor: send SIGTERM to subprocess")
				cmd.Process.Signal(syscall.SIGTERM)
				res, err := cmd.Process.Wait()
				log.Println("supervisor: subprocess stopped with", res, err)
			}

			log.Println(color.YellowString("supervisor: Stopped."))

			os.Exit(143)
			return nil
		},
	}
}
Ejemplo n.º 7
0
func (s *storage) Remove(key string) error {
	s.Log.WithTags(spec.Tags{C: nil, L: "D", O: s, V: 13}, "call Remove")

	action := func() error {
		conn := s.Pool.Get()
		defer conn.Close()

		_, err := redis.Int64(conn.Do("DEL", s.withPrefix(key)))
		if err != nil {
			return maskAny(err)
		}

		return nil
	}

	err := backoff.Retry(s.Instrumentation.WrapFunc("Remove", action), s.BackoffFactory())
	if err != nil {
		return maskAny(err)
	}

	return nil
}
Ejemplo n.º 8
0
func (s *service) CreateMax(max int) (int, error) {
	// Define the action.
	var result int
	action := func() error {
		done := make(chan struct{}, 1)
		fail := make(chan error, 1)

		go func() {
			m := big.NewInt(int64(max))
			j, err := s.RandFactory(s.RandReader, m)
			if err != nil {
				fail <- maskAny(err)
				return
			}

			result = int(j.Int64())

			done <- struct{}{}
		}()

		select {
		case <-time.After(s.Timeout):
			return maskAnyf(timeoutError, "after %s", s.Timeout)
		case err := <-fail:
			return maskAny(err)
		case <-done:
			return nil
		}
	}

	// Execute the action wrapped with a retrier.
	err := backoff.Retry(action, s.BackoffFactory())
	if err != nil {
		return 0, maskAny(err)
	}

	return result, nil
}
Ejemplo n.º 9
0
// Try try operation timeout, and retry backoff
func Try(timeout time.Duration, operation func() error) error {
	exponentialBackOff := backoff.NewExponentialBackOff()
	exponentialBackOff.MaxElapsedTime = timeout
	err := backoff.Retry(operation, exponentialBackOff)
	return err
}
Ejemplo n.º 10
0
func (p DeployMarathon) Install(data manifest.Manifest) error {
	marathonApi, err := MarathonClient(data.GetString("marathon-address"))
	if err != nil {
		return err
	}

	fullName := data.GetString("app-name")

	bs, bf, bmax, grace := 5.0, 2.0, 120.0, 30.0
	app := &marathon.Application{
		BackoffSeconds:             &bs,
		BackoffFactor:              &bf,
		MaxLaunchDelaySeconds:      &bmax,
		TaskKillGracePeriodSeconds: &grace,
		UpgradeStrategy: &marathon.UpgradeStrategy{
			MinimumHealthCapacity: 0.5, // rolling update
			MaximumOverCapacity:   0.0,
		},
	}

	portArgs := ""
	if port := data.GetString("listen-port"); port != "" {
		portArgs = "--port " + port
	}

	app.Name(fullName)
	app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd")))
	app.Count(data.GetInt("instances"))
	app.Memory(float64(data.GetInt("mem")))

	if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil {
		app.CPU(cpu)
	}

	if cluster := data.GetString("cluster"); cluster != "" {
		cs := strings.SplitN(cluster, ":", 2)
		app.AddConstraint(cs[0], "CLUSTER", cs[1])
		app.AddLabel(cs[0], cs[1])
	}

	for _, cons := range data.GetArray("constraints") {
		if consArr, ok := cons.Unwrap().([]interface{}); ok {
			consStrings := make([]string, len(consArr))
			for i, c := range consArr {
				consStrings[i] = fmt.Sprintf("%s", c)
			}
			app.AddConstraint(consStrings...)
		}
	}

	for _, port := range data.GetArray("ports") {
		app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0)))
	}

	app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app

	for k, v := range data.GetMap("environment") {
		app.AddEnv(k, fmt.Sprintf("%s", v.Unwrap()))
	}

	app.AddUris(data.GetString("package-uri"))

	// todo: в манифесте задавать массив healthchecks, их использовтаь в марафоне и консул-супервизоре
	// todo: открыть сетевой доступ от марафона до мезос-агентов, чтобы марафон мог хелсчеки посылать

	//if portArgs != "" {
	//	health := marathon.NewDefaultHealthCheck()
	//	health.Protocol = "TCP"
	//	health.IntervalSeconds = 5
	//	*health.PortIndex = 0
	//	app.AddHealthCheck(*health)
	//}

	if _, err := marathonApi.UpdateApplication(app, false); err != nil {
		color.Yellow("marathon <- %s", app)
		return err
	}

	color.Green("marathon <- %s", app)

	consulApi, err := utils.ConsulClient(data.GetString("consul-address"))
	if err != nil {
		return err
	}

	if err := utils.RegisterPluginData("deploy.marathon", data.GetString("app-name"), data.String(), data.GetString("consul-address")); err != nil {
		return err
	}

	return backoff.Retry(func() error {
		services, _, err := consulApi.Health().Service(fullName, "", true, nil)

		if err != nil {
			log.Println(color.RedString("Error in check health in consul: %v", err))
			return err
		}

		if len(services) == 0 {
			log.Printf("Service `%s` not started yet! Retry...", fullName)
			return fmt.Errorf("Service `%s` not started!", fullName)
		}

		log.Println(color.GreenString("Service `%s` successfully started!", fullName))
		return nil
	}, backoff.NewExponentialBackOff())
}
Ejemplo n.º 11
0
func (p ReleaseHttp) Run(data manifest.Manifest) error {
	if !data.Has("routes") {
		log.Println("No routes configured for release.")
		return nil
	}

	consul, err := utils.ConsulClient(data.GetString("consul-address"))
	if err != nil {
		return err
	}

	fullName := data.GetString("full-name")

	// check current service is alive and healthy
	if err := backoff.Retry(func() error {
		services, _, err := consul.Health().Service(fullName, "", true, nil)
		if err != nil {
			log.Println(color.RedString("Error in check health in consul: %v", err))
			return err
		}

		if len(services) == 0 {
			log.Printf("Service `%s` not started yet! Retry...", fullName)
			return fmt.Errorf("Service `%s` not started!", fullName)
		} else {
			log.Printf("Service `%s` started with %v instances.", fullName, len(services))
			return nil
		}
	}, backoff.NewExponentialBackOff()); err != nil {
		return err
	}

	routeVars := make(map[string]string, 0)
	if data.Has("route") {
		if err := json.Unmarshal([]byte(data.GetString("route")), &routeVars); err != nil {
			log.Println(color.RedString("Error parse route json: %v, %s", err, data.GetString("route")))
			return err
		}
	}

	// collect routes
	routes := make([]map[string]string, 0)
	for _, route := range data.GetArray("routes") {
		if !route.Has("host") {
			log.Printf("Not found 'host': %s, skip...", route.String())
			continue
		}

		fields := make(map[string]string)
		for k, v := range route.Unwrap().(map[string]interface{}) {
			fields[k] = fmt.Sprintf("%v", v)
		}

		routes = append(routes, utils.MergeMaps(fields, routeVars))
	}

	if len(routes) == 0 {
		log.Println("No routes configured for release.")
		return nil
	}

	routesJson, err := json.MarshalIndent(routes, "", "  ")
	if err != nil {
		return err
	}

	// write routes to consul kv
	if err := utils.PutConsulKv(consul, "services/routes/"+fullName, string(routesJson)); err != nil {
		return err
	}

	log.Println(color.GreenString("Service `%s` released with routes: %s", fullName, string(routesJson)))

	// find old services with the same routes
	existsRoutes, err := utils.ListConsulKv(consul, "services/routes/"+data.GetString("name-prefix"), nil)
	if err != nil {
		return err
	}

	for _, existsRoute := range existsRoutes {
		if existsRoute.Key != fmt.Sprintf("services/routes/%s", fullName) { // skip current service
			oldRoutes := make([]map[string]string, 0)
			if err := json.Unmarshal(existsRoute.Value, &oldRoutes); err != nil {
				return err
			}

		OuterLoop:
			for _, route := range routes {
				for _, oldRoute := range oldRoutes {
					if utils.MapsEqual(route, oldRoute) {
						outdated := strings.TrimPrefix(existsRoute.Key, "services/routes/")
						log.Println(color.GreenString("Found %s with the same routes %v. Remove it!", outdated, string(existsRoute.Value)))

						if err := utils.DelConsulKv(consul, existsRoute.Key); err != nil {
							return err
						}

						if err := utils.MarkAsOutdated(consul, outdated, 10*time.Minute); err != nil {
							return err
						}

						break OuterLoop
					}
				}
			}
		}
	}

	return nil
}