コード例 #1
0
func (ztr *ZipkinTaskRunner) RunTask(context *framework.Variables, application *framework.Application, task map[string]string) error {
	api := context.MustGet(fmt.Sprintf("%s.api", application.ID))

	id, ok := task["id"]
	if !ok {
		return errors.New("Missing task id")
	}

	taskType, ok := task["type"]
	if !ok {
		return fmt.Errorf("Missing task type for id %s", id)
	}

	client := NewZipkinMesosClient(api)
	_, err := client.Add(task)
	if err != nil {
		return err
	}

	response, err := client.Start(task)
	if err != nil {
		return err
	}

	if taskType == "query" {
		return ztr.fillTaskContext(context, application, response)
	}

	return nil
}
コード例 #2
0
func (etr *ExhibitorTaskRunner) RunTask(context *framework.Variables, application *framework.Application, task map[string]string) error {
	api := context.MustGet(fmt.Sprintf("%s.api", application.ID))

	client := NewExhibitorMesosClient(api)
	_, err := client.Add(task)
	if err != nil {
		return err
	}

	_, err = client.Update(task)
	if err != nil {
		return err
	}

	response, err := client.Start(task)
	if err != nil {
		return err
	}

	err = etr.fillTaskContext(context, application, response)
	if err != nil {
		return err
	}

	return client.AwaitZookeeperRunning()
}
コード例 #3
0
func (etr *ElasticsearchTaskRunner) FillContext(context *framework.Variables, application *framework.Application, task marathon.Task) error {
	context.SetStackVariable(fmt.Sprintf("%s.host", application.ID), task.Host)
	for idx, port := range task.Ports {
		context.SetStackVariable(fmt.Sprintf("%s.port%d", application.ID, idx), fmt.Sprint(port))
	}
	api := fmt.Sprintf("http://%s:%d", task.Host, task.Ports[0])
	context.SetStackVariable(fmt.Sprintf("%s.api", application.ID), api)

	client := NewElasticsearchClient(api)
	err := client.AwaitRunning(etr.awaitTimeout, etr.backoffTimeout)
	if err != nil {
		return err
	}

	tasks, err := client.GetTasks()
	if err != nil {
		return err
	}

	for idx, task := range tasks {
		context.SetStackVariable(fmt.Sprintf("%s.%d.httpAddress", application.ID, idx), task.HttpAddress)
		context.SetStackVariable(fmt.Sprintf("%s.%d.transportAddress", application.ID, idx), task.TransportAddress)
	}

	return nil
}
コード例 #4
0
func (str *StatsdTaskRunner) RunTask(context *framework.Variables, application *framework.Application, task map[string]string) error {
	api := context.MustGet(fmt.Sprintf("%s.api", application.ID))

	client := NewStatsdMesosClient(api)
	err := client.Update(task)
	if err != nil {
		return err
	}
	return client.Start(task)
}
コード例 #5
0
func (dtr *DSE02xTaskRunner) fillTaskContext(context *framework.Variables, application *framework.Application, response []byte) error {
	startResponse := new(DSEMesos02xStartResponse)
	err := json.Unmarshal(response, &startResponse)
	if err != nil {
		return err
	}

	if startResponse.Status == "timeout" {
		return errors.New("Timed out")
	}

	servers := make([]string, 0)
	for _, node := range startResponse.Nodes {
		nodeEndpoint := fmt.Sprintf("%s:%d", node.Runtime.Address, node.Runtime.Reservation.Ports["cql"])

		context.SetStackVariable(fmt.Sprintf("%s.cassandra-%s", application.ID, fmt.Sprint(node.ID)), nodeEndpoint)
		context.SetStackVariable(fmt.Sprintf("%s.cassandra-%s.host", application.ID, fmt.Sprint(node.ID)), node.Runtime.Address)
		for name, port := range node.Runtime.Reservation.Ports {
			context.SetStackVariable(fmt.Sprintf("%s.cassandra-%s.%sPort", application.ID, fmt.Sprint(node.ID), name), fmt.Sprint(port))
		}

		servers = append(servers, nodeEndpoint)
	}

	context.SetStackVariable(fmt.Sprintf("%s.cassandraConnect", application.ID), strings.Join(servers, ","))
	return nil
}
コード例 #6
0
func (dtr *DSETaskRunner) RunTask(context *framework.Variables, application *framework.Application, task map[string]string) error {
	api := context.MustGet(fmt.Sprintf("%s.api", application.ID))

	client := NewDSEMesosClient(api)
	_, err := client.Add(task)
	if err != nil {
		return err
	}

	response, err := client.Start(task)
	if err != nil {
		return err
	}

	return dtr.fillTaskContext(context, application, response)
}
コード例 #7
0
func (dtr *DSETaskRunner) fillTaskContext(context *framework.Variables, application *framework.Application, response map[string]interface{}) error {
	servers := make([]string, 0)
	serversMap := make(map[string]string)

	value, ok := response["value"].(map[string]interface{})
	if !ok {
		return errors.New("Wrong value field")
	}

	clusterArr, ok := value["cluster"].([]interface{})
	if !ok {
		return errors.New("Wrong cluster field")
	}

	for _, clusterIface := range clusterArr {
		cluster, ok := clusterIface.(map[string]interface{})
		if !ok {
			return errors.New("Wrong cluster field")
		}

		id, ok := cluster["id"].(string)
		if !ok {
			return errors.New("Wrong id field")
		}

		runtime, ok := cluster["runtime"].(map[string]interface{})
		if !ok {
			return errors.New("Wrong runtime field")
		}

		hostname, ok := runtime["hostname"].(string)
		if !ok {
			return errors.New("Wrong hostname field")
		}

		serversMap[id] = hostname
		servers = append(servers, hostname)
	}

	for id, host := range servers {
		context.SetStackVariable(fmt.Sprintf("%s.cassandra-node-%s", application.ID, fmt.Sprint(id)), host)
	}

	context.SetStackVariable(fmt.Sprintf("%s.cassandraConnect", application.ID), strings.Join(servers, ","))
	return nil
}
コード例 #8
0
func (etr *ExhibitorTaskRunner) fillTaskContext(context *framework.Variables, application *framework.Application, response *exhibitorCluster) error {
	servers := make([]string, 0)
	serversMap := make(map[string]string)

	for _, server := range response.Cluster {
		hostname := fmt.Sprintf("%s:%s", server.Config.Hostname, etr.getClientPort(server))

		serversMap[server.Id] = hostname
		servers = append(servers, hostname)
	}

	for id, host := range servers {
		context.SetStackVariable(fmt.Sprintf("%s.exhibitor-%s", application.ID, fmt.Sprint(id)), host)
	}

	context.SetStackVariable(fmt.Sprintf("%s.zkConnect", application.ID), strings.Join(servers, ","))
	return nil
}
コード例 #9
0
func (ztr *ZipkinTaskRunner) fillTaskContext(context *framework.Variables, application *framework.Application, response map[string]interface{}) error {
	valueArr, ok := response["value"].([]interface{})
	if !ok {
		return errors.New("Wrong value field")
	}

	for _, valueIface := range valueArr {
		value, ok := valueIface.(map[string]interface{})
		if !ok {
			return errors.New("Wrong value interface field")
		}

		id, ok := value["id"].(string)
		if !ok {
			return errors.New("Wrong id field")
		}

		config, ok := value["config"].(map[string]interface{})
		if !ok {
			return errors.New("Wrong config field")
		}

		hostname, ok := config["hostname"].(string)
		if !ok {
			return errors.New("Wrong hostname field")
		}

		env, ok := config["env"].(map[string]interface{})
		if !ok {
			return errors.New("Wrong env field")
		}

		queryPort, ok := env["QUERY_PORT"].(string)
		if !ok {
			return errors.New("Wrong QUERY_PORT env")
		}

		context.SetStackVariable(fmt.Sprintf("%s.query-%s.endpoint", application.ID, id), fmt.Sprintf("%s:%s", hostname, queryPort))
	}

	return nil
}
コード例 #10
0
func (dtr *DSETaskRunner) FillContext(context *framework.Variables, application *framework.Application, task marathon.Task) error {
	context.SetStackVariable(fmt.Sprintf("%s.host", application.ID), task.Host)
	for idx, port := range task.Ports {
		context.SetStackVariable(fmt.Sprintf("%s.port%d", application.ID, idx), fmt.Sprint(port))
	}
	context.SetStackVariable(fmt.Sprintf("%s.api", application.ID), fmt.Sprintf("http://%s:%d", task.Host, task.Ports[0]))

	return nil
}
コード例 #11
0
ファイル: server.go プロジェクト: elodina/stack-deploy
func (sc *ServerCommand) Run(args []string) int {
	schedulerConfig := framework.NewSchedulerConfig()

	var (
		flags             = flag.NewFlagSet("server", flag.ExitOnError)
		marathonURL       = flags.String("marathon", "http://127.0.0.1:8080", "Marathon address <ip:port>.")
		persistentStorage = flags.String("storage", "", "Storage to store stack-deploy runtime information to recover from failovers. Required.")
		api               = flags.String("api", "0.0.0.0:4200", "Stack-deploy server binding address")
		bootstrap         = flags.String("bootstrap", "", "Stack file to bootstrap with.")
		cassandra         = flags.String("cassandra", "127.0.0.1", "Cassandra cluster IPs, comma-separated")
		keyspace          = flags.String("keyspace", "stack_deploy", "Cassandra keyspace")
		proto             = flags.Int("proto", 3, "Cassandra protocol version")
		connectRetries    = flags.Int("connect.retries", 10, "Number of retries to connect to either Marathon or Cassandra")
		connectBackoff    = flags.Duration("connect.backoff", 10*time.Second, "Backoff between connection attempts to either Marathon or Cassandra")
		debug             = flags.Bool("debug", false, "Flag for debug mode")
		dev               = flags.Bool("dev", false, "Flag for developer mode")
		variables         = make(vars)
	)
	flags.StringVar(&schedulerConfig.Master, "master", "127.0.0.1:5050", "Mesos Master address <ip:port>.")
	flags.StringVar(&schedulerConfig.User, "framework.user", "", "Mesos user. Defaults to current system user.")
	flags.StringVar(&schedulerConfig.FrameworkName, "framework.name", "stack-deploy", "Mesos framework name. Defaults to stack-deploy.")
	flags.StringVar(&schedulerConfig.FrameworkRole, "framework.role", "*", "Mesos framework role. Defaults to *.")
	flags.DurationVar(&schedulerConfig.FailoverTimeout, "failover.timeout", 168*time.Hour, "Mesos framework failover timeout. Defaults to 1 week.")
	flags.Var(variables, "var", "Global variables to add to every stack context run by stack-deploy server. Multiple occurrences of this flag allowed.")

	flags.Parse(args)
	if *debug {
		framework.Logger = plog.NewConsoleLogger(plog.DebugLevel, plog.DefaultLogFormat)
	}

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	framework.TaskRunners = sc.runners
	framework.MesosTaskRunners = sc.mesosRunners

	marathonClient, err := sc.connectMarathon(*marathonURL, *connectRetries, *connectBackoff)
	if err != nil {
		log.Fatal(err)
		return 1
	}

	if *persistentStorage == "" {
		if !*dev {
			log.Fatal("--storage flag is required. Examples: 'file:stack-deploy.json', 'zk:zookeeper.service:2181/stack-deploy'")
			return 1
		} else {
			*persistentStorage = "file:stack-deploy.json"
		}
	}

	frameworkStorage, err := framework.NewFrameworkStorage(*persistentStorage)
	if err != nil {
		log.Fatal(err)
		return 1
	}
	schedulerConfig.Storage = frameworkStorage
	frameworkStorage.Load()

	scheduler := framework.NewScheduler(schedulerConfig)
	err = scheduler.GetMesosState().Update()
	if err != nil {
		log.Fatal(err)
		return 1
	}

	if *bootstrap != "" {
		var context *framework.Variables
		if frameworkStorage.BootstrapContext != nil && len(frameworkStorage.BootstrapContext.All()) > 0 {
			log.Info("Restored bootstrap context from persistent storage")
			context = frameworkStorage.BootstrapContext
		} else {
			log.Info("No existing bootstrap context found in persistent storage, bootstrapping")
			var err error
			context, err = sc.Bootstrap(*bootstrap, marathonClient, scheduler, *connectRetries, *connectBackoff)
			if err != nil {
				log.Fatal(err)
				return 1
			}
		}

		log.Infof("Bootstrap context: %s", context)
		log.Infof("Cassandra connect before resolving: %s", *cassandra)
		for k, v := range context.All() {
			*cassandra = strings.Replace(*cassandra, fmt.Sprintf("{%s}", fmt.Sprint(k)), fmt.Sprint(v), -1)
		}
		log.Infof("Cassandra connect after resolving: %s", *cassandra)
		frameworkStorage.BootstrapContext = context
		frameworkStorage.Save()
	}

	var storage framework.Storage
	var userStorage framework.UserStorage
	var stateStorage framework.StateStorage
	var key string
	if !*dev {
		var connection *gocql.Session
		var err error
		storage, connection, err = framework.NewCassandraStorageRetryBackoff(strings.Split(*cassandra, ","), *keyspace, *connectRetries, *connectBackoff, *proto)
		if err != nil {
			panic(err)
		}

		userStorage, key, err = framework.NewCassandraUserStorage(connection, *keyspace)
		if err != nil {
			panic(err)
		}

		stateStorage, err = framework.NewCassandraStateStorage(connection, *keyspace)
		if err != nil {
			panic(err)
		}
	} else {
		log.Warning("Starting in developer mode. DO NOT use this in production!")
		schedulerConfig.FailoverTimeout = time.Duration(0)
		storage = framework.NewInMemoryStorage()
		userStorage = new(framework.NoopUserStorage)
		stateStorage = framework.NewInMemoryStateStorage()
	}

	apiServer := framework.NewApiServer(*api, marathonClient, variables, storage, userStorage, stateStorage, scheduler)
	if key != "" {
		fmt.Printf("***\nAdmin user key: %s\n***\n", key)
	}
	go apiServer.Start()

	<-ctrlc
	return 0
}