func (p DBCreatePostgresql) Run(data manifest.Manifest) error {
if data.GetBool("purge") {
return p.Drop(data)
} else {
return p.Create(data)
}
}
func (p DeployMarathon) Run(data manifest.Manifest) error {
if data.GetBool("purge") {
return p.Uninstall(data)
} else {
return p.Install(data)
}
}
func (p MarathonBuild) Run(data manifest.Manifest) error {
if err := utils.RunCmd("tar -zcf marathon.tar.gz -C %s/ .", data.GetString("source")); err != nil {
return err
}
return utils.RunCmd("curl -vsSf -XPUT -T marathon.tar.gz %s", data.GetString("registry-url"))
}
func (p UploadMarathon) Run(data manifest.Manifest) error {
if err := utils.RunCmd("curl -vsSf -o marathon.tar.gz %s", data.GetString("unstable-url")); err != nil {
return err
}
return utils.RunCmd("curl -vsSf -XPUT -T marathon.tar.gz %s", data.GetString("stable-url"))
}
func (p DeployDebian) Uninstall(data manifest.Manifest) error {
if err := utils.RunSshCmd(
data.GetString("cluster"),
data.GetString("ssh-user"),
fmt.Sprintf("sudo apt-get purge %s -y", data.GetString("package")),
); err != nil {
return err
}
return utils.DeletePluginData("deploy.debian", data.GetString("app-name"), data.GetString("consul-address"))
}
func (p TestAutotest) Run(data manifest.Manifest) error {
if err := utils.RunCmd("rm -rf tests && git clone --depth 1 --single-branch --recursive %s tests", data.GetString("repo")); err != nil {
return fmt.Errorf("Error on clone test git repo: %v", err)
}
return utils.RunCmd(
"cd tests/ && ./test.sh --project=%s --version=%s --suite=%s",
data.GetString("project"),
data.GetString("version"),
data.GetString("suite"),
)
}
func (p ReleaseDebian) Run(data manifest.Manifest) error {
return utils.RunSshCmd(
data.GetString("cluster"),
data.GetString("ssh-user"),
fmt.Sprintf(
"sudo %s/debian-way/release.sh --package='%s' --site='%s' --mode='%s'",
data.GetString("ci-tools-path"),
data.GetString("package"),
data.GetString("site"),
data.GetString("mode"),
),
)
}
func (p DBCreatePostgresql) Drop(data manifest.Manifest) error {
return utils.RunSingleSshCmd(
data.GetString("host"),
data.GetString("ssh-user"),
fmt.Sprintf("sudo -Hu postgres psql -c \\\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='%s';\\\" && sudo -Hu postgres dropdb --if-exists \"%s\"",
data.GetString("target"), data.GetString("target")),
)
}
func (p UploadDebian) Run(data manifest.Manifest) error {
return utils.RunCmd(
`ssh %s "%s %s %s %s"`,
data.GetString("ssh-connection"),
data.GetString("script"),
data.GetString("changes-file"),
data.GetString("src-repo"),
data.GetString("dst-repo"),
)
}
func (p TarballBuild) Run(data manifest.Manifest) error {
utils.RunCmd("rm -rf ./tarball.tmp && mkdir ./tarball.tmp")
for _, f := range data.GetArray("files") {
if file, ok := f.Unwrap().(string); ok {
utils.RunCmd("cp -a %s ./tarball.tmp/", file)
} else if files, ok := f.Unwrap().(map[string]interface{}); ok {
for from, to := range files {
utils.RunCmd("cp -a %s ./tarball.tmp/%s", from, to)
}
}
}
if err := utils.RunCmd("tar -zcf tarball.tar.gz -C ./tarball.tmp/ ."); err != nil {
return err
}
return utils.RunCmd("curl -vsSf -XPUT -T tarball.tar.gz %s", data.GetString("registry-url"))
}
func main() {
manifestFile := kingpin.Flag("manifest", "Path to manifest.yml file.").Default("manifest.yml").String()
plugin := kingpin.Arg("plugin", "Plugin name for run.").String()
vars := *kingpin.Flag("var", "key=value pairs with manifest vars.").StringMap()
dryRun := kingpin.Flag("dry-run", "Show manifest section only").Bool()
noColor := kingpin.Flag("no-color", "Disable colored output").Bool()
pluginData := kingpin.Flag("plugin-data", "Data for plugin").String()
kingpin.Version(version)
kingpin.Parse()
var plugins []manifest.PluginData
var err error
var manifestData *manifest.Manifest
color.NoColor = *noColor
if *pluginData != "" {
manifestData = manifest.LoadJSON(*pluginData)
} else {
manifestData = manifest.Load(*manifestFile, vars)
}
if *plugin == "" && *dryRun {
fmt.Printf("%s\n%s\n%s\n",
color.GreenString(">>> manifest:"),
manifestData.String(),
color.GreenString("<<< manifest: OK\n"))
return
}
if *pluginData != "" {
plugins = []manifest.PluginData{manifestData.GetPluginWithData(*plugin)}
} else {
plugins, err = manifestData.FindPlugins(*plugin)
}
if err != nil {
log.Fatalln(color.RedString("Error find plugins for '%s': %v", *plugin, err))
}
for _, pair := range plugins {
log.Printf("%s\n%s\n\n", color.GreenString(">>> %s:", pair.PluginName), color.CyanString("%s", pair.Data))
if !*dryRun {
if err := pair.Plugin.Run(pair.Data); err != nil {
fmt.Println("")
log.Fatalln(color.RedString("Error on run plugin `%s`: %v", pair.PluginName, err))
} else {
log.Println(color.GreenString("<<< %s: OK", pair.PluginName))
}
}
}
}
func (p SbtBuild) Run(data manifest.Manifest) error {
return utils.RunCmd(
`sbt ';set every version := "%s"' clean "%s" %s`,
data.GetString("version"),
data.GetString("test"),
data.GetStringOr("sbt", ""),
)
}
func (p DeployMarathon) Uninstall(data manifest.Manifest) error {
marathonApi, err := MarathonClient(data.GetString("marathon-address"))
if err != nil {
return err
}
name := data.GetString("app-name")
if _, err := marathonApi.Application(name); err == nil {
if _, err := marathonApi.DeleteApplication(name, false); err != nil {
return err
}
} else {
log.Println(color.YellowString("App `%s` doesnt exists in marathon!", name))
}
return utils.DeletePluginData("deploy.marathon", name, data.GetString("consul-address"))
}
func (p ExternalPlugin) Run(data manifest.Manifest) error {
return utils.RunCmdWithEnv(fmt.Sprintf("%s --plugin-data '%s'", p.Path, strings.Replace(data.String(), "\n", "", -1)), data.ToEnvMap("SERVE_"))
}
func (p BuildDebian) Run(data manifest.Manifest) error {
env := make(map[string]string)
// required fields
env["MANIFEST_PACKAGE"] = data.GetString("package")
env["MANIFEST_INFO_NAME"] = data.GetString("name")
env["MANIFEST_INFO_VERSION"] = data.GetString("version")
env["MANIFEST_BUILD_DEBIAN_SECTION"] = data.GetString("category")
env["MANIFEST_INFO_CATEGORY"] = data.GetString("category")
env["MANIFEST_BUILD_DEBIAN_MAINTAINER_NAME"] = data.GetString("maintainer-name")
env["MANIFEST_BUILD_DEBIAN_MAINTAINER_EMAIL"] = data.GetString("maintainer-email")
env["MANIFEST_BUILD_DEBIAN_INSTALL_ROOT"] = data.GetString("install-root")
daemon := data.GetString("daemon")
daemonArgs := data.GetString("daemon-args")
if daemon != "" && data.GetBool("consul-supervisor") {
daemonArgs = fmt.Sprintf(
"consul supervisor --service '%s/%s' --port $PORT1 start %s %s",
data.GetString("category"),
data.GetString("package"),
daemon,
daemonArgs,
)
daemon = "/usr/local/bin/serve-tools"
}
// optional fields
env["MANIFEST_BUILD_DEBIAN_DAEMON"] = daemon
env["MANIFEST_BUILD_DEBIAN_DAEMON_ARGS"] = daemonArgs
env["MANIFEST_BUILD_DEBIAN_SERVICE_OWNER"] = data.GetString("service-owner")
env["MANIFEST_BUILD_DEBIAN_DAEMON_USER"] = data.GetString("daemon-user")
env["MANIFEST_BUILD_DEBIAN_DAEMON_PORT"] = data.GetString("daemon-port")
env["MANIFEST_BUILD_DEBIAN_MAKE_PIDFILE"] = data.GetString("make-pidfile")
env["MANIFEST_BUILD_DEBIAN_DEPENDS"] = data.GetString("depends")
env["MANIFEST_BUILD_DEBIAN_DESCRIPTION"] = data.GetString("description")
env["MANIFEST_BUILD_DEBIAN_INIT"] = data.GetString("init")
env["MANIFEST_BUILD_DEBIAN_CRON"] = data.GetString("cron")
env["GO_PIPELINE_LABEL"] = data.GetString("build-number")
env["GO_STAGE_COUNTER"] = data.GetString("stage-counter")
return utils.RunCmdWithEnv(
fmt.Sprintf("%s/go/debian-build.sh --distribution=%s", data.GetString("ci-tools-path"), data.GetString("distribution")),
env,
)
}
func (p ShBuild) Run(data manifest.Manifest) error {
return utils.RunCmd(data.GetString("sh"))
}
func (p DeployMarathon) Install(data manifest.Manifest) error {
marathonApi, err := MarathonClient(data.GetString("marathon-address"))
if err != nil {
return err
}
fullName := data.GetString("app-name")
bs, bf, bmax, grace := 5.0, 2.0, 120.0, 30.0
app := &marathon.Application{
BackoffSeconds: &bs,
BackoffFactor: &bf,
MaxLaunchDelaySeconds: &bmax,
TaskKillGracePeriodSeconds: &grace,
UpgradeStrategy: &marathon.UpgradeStrategy{
MinimumHealthCapacity: 0.5, // rolling update
MaximumOverCapacity: 0.0,
},
}
portArgs := ""
if port := data.GetString("listen-port"); port != "" {
portArgs = "--port " + port
}
app.Name(fullName)
app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd")))
app.Count(data.GetInt("instances"))
app.Memory(float64(data.GetInt("mem")))
if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil {
app.CPU(cpu)
}
if cluster := data.GetString("cluster"); cluster != "" {
cs := strings.SplitN(cluster, ":", 2)
app.AddConstraint(cs[0], "CLUSTER", cs[1])
app.AddLabel(cs[0], cs[1])
}
for _, cons := range data.GetArray("constraints") {
if consArr, ok := cons.Unwrap().([]interface{}); ok {
consStrings := make([]string, len(consArr))
for i, c := range consArr {
consStrings[i] = fmt.Sprintf("%s", c)
}
app.AddConstraint(consStrings...)
}
}
for _, port := range data.GetArray("ports") {
app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0)))
}
app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app
for k, v := range data.GetMap("environment") {
app.AddEnv(k, fmt.Sprintf("%s", v.Unwrap()))
}
app.AddUris(data.GetString("package-uri"))
// todo: в манифесте задавать массив healthchecks, их использовтаь в марафоне и консул-супервизоре
// todo: открыть сетевой доступ от марафона до мезос-агентов, чтобы марафон мог хелсчеки посылать
//if portArgs != "" {
// health := marathon.NewDefaultHealthCheck()
// health.Protocol = "TCP"
// health.IntervalSeconds = 5
// *health.PortIndex = 0
// app.AddHealthCheck(*health)
//}
if _, err := marathonApi.UpdateApplication(app, false); err != nil {
color.Yellow("marathon <- %s", app)
return err
}
color.Green("marathon <- %s", app)
consulApi, err := utils.ConsulClient(data.GetString("consul-address"))
if err != nil {
return err
}
if err := utils.RegisterPluginData("deploy.marathon", data.GetString("app-name"), data.String(), data.GetString("consul-address")); err != nil {
return err
}
return backoff.Retry(func() error {
services, _, err := consulApi.Health().Service(fullName, "", true, nil)
if err != nil {
log.Println(color.RedString("Error in check health in consul: %v", err))
return err
}
if len(services) == 0 {
log.Printf("Service `%s` not started yet! Retry...", fullName)
return fmt.Errorf("Service `%s` not started!", fullName)
}
log.Println(color.GreenString("Service `%s` successfully started!", fullName))
return nil
}, backoff.NewExponentialBackOff())
}
func (p DeployTarball) Install(data manifest.Manifest) error {
tmp := "tarball-" + utils.RandomString(16)
dest := data.GetString("install-root") + "/" + data.GetString("package-name")
hooks := make([]string, 0)
for _, hook := range data.GetArray("hooks") {
if hook.Has("postinstall") {
hooks = append(hooks, "sudo "+hook.GetString("postinstall"))
}
}
hookCmd := strings.Join(hooks, " && ")
if hookCmd != "" {
hookCmd = " && " + hookCmd
}
/**
* todo: register package in consul services catalog
*/
if err := utils.RunParallelSshCmd(
data.GetString("cluster"),
data.GetString("ssh-user"),
fmt.Sprint(
"curl -vsSf -o /tmp/"+tmp+".tar.gz "+data.GetString("package-uri"),
" && rm -rf /tmp/"+tmp+"/",
" && mkdir -p /tmp/"+tmp+"/",
" && tar xzf /tmp/"+tmp+".tar.gz -C /tmp/"+tmp+"/",
" && sudo rm -rf "+dest,
" && sudo mkdir -p "+dest,
" && sudo mv /tmp/"+tmp+"/* "+dest+"/",
" && sudo chown -R "+data.GetString("user")+":"+data.GetString("group")+" "+dest+"/",
" && rm -rf /tmp/"+tmp+".tar.gz /tmp/"+tmp+"/",
hookCmd,
),
50,
); err != nil {
return err
}
return utils.RegisterPluginData("deploy.tarball", data.GetString("package-name"), data.String(), data.GetString("consul-address"))
}
func (p DeployTarball) Uninstall(data manifest.Manifest) error {
if err := utils.RunParallelSshCmd(
data.GetString("cluster"),
data.GetString("ssh-user"),
fmt.Sprint("sudo rm -rf "+data.GetString("install-root")+"/"+data.GetString("package-name")),
50,
); err != nil {
return err
}
return utils.DeletePluginData("deploy.tarball", data.GetString("package-name"), data.GetString("consul-address"))
}
func (p DeployDebian) Install(data manifest.Manifest) error {
if err := utils.RunParallelSshCmd(
data.GetString("cluster"),
data.GetString("ssh-user"),
fmt.Sprintf("sudo %s/debian-way/deploy.sh --package='%s' --version='%s'", data.GetString("ci-tools-path"), data.GetString("package"), data.GetString("version")),
data.GetIntOr("parallel", 1),
); err != nil {
return err
}
return utils.RegisterPluginData("deploy.debian", data.GetString("app-name"), data.String(), data.GetString("consul-address"))
}
func (p ReleaseHttp) Run(data manifest.Manifest) error {
if !data.Has("routes") {
log.Println("No routes configured for release.")
return nil
}
consul, err := utils.ConsulClient(data.GetString("consul-address"))
if err != nil {
return err
}
fullName := data.GetString("full-name")
// check current service is alive and healthy
if err := backoff.Retry(func() error {
services, _, err := consul.Health().Service(fullName, "", true, nil)
if err != nil {
log.Println(color.RedString("Error in check health in consul: %v", err))
return err
}
if len(services) == 0 {
log.Printf("Service `%s` not started yet! Retry...", fullName)
return fmt.Errorf("Service `%s` not started!", fullName)
} else {
log.Printf("Service `%s` started with %v instances.", fullName, len(services))
return nil
}
}, backoff.NewExponentialBackOff()); err != nil {
return err
}
routeVars := make(map[string]string, 0)
if data.Has("route") {
if err := json.Unmarshal([]byte(data.GetString("route")), &routeVars); err != nil {
log.Println(color.RedString("Error parse route json: %v, %s", err, data.GetString("route")))
return err
}
}
// collect routes
routes := make([]map[string]string, 0)
for _, route := range data.GetArray("routes") {
if !route.Has("host") {
log.Printf("Not found 'host': %s, skip...", route.String())
continue
}
fields := make(map[string]string)
for k, v := range route.Unwrap().(map[string]interface{}) {
fields[k] = fmt.Sprintf("%v", v)
}
routes = append(routes, utils.MergeMaps(fields, routeVars))
}
if len(routes) == 0 {
log.Println("No routes configured for release.")
return nil
}
routesJson, err := json.MarshalIndent(routes, "", " ")
if err != nil {
return err
}
// write routes to consul kv
if err := utils.PutConsulKv(consul, "services/routes/"+fullName, string(routesJson)); err != nil {
return err
}
log.Println(color.GreenString("Service `%s` released with routes: %s", fullName, string(routesJson)))
// find old services with the same routes
existsRoutes, err := utils.ListConsulKv(consul, "services/routes/"+data.GetString("name-prefix"), nil)
if err != nil {
return err
}
for _, existsRoute := range existsRoutes {
if existsRoute.Key != fmt.Sprintf("services/routes/%s", fullName) { // skip current service
oldRoutes := make([]map[string]string, 0)
if err := json.Unmarshal(existsRoute.Value, &oldRoutes); err != nil {
return err
}
OuterLoop:
for _, route := range routes {
for _, oldRoute := range oldRoutes {
if utils.MapsEqual(route, oldRoute) {
outdated := strings.TrimPrefix(existsRoute.Key, "services/routes/")
log.Println(color.GreenString("Found %s with the same routes %v. Remove it!", outdated, string(existsRoute.Value)))
if err := utils.DelConsulKv(consul, existsRoute.Key); err != nil {
return err
}
if err := utils.MarkAsOutdated(consul, outdated, 10*time.Minute); err != nil {
return err
}
break OuterLoop
}
}
}
}
}
return nil
}
func (p DBCreatePostgresql) Create(data manifest.Manifest) error {
var cmd string
if data.Has("source") {
t := data.GetString("target")
cmd = fmt.Sprintf("sudo -Hu postgres createdb -O %s \"%s\" && sudo -Hu postgres pg_dump \"%s\" | sudo -Hu postgres psql \"%s\"",
data.GetStringOr("db-user", "postgres"), t, data.GetString("source"), t)
} else {
cmd = fmt.Sprintf("sudo -Hu postgres createdb -O %s \"%s\"",
data.GetStringOr("db-user", "postgres"), data.GetString("target"))
}
return utils.RunSingleSshCmd(data.GetString("host"), data.GetString("ssh-user"), cmd)
}