func swarmJoin(name string, image string, token string) error {
store := machine.GetDefaultStore(utils.GetBaseDir())
h, err := loadHost(store, name, utils.GetBaseDir())
if err != nil {
return err
}
url, err := h.GetURL()
if err != nil {
return err
}
ip, err := h.Driver.GetIP()
if err != nil {
return err
}
retryCount := 0
retry:
exec.Command(os.Args[0], []string{
"-H", url,
"--tlscacert=" + h.HostOptions.AuthOptions.CaCertPath,
"--tlscert=" + h.HostOptions.AuthOptions.ClientCertPath,
"--tlskey=" + h.HostOptions.AuthOptions.ClientKeyPath,
"--tlsverify=true",
"rm", "-f", "swarm-agent"}...).Output()
cmd := exec.Command(os.Args[0], []string{
"-H", url,
"--tlscacert=" + h.HostOptions.AuthOptions.CaCertPath,
"--tlscert=" + h.HostOptions.AuthOptions.ClientCertPath,
"--tlskey=" + h.HostOptions.AuthOptions.ClientKeyPath,
"--tlsverify=true",
"run", "-d", "--restart=always",
"--net=bridge",
"--name", "swarm-agent",
image, "join",
"--advertise", ip + ":2376",
token}...)
b, err := cmd.CombinedOutput()
if err == nil {
fmt.Printf("Machine '%s' joined cluster...\n", name)
return nil
} else {
if _, ok := err.(*exec.ExitError); ok {
retryCount++
if retryCount <= 5 {
fmt.Printf("Failed to join '%s'. Retry: %d\n", name, retryCount)
goto retry
}
}
fmt.Println(string(b))
}
return err
}
func DoRmm(cli interface{}, args ...string) error {
cmd := Cli.Subcmd("rmm", []string{"MACHINES"}, "Remove machines", false)
force := cmd.Bool([]string{"f", "-force"}, false, "Force removing machines")
cmd.ParseFlags(args, true)
if len(cmd.Args()) == 0 {
return fmt.Errorf("You must specify a machine or pattern name")
}
isError := false
store := machine.GetDefaultStore(utils.GetBaseDir())
for _, pattern := range cmd.Args() {
for _, hostName := range utils.Generate(pattern) {
h, err := loadHost(store, hostName, utils.GetBaseDir())
if err != nil {
isError = true
log.Errorf("Error removing machine %s: %s", hostName, err)
}
if err := h.Driver.Remove(); err != nil {
if !*force {
isError = true
log.Errorf("Provider error removing machine %q: %s", hostName, err)
continue
}
}
if err := store.Remove(hostName); err != nil {
isError = true
log.Errorf("Error removing machine %q from store: %s", hostName, err)
} else {
log.Infof("Successfully removed %s", hostName)
}
}
}
if isError {
return fmt.Errorf("There was an error removing a machine.")
}
return nil
}
// Usage: gattai provision
func DoProvision(cli interface{}, args ...string) error {
cmd := Cli.Subcmd("provision",
[]string{"PATTERNS"},
"Provision a set of machines. Patterns, e.g. machine-[1:10], are allowed.",
false)
provisionFilename := cmd.String(
[]string{"f", "-file"},
"provision.yml",
"Name of the provision file")
// TODO: EnvVar: "MACHINE_STORAGE_PATH"
machineStoragePath := cmd.String(
[]string{"s", "-storge-path"},
utils.GetBaseDir(),
"Configure Docker Machine's storage path")
quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Do not list machines at the end of provisioning")
cmd.ParseFlags(args, true)
p, err := machine.ReadProvision(*provisionFilename)
if err != nil {
log.Debugf("err: %s", err)
return err
}
// extract pattern
// fmt.Printf("args: %s\n",args)
machineList := p.GetMachineList(cmd.Args()...)
log.Debugf("machines: %s", machineList)
if len(machineList) == 0 {
return errors.New("no machine in list")
}
// create libmachine's store
log.Debugf("storage: %s", *machineStoragePath)
certInfo := machine.GetCertInfo()
authOptions := &auth.AuthOptions{
CertDir: filepath.Join(*machineStoragePath, "certs"),
CaCertPath: certInfo.CaCertPath,
CaPrivateKeyPath: certInfo.CaPrivateKeyPath,
ClientCertPath: certInfo.ClientCertPath,
ClientKeyPath: certInfo.ClientKeyPath,
}
// TODO authOptions :=
if err := cert.BootstrapCertificates(authOptions); err != nil {
log.Fatalf("Error generating certificates: %s", err)
}
store := machine.GetDefaultStore(*machineStoragePath)
spacing := len(machineList) > 1
// check each machine existing
for _, name := range machineList {
parts := strings.SplitN(name, "-", 2)
group := parts[0]
index := -1
if len(parts) > 1 {
// node-master is not a group, but a machine name, for example.
i, err := strconv.Atoi(parts[1])
if err != nil {
group = name
} else {
index = i - 1
}
}
details := p.Machines[group]
if details.BaseAddress != "" {
ip, _, err := net.ParseCIDR(details.BaseAddress)
if err != nil {
return err
}
for i := details.BaseIndex; i <= index; i++ {
utils.IncAddress(ip)
}
os.Setenv("MACHINE_IP", ip.String())
}
if details.PreProvision != nil && len(details.PreProvision) > 0 {
fmt.Println("Processing pre-provision commands...")
for _, pre := range details.PreProvision {
log.Debugf("pre-provision: %s", os.ExpandEnv(pre))
// if strings.HasPrefix(pre, "bash") {
err := executeBash(strings.TrimSpace(os.ExpandEnv(pre)))
if err != nil {
log.Debug(err)
}
// }
}
}
h, err := store.Load(name)
if err != nil {
if _, ok := err.(mcnerror.ErrHostDoesNotExist); ok {
fmt.Printf("Machine '%s' not found, creating...\n", name)
spacing = true
// spew.Dump(hostOptions)
driver, err := driverfactory.NewDriver(details.Driver, name, *machineStoragePath)
if err != nil {
log.Fatalf("Error trying to get driver: %s", err)
}
// TODO populate Env Vars from all hosts
// to use with .SetConfigFromFlags
h, err = store.NewHost(driver)
if err != nil {
log.Fatalf("Error getting new host: %s", err)
}
c := machine.Options(make(map[string]interface{}))
for k, v := range details.Options {
c[k] = v
}
hostOptions := &host.HostOptions{
AuthOptions: &auth.AuthOptions{
CertDir: utils.GetMachineCertDir(),
CaCertPath: certInfo.CaCertPath,
CaPrivateKeyPath: certInfo.CaPrivateKeyPath,
ClientCertPath: certInfo.ClientCertPath,
ClientKeyPath: certInfo.ClientKeyPath,
ServerCertPath: filepath.Join(utils.GetMachineDir(), name, "server.pem"),
ServerKeyPath: filepath.Join(utils.GetMachineDir(), name, "server-key.pem"),
StorePath: filepath.Join(utils.GetMachineDir(), name),
},
EngineOptions: &engine.EngineOptions{
ArbitraryFlags: c.StringSlice("engine-opt"),
Env: c.StringSlice("engine-env"),
InsecureRegistry: c.StringSlice("engine-insecure-registry"),
Labels: c.StringSlice("engine-label"),
RegistryMirror: c.StringSlice("engine-registry-mirror"),
StorageDriver: c.String("engine-storage-driver"),
TlsVerify: true,
InstallURL: c.String("engine-install-url"),
},
SwarmOptions: &swarm.SwarmOptions{
IsSwarm: c.Bool("swarm"),
Image: c.String("swarm-image"),
Master: c.Bool("swarm-master"),
Discovery: c.String("swarm-discovery"),
Address: c.String("swarm-addr"),
Host: c.String("swarm-host"),
Strategy: c.String("swarm-strategy"),
ArbitraryFlags: c.StringSlice("swarm-opt"),
},
}
h.HostOptions = hostOptions
if err := h.Driver.SetConfigFromFlags(details.Options); err != nil {
log.Fatalf("Error setting machine configuration from flags provided: %s", err)
}
// make it compatible with RpcDriver
driverData, err := json.Marshal(h.Driver)
if err != nil {
log.Fatal("Cannot marshal host driver")
}
h.RawDriver = driverData
err = create(store, h, func(hh *host.Host) {
c := machine.Options(make(map[string]interface{}))
for k, v := range details.Options {
c[k] = v
}
kvstoreName := details.NetworkKvstore
log.Debug("Cluster store: " + kvstoreName)
if kvstoreName != "" {
kvstore, err := loadHost(store, kvstoreName, *machineStoragePath)
if err != nil {
panic(err)
}
c, url, err := configureClusterStore(h, kvstore, c)
if err != nil {
panic(err)
} else {
hh.HostOptions.EngineOptions.ArbitraryFlags = c.StringSlice("engine-opt")
saveDiscoveryUrl(url + "/" + kvstoreName)
}
}
})
if err != nil {
log.Errorf("Error creating machine: %s", err)
log.Fatal("You will want to check the provider to make sure the machine and associated resources were properly removed.")
}
// make it compatible with RpcDriver
driverData, err = json.Marshal(h.Driver)
if err != nil {
log.Fatal("Cannot marshal host driver")
}
h.RawDriver = driverData
err = store.Save(h)
if err != nil {
log.Fatalf("Error saving machine: %s", err)
}
}
} else {
fmt.Printf("Machine '%s' exists, starting...\n", name)
h, err = loadHost(store, name, *machineStoragePath)
// TODO reprovision
h.Start()
spacing = false
}
_ = removeAllContainers(h)
// TODO delete all containers during re-provision?
if details.PostProvision != nil && len(details.PostProvision) > 0 {
fmt.Println("Processing post-provision commands...")
for _, post := range details.PostProvision {
log.Debugf("post-provision: %s", post)
if strings.HasPrefix(post, "docker") {
err := engineExecute(h, strings.TrimSpace(post[6:]))
if err != nil {
// if error, goes on
log.Debug(err)
}
}
}
}
if spacing {
if len(machineList) > 1 {
fmt.Println()
}
}
}
if !spacing {
fmt.Println()
}
if *quiet == false {
w := tabwriter.NewWriter(os.Stdout, 5, 1, 3, ' ', 0)
fmt.Fprintln(w, "NAME\tURL\tSTATE")
for _, machineName := range machineList {
h, err := loadHost(store, machineName, utils.GetBaseDir())
items := getHostListItems([]*host.Host{h})
if err == nil {
url, _ := h.GetURL()
fmt.Fprintf(w, "%s\t%s\t%s\n", machineName, url, items[0].State)
}
}
w.Flush()
}
return err
}
// func cmdLs(c *cli.Context) {
func DoLs(cli interface{}, args ...string) error {
// func (cli *DockerCli) CmdLs(args ...string) error {
cmd := Cli.Subcmd("ls", []string{}, "List machines", false)
psFilterArgs := filters.Args{}
quiet := cmd.Bool([]string{"q", "-quiet"}, false, "List quietly")
flFilter := opts.NewListOpts(nil)
cmd.Var(&flFilter, []string{"f", "-filter"}, "Filter output based on conditions provided")
cmd.ParseFlags(args, true)
var err error
// Consolidate all filter flags, and sanity check them.
// They'll get processed in the daemon/server.
for _, f := range flFilter.GetAll() {
if psFilterArgs, err = filters.ParseFlag(f, psFilterArgs); err != nil {
return err
}
}
options := FilterOptions{}
if len(psFilterArgs) > 0 {
options.SwarmName = psFilterArgs["swarm"]
options.DriverName = psFilterArgs["driver"]
options.State = psFilterArgs["state"]
options.Name = psFilterArgs["name"]
}
store := machine.GetDefaultStore(utils.GetBaseDir())
hostList, err := listHosts(store, utils.GetBaseDir()) // TODO
if err != nil {
log.Fatal(err)
}
hostList = filterHosts(hostList, options)
// Just print out the names if we're being quiet
if *quiet {
for _, host := range hostList {
fmt.Println(host.Name)
}
return nil
}
swarmMasters := make(map[string]string)
swarmInfo := make(map[string]string)
w := tabwriter.NewWriter(os.Stdout, 5, 1, 3, ' ', 0)
fmt.Fprintln(w, "NAME\tDRIVER\tSTATE\tURL")
for _, host := range hostList {
swarmOptions := host.HostOptions.SwarmOptions
if swarmOptions.Master {
swarmMasters[swarmOptions.Discovery] = host.Name
}
if swarmOptions.Discovery != "" {
swarmInfo[host.Name] = swarmOptions.Discovery
}
}
items := getHostListItems(hostList)
sortHostListItemsByName(items)
for _, item := range items {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", item.Name, item.DriverName, item.State, item.URL)
}
w.Flush()
return nil
}
func DoSsh(cli interface{}, args ...string) error {
ssh.SetDefaultClient(ssh.External)
cmd := Cli.Subcmd("ssh",
[]string{"MACHINES COMMAND"},
"Run SSH commands on machines specified. Use - to run SSH on the active host.", false)
cmd.ParseFlags(args, true)
store := machine.GetDefaultStore(utils.GetBaseDir())
p, err := machine.ReadProvision("provision.yml")
if err != nil {
log.Debugf("err: %s", err)
return err
}
pattern := cmd.Args()[0]
if pattern == "" {
log.Fatal("Error: Please specify a machine name or pattern.")
}
// TODO if ssh -all
var machineList []string
if pattern == "-" {
name, err := GetActiveHostName()
if err != nil {
return err
}
machineList = []string{name}
} else {
machineList = p.GetMachineList(pattern)
}
if len(machineList) == 1 && len(cmd.Args()) == 1 {
host, err := loadHost(store, machineList[0], utils.GetBaseDir())
if err != nil {
log.Fatal(err)
}
currentState, err := host.Driver.GetState()
if err != nil {
log.Fatal(err)
}
if currentState != state.Running {
log.Fatalf("Error: Cannot run SSH command: Host %q is not running", host.Name)
}
client, err := host.CreateSSHClient()
if err != nil {
log.Fatal(err)
}
if err := client.Shell(cmd.Args()[1:]...); err != nil {
log.Fatal(err)
}
} else {
sshCmd := strings.Join(cmd.Args()[1:], " ")
if strings.TrimSpace(sshCmd) == "" {
return errors.New("Interative shell is not allowed for multiple hosts.")
}
// TODO should limit string channel
limit := len(machineList)
if limit > 4 {
limit = 4
}
outputs := make(chan string, limit)
for _, name := range machineList {
go func(name string) {
host, err := loadHost(store, name, utils.GetBaseDir())
if err != nil {
log.Fatal(err)
}
currentState, err := host.Driver.GetState()
if err != nil {
log.Fatal(err)
}
if currentState != state.Running {
log.Fatalf("Error: Cannot run SSH command: Host %q is not running", host.Name)
}
output, err := host.RunSSHCommand(sshCmd)
if err != nil {
if len(machineList) == 1 {
outputs <- err.Error()
} else {
outputs <- fmt.Sprintf("\n%s:\n%s", name, err.Error())
}
} else {
if len(machineList) == 1 {
outputs <- string(output)
} else {
outputs <- fmt.Sprintf("\n%s:\n%s", name, string(output))
}
}
}(name)
}
for i := 0; i < len(machineList); i++ {
fmt.Print(<-outputs)
}
}
return nil
}
func DoCluster(cli interface{}, args ...string) error {
cmd := Cli.Subcmd("cluster", []string{"MACHINES"}, "Form a cluster with a set of specified machines", false)
master := cmd.String([]string{"m", "-master"}, "", "Configure the cluster masters")
image := cmd.String([]string{"i", "-image"}, "swarm", "Specify Docker Swarm image")
provision := cmd.Bool([]string{"p", "-provision"}, false, "Automatic provision before forming the cluster")
cmd.ParseFlags(args, true)
if *master == "" && len(cmd.Args()) == 0 {
return errors.New("Please specify a set of machines or the cluster master.")
}
// if specify only a group, then master is ${group}-master
if *master == "" && len(cmd.Args()) == 1 {
*master = cmd.Args()[0] + "-master"
}
// do provision if required
if *provision == true {
err := DoProvision(cli, append([]string{"-q", *master}, cmd.Args()...)...)
if err != nil {
return err
}
fmt.Println()
}
// Read existing token
// If not existed, generate one
// TODO support other discoveries
token, err := readToken()
if err != nil {
token, err = generateToken()
if err != nil {
return err
}
}
p, err := machine.ReadProvision("provision.yml")
if err != nil {
log.Debugf("err: %s", err)
return err
}
fmt.Printf("Use discovery: %s\n", token)
// start
for _, machineName := range p.GetMachineList(cmd.Args()...) {
err := swarmJoin(machineName, *image, token)
if err != nil {
return err
}
}
if master != nil {
store := machine.GetDefaultStore(utils.GetBaseDir())
h, err := loadHost(store, *master, utils.GetBaseDir())
if err != nil {
return err
}
err = swarmManage(h, *image, token)
if err != nil {
return err
}
f, err := os.Create(ACTIVE_HOST_FILE)
defer f.Close()
if err != nil {
return err
}
// save config file for cluster
ip, _ := h.Driver.GetIP()
fmt.Fprintf(f, "---\n")
fmt.Fprintf(f, "name: %s\n", h.Name)
fmt.Fprintf(f, "DOCKER_HOST: \"tcp://%s:%d\"\n", ip, 3376)
fmt.Fprintf(f, "DOCKER_CERT_PATH: %s\n", h.HostOptions.AuthOptions.StorePath)
fmt.Fprintf(f, "DOCKER_TLS_VERIFY: 1\n")
fmt.Printf("Active host is now set to '%s' (swarm).\n", h.Name)
}
return nil
}
func startZooKeeper(args ...string) error {
p, err := machine.ReadProvision("provision.yml")
if err != nil {
log.Debugf("err: %s", err)
return err
}
// extract pattern
// fmt.Printf("args: %s\n",args)
machineList := p.GetMachineList(args...)
log.Debugf("machines: %s", machineList)
if len(machineList) == 0 {
return errors.New("no machine in list")
}
store := machine.GetDefaultStore(utils.GetBaseDir())
zooCfg := `
tickTime=2000
dataDir=/var/zookeeper/
clientPort=2181
initLimit=5
syncLimit=2
{{range $id, $ip := .}}server.{{ inc $id }}={{ $ip }}:2888:3888
{{ end }}
`
addHosts := []string{}
hosts := []*host.Host{}
for _, name := range machineList {
h, err := loadHost(store, name, utils.GetBaseDir())
if err != nil {
return err
}
hosts = append(hosts, h)
ip, err := h.Driver.GetIP()
if err != nil {
return err
}
addHosts = append(addHosts, "--add-host", name+":"+ip)
}
tmpl, err := template.New("zoo").Funcs(template.FuncMap{
"inc": func(i int) int { return i + 1 },
}).Parse(zooCfg)
if err != nil {
return err
}
var str bytes.Buffer
err = tmpl.Execute(&str, machineList) // change ips to machine name
if err != nil {
return err
}
for _, h := range hosts {
provisioner, err := provision.DetectProvisioner(h.Driver)
// dockerDir := provisioner.GetDockerOptionsDir()
// authOptions := setRemoteAuthOptions(provisioner)
url, err := h.GetURL()
if err != nil {
return err
}
exec.Command(os.Args[0], []string{
"-H", url,
"--tlscacert=" + h.HostOptions.AuthOptions.CaCertPath,
"--tlscert=" + h.HostOptions.AuthOptions.ClientCertPath,
"--tlskey=" + h.HostOptions.AuthOptions.ClientKeyPath,
"--tlsverify=true",
"rm", "-f", "swarm-discovery-service"}...).Output()
// skip error checking
provisioner.SSHCommand("sudo mkdir -p /opt/zookeeper/conf")
// copy zooCfg
transferCmdFmt := "printf '%%s' '%s' | sudo tee %s"
if _, err := provisioner.SSHCommand(fmt.Sprintf(transferCmdFmt, str.String(), "/opt/zookeeper/conf/zoo.cfg")); err != nil {
return err
}
cmd := exec.Command(os.Args[0], append([]string{
"-H", url,
"--tlscacert=" + h.HostOptions.AuthOptions.CaCertPath,
"--tlscert=" + h.HostOptions.AuthOptions.ClientCertPath,
"--tlskey=" + h.HostOptions.AuthOptions.ClientKeyPath,
"--tlsverify=true",
"run", "-d", "--restart=always",
"--name", "swarm-discovery-service",
"-v", "/opt/zookeeper/conf:/opt/zookeeper/conf"},
append(addHosts,
"-p", "2181:2181",
"-p", "2888:2888",
"-p", "3888:3888",
"jplock/zookeeper")...)...)
output, err := cmd.CombinedOutput()
if err == nil {
fmt.Printf("ZK '%s' started successfully...\n", h.Name)
}
if err != nil {
fmt.Print(string(output))
return err
}
}
return nil
}
