// RemoteSshCommand invokes the given command on a host and port
func RemoteSshCommand(user string, privateKey string, hostPort string, cmd string) error {
sshConfig := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
PublicKeyFile(privateKey),
},
}
if sshConfig == nil {
log.Info("Whoah!")
}
connection, err := ssh.Dial("tcp", hostPort, sshConfig)
if err != nil {
return fmt.Errorf("Failed to dial: %s", err)
}
session, err := connection.NewSession()
if err != nil {
return fmt.Errorf("Failed to create session: %s", err)
}
defer session.Close()
modes := ssh.TerminalModes{
// ssh.ECHO: 0, // disable echoing
ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
}
if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
return fmt.Errorf("Request for pseudo terminal failed: %s", err)
}
stdin, err := session.StdinPipe()
if err != nil {
return fmt.Errorf("Unable to setup stdin for session: %v", err)
}
go io.Copy(stdin, os.Stdin)
stdout, err := session.StdoutPipe()
if err != nil {
return fmt.Errorf("Unable to setup stdout for session: %v", err)
}
go io.Copy(os.Stdout, stdout)
stderr, err := session.StderrPipe()
if err != nil {
return fmt.Errorf("Unable to setup stderr for session: %v", err)
}
go io.Copy(os.Stderr, stderr)
log.Info("Running command %s", cmd)
err = session.Run(cmd)
if err != nil {
return fmt.Errorf("Failed to run command: "+cmd+": %v", err)
}
return nil
}
func deletePodsForOldHosts(c *client.Client, ns string, annotations map[string]string, pods *api.PodList, hostEntries []HostEntry) {
for annKey, podName := range annotations {
if strings.HasPrefix(annKey, AnsibleHostPodAnnotationPrefix) {
hostName := annKey[len(AnsibleHostPodAnnotationPrefix):]
if k8s.PodIsRunning(pods, podName) {
hostEntry := getHostEntryByName(hostEntries, hostName)
if hostEntry == nil {
log.Info("Deleting pod %s as there is no longer an Ansible inventory host called %s", podName, hostName)
c.Pods(ns).Delete(podName, nil)
}
}
}
}
}
func run(c *cli.Context) {
log.Info("Running GoSupervise!")
port, err := osExpandAndVerifyGlobal(c, "port")
if err != nil {
fail(err)
}
command, err := osExpandAndVerify(c, "command")
if err != nil {
fail(err)
}
host, err := osExpandAndVerify(c, "host")
if err != nil {
fail(err)
}
user, err := osExpandAndVerify(c, "user")
if err != nil {
fail(err)
}
useWinRM := c.Bool("winrm")
if useWinRM {
password, err := osExpandAndVerify(c, "password")
if err != nil {
fail(err)
}
err = winrm.RemoteWinRmCommand(user, password, host, port, command)
} else {
privatekey, err := osExpandAndVerify(c, "privatekey")
if err != nil {
fail(err)
}
hostPort := host + ":" + port
err = ssh.RemoteSshCommand(user, privatekey, hostPort, command)
}
if err != nil {
log.Err("Failed: %v", err)
}
}
// ChooseHostAndPrivateKey parses the given Ansbile inventory file for the hosts
// and chooses a single host inside it, returning the host name and the private key
func ChooseHostAndPrivateKey(inventoryFile string, hosts string, c *client.Client, ns string, rcName string) (*HostEntry, error) {
hostEntries, err := LoadHostEntries(inventoryFile, hosts)
if err != nil {
return nil, err
}
log.Info("Found %d host entries", len(hostEntries))
// lets pick a random entry
if len(hostEntries) > 0 {
thisPodName := os.Getenv("HOSTNAME")
if len(thisPodName) == 0 {
return nil, fmt.Errorf("Could not find the pod name using $HOSTNAME!")
}
retryAttempts := 20
for i := 0; i < retryAttempts; i++ {
if i > 0 {
// lets sleep before retrying
time.Sleep(time.Duration(random(1000, 20000)) * time.Millisecond)
}
if c == nil {
return nil, fmt.Errorf("No Kubernetes Client specified!")
}
rc, err := c.ReplicationControllers(ns).Get(rcName)
if err != nil {
return nil, err
}
if rc == nil {
return nil, fmt.Errorf("No ReplicationController found for name %s", rcName)
}
pods, err := c.Pods(ns).List(nil, nil)
if err != nil {
return nil, err
}
metadata := &rc.ObjectMeta
resourceVersion := metadata.ResourceVersion
if metadata.Annotations == nil {
metadata.Annotations = make(map[string]string)
}
annotations := metadata.Annotations
log.Info("found RC with name %s.%s and version %s", ns, rcName, resourceVersion)
filteredHostEntries := hostEntries
for annKey, podName := range annotations {
if strings.HasPrefix(annKey, AnsibleHostPodAnnotationPrefix) {
hostName := annKey[len(AnsibleHostPodAnnotationPrefix):]
if k8s.PodIsRunning(pods, podName) {
if podName != thisPodName {
log.Info("Pod %s podName has already claimed host %s", podName, hostName)
filteredHostEntries = removeHostEntry(filteredHostEntries, hostName)
}
} else {
// lets remove this annotation as the pod is no longer valid
log.Info("Pod %s is no longer running so removing the annotation %s", podName, annKey)
delete(metadata.Annotations, annKey)
}
}
}
count := len(filteredHostEntries)
if count == 0 {
log.Info("There are no more hosts available to be supervised by this pod!")
return nil, fmt.Errorf("No more hosts available to be supervised!")
}
log.Info("After filtering out hosts owned by other pods we have %v host entries left", count)
pickedEntry := filteredHostEntries[random(0, count)]
hostName := pickedEntry.Name
if len(pickedEntry.Host) == 0 {
return nil, fmt.Errorf("Could not find host name for entry %s", pickedEntry.Name)
}
if len(pickedEntry.PrivateKey) == 0 {
return nil, fmt.Errorf("Could not find PrivateKey for entry %s", pickedEntry.Name)
}
if len(pickedEntry.User) == 0 {
return nil, fmt.Errorf("Could not find User for entry %s", pickedEntry.Name)
}
// lets try pick this pod
annotations[AnsibleHostPodAnnotationPrefix+hostName] = thisPodName
_, err = c.ReplicationControllers(ns).Update(rc)
if err != nil {
log.Info("Failed to update the RC, could be concurrent update failure: %s", err)
} else {
log.Info("Picked host " + pickedEntry.Host)
return &pickedEntry, nil
}
}
}
return nil, fmt.Errorf("Could not find any hosts for inventory file %s and hosts %s", inventoryFile, hosts)
}
func UpdateAnsibleRC(inventoryFile string, hosts string, c *client.Client, ns string, rcFile string) (*api.ReplicationController, error) {
rcConfig, err := k8s.ReadReplicationControllerFromFile(rcFile)
if err != nil {
return nil, err
}
gitUrl, err := findGitUrl()
if err != nil {
return nil, err
}
if len(gitUrl) == 0 {
return nil, fmt.Errorf("Could not find git URL in git configu file %s", gitConfig)
}
podSpec := k8s.GetOrCreatePodSpec(rcConfig)
container := k8s.GetFirstContainerOrCreate(rcConfig)
if len(container.Image) == 0 {
container.Image = "fabric8/gosupervise"
}
if len(container.Name) == 0 {
container.Name = "gosupervise"
}
if len(container.ImagePullPolicy) == 0 {
container.ImagePullPolicy = "IfNotPresent"
}
k8s.EnsureContainerHasEnvVar(container, EnvHosts, hosts)
command := k8s.GetContainerEnvVar(container, EnvCommand)
if len(command) == 0 {
return nil, fmt.Errorf("No environemnt variable value defined for %s in ReplicationController YAML file %s", EnvCommand, rcFile)
}
volumeName := "playbook-volume"
k8s.EnsurePodSpecHasGitVolume(podSpec, volumeName, gitUrl, "master")
k8s.EnsureContainerHasGitVolumeMount(container, volumeName, PlaybookVolumeMount)
hostEntries, err := LoadHostEntries(inventoryFile, hosts)
if err != nil {
return nil, err
}
log.Info("Found %d host entries in the Ansible inventory for %s", len(hostEntries), hosts)
log.Info("Using git URL %s", gitUrl)
rcName := rcConfig.ObjectMeta.Name
isUpdate := true
rc, err := c.ReplicationControllers(ns).Get(rcName)
if err != nil {
isUpdate = false
rc = &api.ReplicationController{
ObjectMeta: api.ObjectMeta{
Namespace: ns,
Name: rcName,
},
}
}
pods, err := c.Pods(ns).List(nil, nil)
if err != nil {
return nil, err
}
// merge the RC configuration to allow configuration
rc.Spec = rcConfig.Spec
metadata := rc.ObjectMeta
resourceVersion := metadata.ResourceVersion
annotations := metadata.Annotations
rcSpec := &rc.Spec
rcSpec.Replicas = len(hostEntries)
log.Info("found RC with name %s and version %s and replicas %d", rcName, resourceVersion, rcSpec.Replicas)
deletePodsForOldHosts(c, ns, annotations, pods, hostEntries)
replicationController := c.ReplicationControllers(ns)
if isUpdate {
_, err = replicationController.Update(rc)
} else {
_, err = replicationController.Create(rc)
}
if err != nil {
log.Info("Failed to update the RC, could be concurrent update failure: %s", err)
return nil, err
}
return rc, nil
}
func runAnsiblePod(c *cli.Context) {
args := c.Args()
if len(args) < 2 {
log.Die("Expected at least 2 arguments!")
}
hosts := args[0]
command := strings.Join(args[1:], " ")
log.Info("running command on a host from %s and command `%s`", hosts, command)
f := cmdutil.NewFactory(nil)
if f == nil {
log.Die("Failed to create Kuberentes client factory!")
}
kubeclient, _ := f.Client()
if kubeclient == nil {
log.Die("Failed to create Kuberentes client!")
}
ns, _, _ := f.DefaultNamespace()
if len(ns) == 0 {
ns = "default"
}
rcFile, err := osExpandAndVerify(c, "rc")
if err != nil {
fail(err)
}
port, err := osExpandAndVerifyGlobal(c, "port")
if err != nil {
fail(err)
}
inventory, err := osExpandAndVerify(c, "inventory")
if err != nil {
fail(err)
}
rc, err := k8s.ReadReplicationControllerFromFile(rcFile)
if err != nil {
fail(err)
}
rcName := rc.ObjectMeta.Name
if len(rcName) == 0 {
log.Die("No ReplicationController name in the yaml file %s", rcFile)
}
hostEntry, err := ansible.ChooseHostAndPrivateKey(inventory, hosts, kubeclient, ns, rcName)
if err != nil {
fail(err)
}
host := hostEntry.Host
user := hostEntry.User
useWinRM := c.Bool("winrm") || hostEntry.UseWinRM
if useWinRM {
log.Info("Using WinRM to connect to the hosts %s", hosts)
password, err := osExpandAndVerify(c, "password")
if err != nil {
fail(err)
}
err = winrm.RemoteWinRmCommand(user, password, host, port, command)
} else {
privatekey := hostEntry.PrivateKey
hostPort := host + ":" + port
err = ssh.RemoteSshCommand(user, privatekey, hostPort, command)
}
if err != nil {
log.Err("Failed: %v", err)
}
}