// Push pushes an image to the registry.
//
// This finds the appropriate registry by looking it up in etcd.
//
// Params:
// - client (etcd.Getter): Client to do etcd lookups.
// - tag (string): Tag to push.
//
// Returns:
//
func Push(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
// docker tag deis/slugrunner:lastest HOST:PORT/deis/slugrunner:latest
// docker push HOST:PORT/deis/slugrunner:latest
client := p.Get("client", nil).(etcd.Getter)
host, err := client.Get("/deis/registry/host", false, false)
if err != nil || host.Node == nil {
return nil, err
}
port, err := client.Get("/deis/registry/port", false, false)
if err != nil || host.Node == nil {
return nil, err
}
registry := host.Node.Value + ":" + port.Node.Value
tag := p.Get("tag", "").(string)
log.Infof(c, "Pushing %s to %s. This may take some time.", tag, registry)
rem := path.Join(registry, tag)
out, err := exec.Command("docker", "tag", "-f", tag, rem).CombinedOutput()
if err != nil {
log.Warnf(c, "Failed to tag %s on host %s: %s (%s)", tag, rem, err, out)
}
out, err = exec.Command("docker", "-D", "push", rem).CombinedOutput()
if err != nil {
log.Warnf(c, "Failed to push %s to host %s: %s (%s)", tag, rem, err, out)
return nil, err
}
log.Infof(c, "Finished pushing %s to %s.", tag, registry)
return nil, nil
}
// createRepo creates a new Git repo if it is not present already.
//
// Largely inspired by gitreceived from Flynn.
//
// Returns a bool indicating whether a project was created (true) or already
// existed (false).
func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) {
createLock.Lock()
defer createLock.Unlock()
fi, err := os.Stat(repoPath)
if err == nil && fi.IsDir() {
// Nothing to do.
log.Infof(c, "Directory %s already exists.", repoPath)
return false, nil
} else if os.IsNotExist(err) {
log.Infof(c, "Creating new directory at %s", repoPath)
// Create directory
if err := os.MkdirAll(repoPath, 0755); err != nil {
log.Warnf(c, "Failed to create repository: %s", err)
return false, err
}
cmd := exec.Command("git", "init", "--bare")
cmd.Dir = repoPath
if out, err := cmd.CombinedOutput(); err != nil {
log.Warnf(c, "git init output: %s", out)
return false, err
}
hook, err := prereceiveHook(map[string]string{"GitHome": gitHome})
if err != nil {
return true, err
}
ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755)
return true, nil
} else if err == nil {
return false, errors.New("Expected directory, found file.")
}
return false, err
}
// FindSSHUser finds an SSH user by public key.
//
// Some parts of the system require that we know not only the SSH key, but also
// the name of the user. That information is stored in etcd.
//
// Params:
// - client (EtcdGetter)
// - fingerprint (string): The fingerprint of the SSH key.
//
// Returns:
// - username (string)
func FindSSHUser(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
client := p.Get("client", nil).(Getter)
fingerprint := p.Get("fingerprint", nil).(string)
res, err := client.Get("/deis/builder/users", false, true)
if err != nil {
log.Warnf(c, "Error querying etcd: %s", err)
return "", err
} else if res.Node == nil || !res.Node.Dir {
log.Warnf(c, "No users found in etcd.")
return "", errors.New("Users not found")
}
for _, user := range res.Node.Nodes {
log.Infof(c, "Checking user %s", user.Key)
for _, keyprint := range user.Nodes {
if strings.HasSuffix(keyprint.Key, fingerprint) {
parts := strings.Split(user.Key, "/")
username := parts[len(parts)-1]
log.Infof(c, "Found user %s for fingerprint %s", username, fingerprint)
return username, nil
}
}
}
return "", fmt.Errorf("User not found for fingerprint %s", fingerprint)
}
// createRepo creates a new Git repo if it is not present already.
//
// Largely inspired by gitreceived from Flynn.
//
// Returns a bool indicating whether a project was created (true) or already
// existed (false).
func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) {
createLock.Lock()
defer createLock.Unlock()
if fi, err := os.Stat(repoPath); err == nil {
if fi.IsDir() {
configPath := filepath.Join(repoPath, "config")
if _, cerr := os.Stat(configPath); cerr == nil {
log.Debugf(c, "Directory '%s' already exists.", repoPath)
return true, nil
} else {
log.Warnf(c, "No config file found at `%s`; removing it and recreating.", repoPath)
if err := os.RemoveAll(repoPath); err != nil {
return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err)
}
}
} else {
log.Warnf(c, "Path '%s' is not a directory; removing it and recreating.", repoPath)
if err := os.RemoveAll(repoPath); err != nil {
return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err)
}
}
} else if os.IsNotExist(err) {
log.Debugf(c, "Unable to get stat for path '%s': %s .", repoPath, err)
} else {
return false, err
}
return initRepo(repoPath, gitHome, c)
}
// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// - operation (string): e.g. git-receive-pack
// - repoName (string): The repository name, in the form '/REPO.git'.
// - channel (ssh.Channel): The channel.
// - request (*ssh.Request): The channel.
// - gitHome (string): Defaults to /home/git.
// - fingerprint (string): The fingerprint of the user's SSH key.
// - user (string): The name of the Deis user.
//
// Returns:
// - nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
return nil, fmt.Errorf("Missing requirements %q", z)
}
repoName := p.Get("repoName", "").(string)
operation := p.Get("operation", "").(string)
channel := p.Get("channel", nil).(ssh.Channel)
gitHome := p.Get("gitHome", "/home/git").(string)
fingerprint := p.Get("fingerprint", nil).(string)
user := p.Get("user", "").(string)
repo, err := cleanRepoName(repoName)
if err != nil {
log.Warnf(c, "Illegal repo name: %s.", err)
channel.Stderr().Write([]byte("No repo given"))
return nil, err
}
repo += ".git"
if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil {
log.Infof(c, "Did not create new repo: %s", err)
}
cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
log.Infof(c, strings.Join(cmd.Args, " "))
var errbuff bytes.Buffer
cmd.Dir = gitHome
cmd.Env = []string{
fmt.Sprintf("RECEIVE_USER=%s", user),
fmt.Sprintf("RECEIVE_REPO=%s", repo),
fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
}
cmd.Env = append(cmd.Env, os.Environ()...)
done := plumbCommand(cmd, channel, &errbuff)
if err := cmd.Start(); err != nil {
log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes())
return nil, err
}
fmt.Printf("Waiting for git-receive to run.\n")
done.Wait()
fmt.Printf("Waiting for deploy.\n")
if err := cmd.Wait(); err != nil {
log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes())
return nil, err
}
if errbuff.Len() > 0 {
log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
}
log.Infof(c, "Deploy complete.\n")
return nil, nil
}
// initRepo create a directory and init a new Git repo
func initRepo(repoPath, gitHome string, c cookoo.Context) (bool, error) {
log.Infof(c, "Creating new directory at %s", repoPath)
// Create directory
if err := os.MkdirAll(repoPath, 0755); err != nil {
log.Warnf(c, "Failed to create repository: %s", err)
return false, err
}
cmd := exec.Command("git", "init", "--bare")
cmd.Dir = repoPath
if out, err := cmd.CombinedOutput(); err != nil {
log.Warnf(c, "git init output: %s", out)
return false, err
}
hook, err := prereceiveHook(map[string]string{"GitHome": gitHome})
if err != nil {
return true, err
}
ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755)
return true, nil
}
// iam injects info into the environment about a host's self.
//
// Sets the following environment variables. (Values represent the data format.
// Instances will get its own values.)
//
// MY_NODEIP=10.245.1.3
// MY_SERVICE_IP=10.22.1.4
// MY_PORT_PEER=2380
// MY_PORT_CLIENT=2379
// MY_NAMESPACE=default
// MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp
// MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272
// MY_APISERVER=https://10.247.0.1:443
// MY_NAME=deis-etcd-1-336jp
// MY_IP=10.246.44.7
// MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON
// MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON
// MY_PORT_CLIENT=4100
// MY_PORT_PEER=2380
func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
me, err := aboutme.FromEnv()
if err != nil {
log.Errf(c, "Failed aboutme.FromEnv: %s", err)
log.Warn(c, "Attempting to recover.")
}
// This will try to recover whenever IP is not set. Only some fields
// can be recovered. But all we really need is IP and Name.
if strings.TrimSpace(me.IP) == "" {
log.Warn(c, "No IP found by API query.")
ip, err := aboutme.MyIP()
if err != nil || ip == "" {
// Force pod death.
log.Errf(c, "Failed to get an IP address: %s", err)
os.Exit(5)
}
me.IP = ip
}
if strings.TrimSpace(me.Name) == "" {
// Try to set name from DAPI.
me.Name = os.Getenv("POD_NAME")
log.Warnf(c, "Setting name to %q", me.Name)
}
if strings.TrimSpace(me.Namespace) == "" {
// Try to set namespace from DAPI.
me.Namespace = os.Getenv("POD_NAMESPACE")
log.Warnf(c, "Setting name to %q", me.Namespace)
}
me.ShuntEnv()
os.Setenv("ETCD_NAME", me.Name)
c.Put("ETCD_NAME", me.Name)
passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT")
passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER")
return nil, nil
}
// setJoinMode determines what mode to start the etcd server in.
//
// In discovery mode, this will use the discovery URL to join a new cluster.
// In "existing" mode, this will join to an existing cluster directly.
//
// Params:
// - client (etcd.Getter): initialized etcd client
// - path (string): path to get. This will go through os.ExpandEnv().
// - desiredLen (string): The number of nodes to expect in etcd. This is
// usually stored as a string.
//
// Returns:
// string "existing" or "new"
func setJoinMode(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
cli := p.Get("client", nil).(client.Client)
dlen := p.Get("desiredLen", "3").(string)
path := p.Get("path", "").(string)
path = os.ExpandEnv(path)
state := "existing"
dint, err := strconv.Atoi(dlen)
if err != nil {
log.Warnf(c, "Expected integer length, got '%s'. Defaulting to 3", dlen)
dint = 3
}
// Ideally, this should look in the /deis/status directory in the discovery
// service. That will indicate how many members have been online in the
// last few hours. This is a good indicator of whether a cluster exists,
// even if not all the hosts are healthy.
res, err := etcd.SimpleGet(cli, path, true)
if err != nil {
// This means that the discovery server is brand new, and nothing
// has written to it yet. So we're new.
if strings.Contains(err.Error(), "Key not found") {
return "new", nil
}
log.Errf(c, "Failed simple get of %s: %s", path, err)
return state, err
}
if !res.Node.Dir {
//return state, errors.New("Expected a directory node in discovery service")
log.Info(c, "No status information found in discovery service. Assuming new.")
state = "new"
} else if len(res.Node.Nodes) < dint {
log.Info(c, "Cluster has not reached consensus number. Assuming new.")
state = "new"
}
os.Setenv("ETCD_INITIAL_CLUSTER_STATE", state)
return state, nil
}
// Get gets one or more environment variables and puts them into the context.
//
// Parameters passed in are of the form varname => defaultValue.
//
// r.Route("foo", "example").Does(envvar.Get).Using("HOME").WithDefault(".")
//
// As with all environment variables, the default value must be a string.
//
// WARNING: Since parameters are a map, order of processing is not
// guaranteed. If order is important, you'll need to call this command
// multiple times.
//
// For each parameter (`Using` clause), this command will look into the
// environment for a matching variable. If it finds one, it will add that
// variable to the context. If it does not find one, it will expand the
// default value (so you can set a default to something like "$HOST:$PORT")
// and also put the (unexpanded) default value back into the context in case
// any subsequent call to `os.Getenv` occurs.
func Get(c cookoo.Context, params *cookoo.Params) (interface{}, cookoo.Interrupt) {
for name, def := range params.AsMap() {
var val string
if val = os.Getenv(name); len(val) == 0 {
if def == nil {
def = ""
}
def, ok := def.(string)
if !ok {
log.Warnf(c, "Could not convert %s. Type is %T", name, def)
}
val = os.ExpandEnv(def)
// We want to make sure that any subsequent calls to Getenv
// return the same default.
os.Setenv(name, val)
}
c.Put(name, val)
log.Debugf(c, "Name: %s, Val: %s", name, val)
}
return true, nil
}
// listen handles accepting and managing connections. However, since closer
// is len(1), it will not block the sender.
func (s *server) listen(l net.Listener, conf *ssh.ServerConfig, closer chan interface{}) error {
cxt := s.c
log.Info(cxt, "Accepting new connections.")
defer l.Close()
// FIXME: Since Accept blocks, closer may not be checked often enough.
for {
log.Info(cxt, "Checking closer.")
if len(closer) > 0 {
<-closer
log.Info(cxt, "Shutting down SSHD listener.")
return nil
}
conn, err := l.Accept()
if err != nil {
log.Warnf(cxt, "Error during Accept: %s", err)
// We shouldn't kill the listener because of an error.
return err
}
safely.GoDo(cxt, func() {
s.handleConn(conn, conf)
})
}
}
// answer handles answering requests and channel requests
//
// Currently, an exec must be either "ping", "git-receive-pack" or
// "git-upload-pack". Anything else will result in a failure response. Right
// now, we leave the channel open on failure because it is unclear what the
// correct behavior for a failed exec is.
//
// Support for setting environment variables via `env` has been disabled.
func (s *server) answer(channel ssh.Channel, requests <-chan *ssh.Request, sshConn string) error {
defer channel.Close()
// Answer all the requests on this connection.
for req := range requests {
ok := false
// I think that ideally what we want to do here is pass this on to
// the Cookoo router and let it handle each Type on its own.
switch req.Type {
case "env":
o := &EnvVar{}
ssh.Unmarshal(req.Payload, o)
fmt.Printf("Key='%s', Value='%s'\n", o.Name, o.Value)
req.Reply(true, nil)
case "exec":
clean := cleanExec(req.Payload)
parts := strings.SplitN(clean, " ", 2)
router := s.c.Get("cookoo.Router", nil).(*cookoo.Router)
// TODO: Should we unset the context value 'cookoo.Router'?
// We need a shallow copy of the context to avoid race conditions.
cxt := s.c.Copy()
cxt.Put("SSH_CONNECTION", sshConn)
// Only allow commands that we know about.
switch parts[0] {
case "ping":
cxt.Put("channel", channel)
cxt.Put("request", req)
sshPing := cxt.Get("route.sshd.sshPing", "sshPing").(string)
err := router.HandleRequest(sshPing, cxt, true)
if err != nil {
log.Warnf(s.c, "Error pinging: %s", err)
}
return err
case "git-receive-pack", "git-upload-pack":
if len(parts) < 2 {
log.Warn(s.c, "Expected two-part command.\n")
req.Reply(ok, nil)
break
}
req.Reply(true, nil) // We processed. Yay.
cxt.Put("channel", channel)
cxt.Put("request", req)
cxt.Put("operation", parts[0])
cxt.Put("repository", parts[1])
sshGitReceive := cxt.Get("route.sshd.sshGitReceive", "sshGitReceive").(string)
err := router.HandleRequest(sshGitReceive, cxt, true)
var xs uint32
if err != nil {
log.Errf(s.c, "Failed git receive: %v", err)
xs = 1
}
sendExitStatus(xs, channel)
return nil
default:
log.Warnf(s.c, "Illegal command is '%s'\n", clean)
req.Reply(false, nil)
return nil
}
if err := sendExitStatus(0, channel); err != nil {
log.Errf(s.c, "Failed to write exit status: %s", err)
}
return nil
default:
// We simply ignore all of the other cases and leave the
// channel open to take additional requests.
log.Infof(s.c, "Received request of type %s\n", req.Type)
req.Reply(false, nil)
}
}
return nil
}
// RemoveStaleMembers deletes cluster members whose pods are no longer running.
//
// This queries Kubernetes to determine what etcd pods are running, and then
// compares that to the member list in the etcd cluster. It removes any
// cluster members who are no longer in the pod list.
//
// The purpose of this is to keep the cluster membership from deadlocking
// when inactive members prevent consensus building.
//
// Params:
// - client (etcd/client.Client): The etcd client
// - label (string): The pod label indicating an etcd node
// - namespace (string): The namespace we're operating in
func RemoveStaleMembers(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
eclient := p.Get("client", nil).(client.Client)
label := p.Get("label", "name=deis-etcd-1").(string)
ns := p.Get("namespace", "default").(string)
// Should probably pass in the client from the context.
klient, err := k8s.PodClient()
if err != nil {
log.Errf(c, "Could not create a Kubernetes client: %s", err)
return nil, err
}
mapi := client.NewMembersAPI(eclient)
members := map[string]bool{}
idmap := map[string]string{}
// Get members from etcd
mm, err := mapi.List(dctx())
if err != nil {
log.Warnf(c, "Could not get a list of etcd members: %s", err)
return nil, err
}
for _, member := range mm {
members[member.Name] = false
idmap[member.Name] = member.ID
}
// Get the pods running with the given label
labelSelector, err := labels.Parse(label)
if err != nil {
log.Errf(c, "Selector failed to parse: %s", err)
return nil, err
}
pods, err := klient.Pods(ns).List(api.ListOptions{LabelSelector: labelSelector})
if err != nil {
return nil, err
}
for _, item := range pods.Items {
if _, ok := members[item.Name]; !ok {
log.Infof(c, "Etcd pod %s is not in cluster yet.", item.Name)
} else {
members[item.Name] = true
}
}
// Anything marked false in members should be removed from etcd.
deleted := 0
for k, v := range members {
if !v {
log.Infof(c, "Deleting %s (%s) from etcd cluster members", k, idmap[k])
if err := mapi.Remove(dctx(), idmap[k]); err != nil {
log.Errf(c, "Failed to remove %s from cluster. Skipping. %s", k, err)
} else {
deleted++
}
}
}
return deleted, nil
}
// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// - operation (string): e.g. git-receive-pack
// - repoName (string): The repository name, in the form '/REPO.git'.
// - channel (ssh.Channel): The channel.
// - request (*ssh.Request): The channel.
// - gitHome (string): Defaults to /home/git.
// - fingerprint (string): The fingerprint of the user's SSH key.
// - user (string): The name of the Deis user.
//
// Returns:
// - nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
return nil, fmt.Errorf("Missing requirements %q", z)
}
repoName := p.Get("repoName", "").(string)
operation := p.Get("operation", "").(string)
channel := p.Get("channel", nil).(ssh.Channel)
gitHome := p.Get("gitHome", "/home/git").(string)
fingerprint := p.Get("fingerprint", nil).(string)
user := p.Get("user", "").(string)
log.Debugf(c, "receiving git repo name: %s, operation: %s, fingerprint: %s, user: %s", repoName, operation, fingerprint, user)
repo, err := cleanRepoName(repoName)
if err != nil {
log.Warnf(c, "Illegal repo name: %s.", err)
channel.Stderr().Write([]byte("No repo given"))
return nil, err
}
repo += ".git"
repoPath := filepath.Join(gitHome, repo)
log.Debugf(c, "creating repo directory %s", repoPath)
if _, err := createRepo(c, repoPath); err != nil {
err = fmt.Errorf("Did not create new repo (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
log.Debugf(c, "writing pre-receive hook under %s", repoPath)
if err := createPreReceiveHook(c, gitHome, repoPath); err != nil {
err = fmt.Errorf("Did not write pre-receive hook (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
log.Infof(c, strings.Join(cmd.Args, " "))
var errbuff bytes.Buffer
cmd.Dir = gitHome
cmd.Env = []string{
fmt.Sprintf("RECEIVE_USER=%s", user),
fmt.Sprintf("RECEIVE_REPO=%s", repo),
fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
}
cmd.Env = append(cmd.Env, os.Environ()...)
log.Debugf(c, "Working Dir: %s", cmd.Dir)
log.Debugf(c, "Environment: %s", strings.Join(cmd.Env, ","))
inpipe, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
cmd.Stdout = channel
cmd.Stderr = io.MultiWriter(channel.Stderr(), &errbuff)
if err := cmd.Start(); err != nil {
err = fmt.Errorf("Failed to start git pre-receive hook: %s (%s)", err, errbuff.Bytes())
log.Warnf(c, err.Error())
return nil, err
}
if _, err := io.Copy(inpipe, channel); err != nil {
err = fmt.Errorf("Failed to write git objects into the git pre-receive hook (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
fmt.Println("Waiting for git-receive to run.")
fmt.Println("Waiting for deploy.")
if err := cmd.Wait(); err != nil {
err = fmt.Errorf("Failed to run git pre-receive hook: %s (%s)", errbuff.Bytes(), err)
log.Errf(c, err.Error())
return nil, err
}
if errbuff.Len() > 0 {
log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
}
log.Infof(c, "Deploy complete.\n")
return nil, nil
}