// Push pushes an image to the registry. // // This finds the appropriate registry by looking it up in etcd. // // Params: // - client (etcd.Getter): Client to do etcd lookups. // - tag (string): Tag to push. // // Returns: // func Push(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { // docker tag deis/slugrunner:lastest HOST:PORT/deis/slugrunner:latest // docker push HOST:PORT/deis/slugrunner:latest client := p.Get("client", nil).(etcd.Getter) host, err := client.Get("/deis/registry/host", false, false) if err != nil || host.Node == nil { return nil, err } port, err := client.Get("/deis/registry/port", false, false) if err != nil || host.Node == nil { return nil, err } registry := host.Node.Value + ":" + port.Node.Value tag := p.Get("tag", "").(string) log.Infof(c, "Pushing %s to %s. This may take some time.", tag, registry) rem := path.Join(registry, tag) out, err := exec.Command("docker", "tag", "-f", tag, rem).CombinedOutput() if err != nil { log.Warnf(c, "Failed to tag %s on host %s: %s (%s)", tag, rem, err, out) } out, err = exec.Command("docker", "-D", "push", rem).CombinedOutput() if err != nil { log.Warnf(c, "Failed to push %s to host %s: %s (%s)", tag, rem, err, out) return nil, err } log.Infof(c, "Finished pushing %s to %s.", tag, registry) return nil, nil }
// createRepo creates a new Git repo if it is not present already. // // Largely inspired by gitreceived from Flynn. // // Returns a bool indicating whether a project was created (true) or already // existed (false). func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) { createLock.Lock() defer createLock.Unlock() fi, err := os.Stat(repoPath) if err == nil && fi.IsDir() { // Nothing to do. log.Infof(c, "Directory %s already exists.", repoPath) return false, nil } else if os.IsNotExist(err) { log.Infof(c, "Creating new directory at %s", repoPath) // Create directory if err := os.MkdirAll(repoPath, 0755); err != nil { log.Warnf(c, "Failed to create repository: %s", err) return false, err } cmd := exec.Command("git", "init", "--bare") cmd.Dir = repoPath if out, err := cmd.CombinedOutput(); err != nil { log.Warnf(c, "git init output: %s", out) return false, err } hook, err := prereceiveHook(map[string]string{"GitHome": gitHome}) if err != nil { return true, err } ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755) return true, nil } else if err == nil { return false, errors.New("Expected directory, found file.") } return false, err }
// FindSSHUser finds an SSH user by public key. // // Some parts of the system require that we know not only the SSH key, but also // the name of the user. That information is stored in etcd. // // Params: // - client (EtcdGetter) // - fingerprint (string): The fingerprint of the SSH key. // // Returns: // - username (string) func FindSSHUser(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { client := p.Get("client", nil).(Getter) fingerprint := p.Get("fingerprint", nil).(string) res, err := client.Get("/deis/builder/users", false, true) if err != nil { log.Warnf(c, "Error querying etcd: %s", err) return "", err } else if res.Node == nil || !res.Node.Dir { log.Warnf(c, "No users found in etcd.") return "", errors.New("Users not found") } for _, user := range res.Node.Nodes { log.Infof(c, "Checking user %s", user.Key) for _, keyprint := range user.Nodes { if strings.HasSuffix(keyprint.Key, fingerprint) { parts := strings.Split(user.Key, "/") username := parts[len(parts)-1] log.Infof(c, "Found user %s for fingerprint %s", username, fingerprint) return username, nil } } } return "", fmt.Errorf("User not found for fingerprint %s", fingerprint) }
// createRepo creates a new Git repo if it is not present already. // // Largely inspired by gitreceived from Flynn. // // Returns a bool indicating whether a project was created (true) or already // existed (false). func createRepo(c cookoo.Context, repoPath, gitHome string) (bool, error) { createLock.Lock() defer createLock.Unlock() if fi, err := os.Stat(repoPath); err == nil { if fi.IsDir() { configPath := filepath.Join(repoPath, "config") if _, cerr := os.Stat(configPath); cerr == nil { log.Debugf(c, "Directory '%s' already exists.", repoPath) return true, nil } else { log.Warnf(c, "No config file found at `%s`; removing it and recreating.", repoPath) if err := os.RemoveAll(repoPath); err != nil { return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err) } } } else { log.Warnf(c, "Path '%s' is not a directory; removing it and recreating.", repoPath) if err := os.RemoveAll(repoPath); err != nil { return false, fmt.Errorf("Unable to remove path '%s': %s", repoPath, err) } } } else if os.IsNotExist(err) { log.Debugf(c, "Unable to get stat for path '%s': %s .", repoPath, err) } else { return false, err } return initRepo(repoPath, gitHome, c) }
// Receive receives a Git repo. // This will only work for git-receive-pack. // // Params: // - operation (string): e.g. git-receive-pack // - repoName (string): The repository name, in the form '/REPO.git'. // - channel (ssh.Channel): The channel. // - request (*ssh.Request): The channel. // - gitHome (string): Defaults to /home/git. // - fingerprint (string): The fingerprint of the user's SSH key. // - user (string): The name of the Deis user. // // Returns: // - nothing func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok { return nil, fmt.Errorf("Missing requirements %q", z) } repoName := p.Get("repoName", "").(string) operation := p.Get("operation", "").(string) channel := p.Get("channel", nil).(ssh.Channel) gitHome := p.Get("gitHome", "/home/git").(string) fingerprint := p.Get("fingerprint", nil).(string) user := p.Get("user", "").(string) repo, err := cleanRepoName(repoName) if err != nil { log.Warnf(c, "Illegal repo name: %s.", err) channel.Stderr().Write([]byte("No repo given")) return nil, err } repo += ".git" if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil { log.Infof(c, "Did not create new repo: %s", err) } cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo)) log.Infof(c, strings.Join(cmd.Args, " ")) var errbuff bytes.Buffer cmd.Dir = gitHome cmd.Env = []string{ fmt.Sprintf("RECEIVE_USER=%s", user), fmt.Sprintf("RECEIVE_REPO=%s", repo), fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint), fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo), fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)), } cmd.Env = append(cmd.Env, os.Environ()...) done := plumbCommand(cmd, channel, &errbuff) if err := cmd.Start(); err != nil { log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes()) return nil, err } fmt.Printf("Waiting for git-receive to run.\n") done.Wait() fmt.Printf("Waiting for deploy.\n") if err := cmd.Wait(); err != nil { log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes()) return nil, err } if errbuff.Len() > 0 { log.Warnf(c, "Unreported error: %s", errbuff.Bytes()) } log.Infof(c, "Deploy complete.\n") return nil, nil }
// initRepo create a directory and init a new Git repo func initRepo(repoPath, gitHome string, c cookoo.Context) (bool, error) { log.Infof(c, "Creating new directory at %s", repoPath) // Create directory if err := os.MkdirAll(repoPath, 0755); err != nil { log.Warnf(c, "Failed to create repository: %s", err) return false, err } cmd := exec.Command("git", "init", "--bare") cmd.Dir = repoPath if out, err := cmd.CombinedOutput(); err != nil { log.Warnf(c, "git init output: %s", out) return false, err } hook, err := prereceiveHook(map[string]string{"GitHome": gitHome}) if err != nil { return true, err } ioutil.WriteFile(filepath.Join(repoPath, "hooks", "pre-receive"), hook, 0755) return true, nil }
// iam injects info into the environment about a host's self. // // Sets the following environment variables. (Values represent the data format. // Instances will get its own values.) // // MY_NODEIP=10.245.1.3 // MY_SERVICE_IP=10.22.1.4 // MY_PORT_PEER=2380 // MY_PORT_CLIENT=2379 // MY_NAMESPACE=default // MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp // MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272 // MY_APISERVER=https://10.247.0.1:443 // MY_NAME=deis-etcd-1-336jp // MY_IP=10.246.44.7 // MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON // MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON // MY_PORT_CLIENT=4100 // MY_PORT_PEER=2380 func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { me, err := aboutme.FromEnv() if err != nil { log.Errf(c, "Failed aboutme.FromEnv: %s", err) log.Warn(c, "Attempting to recover.") } // This will try to recover whenever IP is not set. Only some fields // can be recovered. But all we really need is IP and Name. if strings.TrimSpace(me.IP) == "" { log.Warn(c, "No IP found by API query.") ip, err := aboutme.MyIP() if err != nil || ip == "" { // Force pod death. log.Errf(c, "Failed to get an IP address: %s", err) os.Exit(5) } me.IP = ip } if strings.TrimSpace(me.Name) == "" { // Try to set name from DAPI. me.Name = os.Getenv("POD_NAME") log.Warnf(c, "Setting name to %q", me.Name) } if strings.TrimSpace(me.Namespace) == "" { // Try to set namespace from DAPI. me.Namespace = os.Getenv("POD_NAMESPACE") log.Warnf(c, "Setting name to %q", me.Namespace) } me.ShuntEnv() os.Setenv("ETCD_NAME", me.Name) c.Put("ETCD_NAME", me.Name) passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT") passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER") return nil, nil }
// setJoinMode determines what mode to start the etcd server in. // // In discovery mode, this will use the discovery URL to join a new cluster. // In "existing" mode, this will join to an existing cluster directly. // // Params: // - client (etcd.Getter): initialized etcd client // - path (string): path to get. This will go through os.ExpandEnv(). // - desiredLen (string): The number of nodes to expect in etcd. This is // usually stored as a string. // // Returns: // string "existing" or "new" func setJoinMode(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { cli := p.Get("client", nil).(client.Client) dlen := p.Get("desiredLen", "3").(string) path := p.Get("path", "").(string) path = os.ExpandEnv(path) state := "existing" dint, err := strconv.Atoi(dlen) if err != nil { log.Warnf(c, "Expected integer length, got '%s'. Defaulting to 3", dlen) dint = 3 } // Ideally, this should look in the /deis/status directory in the discovery // service. That will indicate how many members have been online in the // last few hours. This is a good indicator of whether a cluster exists, // even if not all the hosts are healthy. res, err := etcd.SimpleGet(cli, path, true) if err != nil { // This means that the discovery server is brand new, and nothing // has written to it yet. So we're new. if strings.Contains(err.Error(), "Key not found") { return "new", nil } log.Errf(c, "Failed simple get of %s: %s", path, err) return state, err } if !res.Node.Dir { //return state, errors.New("Expected a directory node in discovery service") log.Info(c, "No status information found in discovery service. Assuming new.") state = "new" } else if len(res.Node.Nodes) < dint { log.Info(c, "Cluster has not reached consensus number. Assuming new.") state = "new" } os.Setenv("ETCD_INITIAL_CLUSTER_STATE", state) return state, nil }
// Get gets one or more environment variables and puts them into the context. // // Parameters passed in are of the form varname => defaultValue. // // r.Route("foo", "example").Does(envvar.Get).Using("HOME").WithDefault(".") // // As with all environment variables, the default value must be a string. // // WARNING: Since parameters are a map, order of processing is not // guaranteed. If order is important, you'll need to call this command // multiple times. // // For each parameter (`Using` clause), this command will look into the // environment for a matching variable. If it finds one, it will add that // variable to the context. If it does not find one, it will expand the // default value (so you can set a default to something like "$HOST:$PORT") // and also put the (unexpanded) default value back into the context in case // any subsequent call to `os.Getenv` occurs. func Get(c cookoo.Context, params *cookoo.Params) (interface{}, cookoo.Interrupt) { for name, def := range params.AsMap() { var val string if val = os.Getenv(name); len(val) == 0 { if def == nil { def = "" } def, ok := def.(string) if !ok { log.Warnf(c, "Could not convert %s. Type is %T", name, def) } val = os.ExpandEnv(def) // We want to make sure that any subsequent calls to Getenv // return the same default. os.Setenv(name, val) } c.Put(name, val) log.Debugf(c, "Name: %s, Val: %s", name, val) } return true, nil }
// listen handles accepting and managing connections. However, since closer // is len(1), it will not block the sender. func (s *server) listen(l net.Listener, conf *ssh.ServerConfig, closer chan interface{}) error { cxt := s.c log.Info(cxt, "Accepting new connections.") defer l.Close() // FIXME: Since Accept blocks, closer may not be checked often enough. for { log.Info(cxt, "Checking closer.") if len(closer) > 0 { <-closer log.Info(cxt, "Shutting down SSHD listener.") return nil } conn, err := l.Accept() if err != nil { log.Warnf(cxt, "Error during Accept: %s", err) // We shouldn't kill the listener because of an error. return err } safely.GoDo(cxt, func() { s.handleConn(conn, conf) }) } }
// answer handles answering requests and channel requests // // Currently, an exec must be either "ping", "git-receive-pack" or // "git-upload-pack". Anything else will result in a failure response. Right // now, we leave the channel open on failure because it is unclear what the // correct behavior for a failed exec is. // // Support for setting environment variables via `env` has been disabled. func (s *server) answer(channel ssh.Channel, requests <-chan *ssh.Request, sshConn string) error { defer channel.Close() // Answer all the requests on this connection. for req := range requests { ok := false // I think that ideally what we want to do here is pass this on to // the Cookoo router and let it handle each Type on its own. switch req.Type { case "env": o := &EnvVar{} ssh.Unmarshal(req.Payload, o) fmt.Printf("Key='%s', Value='%s'\n", o.Name, o.Value) req.Reply(true, nil) case "exec": clean := cleanExec(req.Payload) parts := strings.SplitN(clean, " ", 2) router := s.c.Get("cookoo.Router", nil).(*cookoo.Router) // TODO: Should we unset the context value 'cookoo.Router'? // We need a shallow copy of the context to avoid race conditions. cxt := s.c.Copy() cxt.Put("SSH_CONNECTION", sshConn) // Only allow commands that we know about. switch parts[0] { case "ping": cxt.Put("channel", channel) cxt.Put("request", req) sshPing := cxt.Get("route.sshd.sshPing", "sshPing").(string) err := router.HandleRequest(sshPing, cxt, true) if err != nil { log.Warnf(s.c, "Error pinging: %s", err) } return err case "git-receive-pack", "git-upload-pack": if len(parts) < 2 { log.Warn(s.c, "Expected two-part command.\n") req.Reply(ok, nil) break } req.Reply(true, nil) // We processed. Yay. cxt.Put("channel", channel) cxt.Put("request", req) cxt.Put("operation", parts[0]) cxt.Put("repository", parts[1]) sshGitReceive := cxt.Get("route.sshd.sshGitReceive", "sshGitReceive").(string) err := router.HandleRequest(sshGitReceive, cxt, true) var xs uint32 if err != nil { log.Errf(s.c, "Failed git receive: %v", err) xs = 1 } sendExitStatus(xs, channel) return nil default: log.Warnf(s.c, "Illegal command is '%s'\n", clean) req.Reply(false, nil) return nil } if err := sendExitStatus(0, channel); err != nil { log.Errf(s.c, "Failed to write exit status: %s", err) } return nil default: // We simply ignore all of the other cases and leave the // channel open to take additional requests. log.Infof(s.c, "Received request of type %s\n", req.Type) req.Reply(false, nil) } } return nil }
// RemoveStaleMembers deletes cluster members whose pods are no longer running. // // This queries Kubernetes to determine what etcd pods are running, and then // compares that to the member list in the etcd cluster. It removes any // cluster members who are no longer in the pod list. // // The purpose of this is to keep the cluster membership from deadlocking // when inactive members prevent consensus building. // // Params: // - client (etcd/client.Client): The etcd client // - label (string): The pod label indicating an etcd node // - namespace (string): The namespace we're operating in func RemoveStaleMembers(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { eclient := p.Get("client", nil).(client.Client) label := p.Get("label", "name=deis-etcd-1").(string) ns := p.Get("namespace", "default").(string) // Should probably pass in the client from the context. klient, err := k8s.PodClient() if err != nil { log.Errf(c, "Could not create a Kubernetes client: %s", err) return nil, err } mapi := client.NewMembersAPI(eclient) members := map[string]bool{} idmap := map[string]string{} // Get members from etcd mm, err := mapi.List(dctx()) if err != nil { log.Warnf(c, "Could not get a list of etcd members: %s", err) return nil, err } for _, member := range mm { members[member.Name] = false idmap[member.Name] = member.ID } // Get the pods running with the given label labelSelector, err := labels.Parse(label) if err != nil { log.Errf(c, "Selector failed to parse: %s", err) return nil, err } pods, err := klient.Pods(ns).List(api.ListOptions{LabelSelector: labelSelector}) if err != nil { return nil, err } for _, item := range pods.Items { if _, ok := members[item.Name]; !ok { log.Infof(c, "Etcd pod %s is not in cluster yet.", item.Name) } else { members[item.Name] = true } } // Anything marked false in members should be removed from etcd. deleted := 0 for k, v := range members { if !v { log.Infof(c, "Deleting %s (%s) from etcd cluster members", k, idmap[k]) if err := mapi.Remove(dctx(), idmap[k]); err != nil { log.Errf(c, "Failed to remove %s from cluster. Skipping. %s", k, err) } else { deleted++ } } } return deleted, nil }
// Receive receives a Git repo. // This will only work for git-receive-pack. // // Params: // - operation (string): e.g. git-receive-pack // - repoName (string): The repository name, in the form '/REPO.git'. // - channel (ssh.Channel): The channel. // - request (*ssh.Request): The channel. // - gitHome (string): Defaults to /home/git. // - fingerprint (string): The fingerprint of the user's SSH key. // - user (string): The name of the Deis user. // // Returns: // - nothing func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok { return nil, fmt.Errorf("Missing requirements %q", z) } repoName := p.Get("repoName", "").(string) operation := p.Get("operation", "").(string) channel := p.Get("channel", nil).(ssh.Channel) gitHome := p.Get("gitHome", "/home/git").(string) fingerprint := p.Get("fingerprint", nil).(string) user := p.Get("user", "").(string) log.Debugf(c, "receiving git repo name: %s, operation: %s, fingerprint: %s, user: %s", repoName, operation, fingerprint, user) repo, err := cleanRepoName(repoName) if err != nil { log.Warnf(c, "Illegal repo name: %s.", err) channel.Stderr().Write([]byte("No repo given")) return nil, err } repo += ".git" repoPath := filepath.Join(gitHome, repo) log.Debugf(c, "creating repo directory %s", repoPath) if _, err := createRepo(c, repoPath); err != nil { err = fmt.Errorf("Did not create new repo (%s)", err) log.Warnf(c, err.Error()) return nil, err } log.Debugf(c, "writing pre-receive hook under %s", repoPath) if err := createPreReceiveHook(c, gitHome, repoPath); err != nil { err = fmt.Errorf("Did not write pre-receive hook (%s)", err) log.Warnf(c, err.Error()) return nil, err } cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo)) log.Infof(c, strings.Join(cmd.Args, " ")) var errbuff bytes.Buffer cmd.Dir = gitHome cmd.Env = []string{ fmt.Sprintf("RECEIVE_USER=%s", user), fmt.Sprintf("RECEIVE_REPO=%s", repo), fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint), fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo), fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)), } cmd.Env = append(cmd.Env, os.Environ()...) log.Debugf(c, "Working Dir: %s", cmd.Dir) log.Debugf(c, "Environment: %s", strings.Join(cmd.Env, ",")) inpipe, err := cmd.StdinPipe() if err != nil { return nil, err } cmd.Stdout = channel cmd.Stderr = io.MultiWriter(channel.Stderr(), &errbuff) if err := cmd.Start(); err != nil { err = fmt.Errorf("Failed to start git pre-receive hook: %s (%s)", err, errbuff.Bytes()) log.Warnf(c, err.Error()) return nil, err } if _, err := io.Copy(inpipe, channel); err != nil { err = fmt.Errorf("Failed to write git objects into the git pre-receive hook (%s)", err) log.Warnf(c, err.Error()) return nil, err } fmt.Println("Waiting for git-receive to run.") fmt.Println("Waiting for deploy.") if err := cmd.Wait(); err != nil { err = fmt.Errorf("Failed to run git pre-receive hook: %s (%s)", errbuff.Bytes(), err) log.Errf(c, err.Error()) return nil, err } if errbuff.Len() > 0 { log.Warnf(c, "Unreported error: %s", errbuff.Bytes()) } log.Infof(c, "Deploy complete.\n") return nil, nil }