// RemoveMemberByName removes a member whose name matches the given. // // Params: // - client(client.Client): An etcd client // - name (string): The name to remove // Returns: // true if the member was found, false otherwise. func RemoveMemberByName(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { cli := p.Get("client", nil).(client.Client) name := p.Get("name", "____").(string) mem := client.NewMembersAPI(cli) members, err := mem.List(dctx()) if err != nil { log.Errf(c, "Could not get a list of members: %s", err) return false, err } remIDs := []string{} for _, member := range members { if member.Name == name { log.Infof(c, "Removing member %s (ID: %s)", name, member.ID) // If this is synchronizable, we should do it in parallel. if err := mem.Remove(dctx(), member.ID); err != nil { log.Errf(c, "Failed to remove member: %s", err) return len(remIDs) > 0, err } remIDs = append(remIDs, member.ID) } } return len(remIDs) > 0, nil }
// ParseHostKeys parses the host key files. // // By default it looks in /etc/ssh for host keys of the patterh ssh_host_{{TYPE}}_key. // // Params: // - keytypes ([]string): Key types to parse. Defaults to []string{rsa, dsa, ecdsa} // - enableV1 (bool): Allow V1 keys. By default this is disabled. // - path (string): Override the lookup pattern. If %s, it will be replaced with the keytype. // // Returns: // []ssh.Signer func ParseHostKeys(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { log.Debugf(c, "Parsing ssh host keys") hostKeyTypes := p.Get("keytypes", []string{"rsa", "dsa", "ecdsa"}).([]string) pathTpl := p.Get("path", "/etc/ssh/ssh_host_%s_key").(string) hostKeys := make([]ssh.Signer, 0, len(hostKeyTypes)) for _, t := range hostKeyTypes { path := fmt.Sprintf(pathTpl, t) if key, err := ioutil.ReadFile(path); err == nil { if hk, err := ssh.ParsePrivateKey(key); err == nil { log.Infof(c, "Parsed host key %s.", path) hostKeys = append(hostKeys, hk) } else { log.Errf(c, "Failed to parse host key %s (skipping): %s", path, err) } } } if c.Get("enableV1", false).(bool) { path := "/etc/ssh/ssh_host_key" if key, err := ioutil.ReadFile(path); err != nil { log.Errf(c, "Failed to read ssh_host_key") } else if hk, err := ssh.ParsePrivateKey(key); err == nil { log.Infof(c, "Parsed host key %s.", path) hostKeys = append(hostKeys, hk) } else { log.Errf(c, "Failed to parse host key %s: %s", path, err) } } return hostKeys, nil }
// iam injects info into the environment about a host's self. // // Sets the following environment variables. (Values represent the data format. // Instances will get its own values.) // // MY_NODEIP=10.245.1.3 // MY_SERVICE_IP=10.22.1.4 // MY_PORT_PEER=2380 // MY_PORT_CLIENT=2379 // MY_NAMESPACE=default // MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp // MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272 // MY_APISERVER=https://10.247.0.1:443 // MY_NAME=deis-etcd-1-336jp // MY_IP=10.246.44.7 // MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON // MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON // MY_PORT_CLIENT=4100 // MY_PORT_PEER=2380 func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { me, err := aboutme.FromEnv() if err != nil { log.Errf(c, "Failed aboutme.FromEnv: %s", err) } else { if strings.TrimSpace(me.IP) == "" { log.Warn(c, "No IP found by API query.") ip, err := aboutme.MyIP() if err != nil || ip == "" { // Force pod death. log.Errf(c, "Failed to get an IP address: %s", err) os.Exit(5) } } me.ShuntEnv() os.Setenv("ETCD_NAME", me.Name) c.Put("ETCD_NAME", me.Name) } passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT") passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER") return nil, nil }
// Watch watches a given path, and executes a git check-repos for each event. // // It starts the watcher and then returns. The watcher runs on its own // goroutine. To stop the watching, send the returned channel a bool. // // Params: // - client (Watcher): An Etcd client. // - path (string): The path to watch func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { // etcdctl -C $ETCD watch --recursive /deis/services path := p.Get("path", "/deis/services").(string) cli := p.Get("client", nil).(client.Client) k := client.NewKeysAPI(cli) watcher := k.Watcher(path, &client.WatcherOptions{Recursive: true}) safely.GoDo(c, func() { for { // TODO: We should probably add cancellation support. response, err := watcher.Next(dctx()) if err != nil { log.Errf(c, "Etcd Watch failed: %s", err) } if response.Node == nil { log.Infof(c, "Unexpected Etcd message: %v", response) } git := exec.Command("/home/git/check-repos") if out, err := git.CombinedOutput(); err != nil { log.Errf(c, "Failed git check-repos: %s", err) log.Infof(c, "Output: %s", out) } } }) return nil, nil }
// keysToEtcd copies local keys into etcd. // // It only fails if it cannot copy ssh_host_key to sshHostKey. All other // abnormal conditions are logged, but not considered to be failures. func keysToEtcd(c cookoo.Context, client Setter, ciphers []string, etcdPath string) error { firstpath := "" lpath := "/etc/ssh/ssh_host_%s_key" privkey := "%s/sshHost%sKey" for _, cipher := range ciphers { path := fmt.Sprintf(lpath, cipher) key := fmt.Sprintf(privkey, etcdPath, cipher) content, err := ioutil.ReadFile(path) if err != nil { log.Infof(c, "No key named %s", path) } else if _, err := client.Set(key, string(content), 0); err != nil { log.Errf(c, "Could not store ssh key in etcd: %s", err) } // Remember the first key's path in case the generic key is missing if firstpath == "" { firstpath = path } } // Now we set the generic key: keypath := "/etc/ssh/ssh_host_key" if _, err := os.Stat(keypath); os.IsNotExist(err) && firstpath != "" { // Use ssh_host_dsa_key if newer ssh-keygen didn't create ssh_host_key keypath = firstpath } if content, err := ioutil.ReadFile(keypath); err != nil { log.Errf(c, "Could not read the %s file.", keypath) return err } else if _, err := client.Set("sshHostKey", string(content), 0); err != nil { log.Errf(c, "Failed to set sshHostKey in etcd.") return err } return nil }
// Watch watches a given path, and executes a git check-repos for each event. // // It starts the watcher and then returns. The watcher runs on its own // goroutine. To stop the watching, send the returned channel a bool. // // Params: // - client (Watcher): An Etcd client. // - path (string): The path to watch // // Returns: // - chan bool: Send this a message to stop the watcher. func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { // etcdctl -C $ETCD watch --recursive /deis/services path := p.Get("path", "/deis/services").(string) cli, ok := p.Has("client") if !ok { return nil, errors.New("No etcd client found.") } client := cli.(Watcher) // Stupid hack because etcd watch seems to be broken, constantly complaining // that the JSON it received is malformed. safely.GoDo(c, func() { for { response, err := client.Watch(path, 0, true, nil, nil) if err != nil { log.Errf(c, "Etcd Watch failed: %s", err) time.Sleep(50 * time.Millisecond) continue } if response.Node == nil { log.Infof(c, "Unexpected Etcd message: %v", response) } git := exec.Command("/home/git/check-repos") if out, err := git.CombinedOutput(); err != nil { log.Errf(c, "Failed git check-repos: %s", err) log.Infof(c, "Output: %s", out) } } }) return nil, nil }
// keysToLocal copies SSH host keys from etcd to the local file system. // // This only fails if the main key, sshHostKey cannot be stored or retrieved. func keysToLocal(c cookoo.Context, client Getter, ciphers []string, etcdPath string) error { lpath := "/etc/ssh/ssh_host_%s_key" privkey := "%s/sshHost%sKey" for _, cipher := range ciphers { path := fmt.Sprintf(lpath, cipher) key := fmt.Sprintf(privkey, etcdPath, cipher) res, err := client.Get(key, false, false) if err != nil || res.Node == nil { continue } content := res.Node.Value if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil { log.Errf(c, "Error writing ssh host key file: %s", err) } } // Now get generic key. res, err := client.Get("sshHostKey", false, false) if err != nil || res.Node == nil { return fmt.Errorf("Failed to get sshHostKey from etcd. %v", err) } content := res.Node.Value if err := ioutil.WriteFile("/etc/ssh/ssh_host_key", []byte(content), 0600); err != nil { log.Errf(c, "Error writing ssh host key file: %s", err) return err } return nil }
// Run starts the Builder service. // // The Builder service is responsible for setting up the local container // environment and then listening for new builds. The main listening service // is SSH. Builder listens for new Git commands and then sends those on to // Git. // // Run returns on of the Status* status code constants. func Run(cmd string) int { reg, router, ocxt := cookoo.Cookoo() log.SetFlags(0) // Time is captured elsewhere. // We layer the context to add better logging and also synchronize // access so that goroutines don't get into race conditions. cxt := cookoo.SyncContext(ocxt) cxt.Put("cookoo.Router", router) cxt.AddLogger("stdout", os.Stdout) // Build the routes. See routes.go. routes(reg) // Bootstrap the background services. If this fails, we stop. if err := router.HandleRequest("boot", cxt, false); err != nil { clog.Errf(cxt, "Fatal errror on boot: %s", err) return StatusLocalError } // Set up the SSH service. ip := os.Getenv("SSH_HOST_IP") if ip == "" { ip = "0.0.0.0" } port := os.Getenv("SSH_HOST_PORT") if port == "" { port = "2223" } cxt.Put(sshd.Address, ip+":"+port) // Supply route names for handling various internal routing. While this // isn't necessary for Cookoo, it makes it easy for us to mock these // routes in tests. c.f. sshd/server.go cxt.Put("route.sshd.pubkeyAuth", "pubkeyAuth") cxt.Put("route.sshd.sshPing", "sshPing") cxt.Put("route.sshd.sshGitReceive", "sshGitReceive") // Start the SSH service. // TODO: We could refactor Serve to be a command, and then run this as // a route. if err := sshd.Serve(reg, router, cxt); err != nil { clog.Errf(cxt, "SSH server failed: %s", err) return StatusLocalError } return StatusOk }
// handleConn handles an individual client connection. // // It manages the connection, but passes channels on to `answer()`. func (s *server) handleConn(conn net.Conn, conf *ssh.ServerConfig) { defer conn.Close() log.Info(s.c, "Accepted connection.") _, chans, reqs, err := ssh.NewServerConn(conn, conf) if err != nil { // Handshake failure. log.Errf(s.c, "Failed handshake: %s (%v)", err, conn) return } // Discard global requests. We're only concerned with channels. safely.GoDo(s.c, func() { ssh.DiscardRequests(reqs) }) condata := sshConnection(conn) // Now we handle the channels. for incoming := range chans { log.Infof(s.c, "Channel type: %s\n", incoming.ChannelType()) if incoming.ChannelType() != "session" { incoming.Reject(ssh.UnknownChannelType, "Unknown channel type") } channel, req, err := incoming.Accept() if err != nil { // Should close request and move on. panic(err) } safely.GoDo(s.c, func() { s.answer(channel, req, condata) }) } conn.Close() }
// ParallelBuild runs multiple docker builds at the same time. // // Params: // -images ([]BuildImg): Images to build // -alwaysFetch (bool): Default false. If set to true, this will always fetch // the Docker image even if it already exists in the registry. // // Returns: // // - Waiter: A *sync.WaitGroup that is waiting for the docker downloads to finish. // // Context: // // This puts 'ParallelBuild.failN" (int) into the context to indicate how many failures // occurred during fetches. func ParallelBuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { images := p.Get("images", []BuildImg{}).([]BuildImg) var wg sync.WaitGroup var m sync.Mutex var fails int for _, img := range images { img := img wg.Add(1) safely.GoDo(c, func() { log.Infof(c, "Starting build for %s (tag: %s)", img.Path, img.Tag) if _, err := buildImg(c, img.Path, img.Tag); err != nil { log.Errf(c, "Failed to build docker image: %s", err) m.Lock() fails++ m.Unlock() } wg.Done() }) } // Number of failures. c.Put("ParallelBuild.failN", fails) return &wg, nil }
// ParallelBuild runs multiple docker builds at the same time. // // Params: // -images ([]BuildImg): Images to build // -alwaysFetch (bool): Default false. If set to true, this will always fetch // the Docker image even if it already exists in the registry. // // Returns: // // - Waiter: A *sync.WaitGroup that is waiting for the docker downloads to finish. // // Context: // // This puts 'ParallelBuild.failN" (int) into the context to indicate how many failures // occurred during fetches. func ParallelBuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { images := p.Get("images", []BuildImg{}).([]BuildImg) var wg sync.WaitGroup var m sync.Mutex var fails int for _, img := range images { img := img // HACK: ensure "docker build" is serialized by allowing only one entry in // the WaitGroup. This works around the "simultaneous docker pull" bug. wg.Wait() wg.Add(1) safely.GoDo(c, func() { log.Infof(c, "Starting build for %s (tag: %s)", img.Path, img.Tag) if _, err := buildImg(c, img.Path, img.Tag); err != nil { log.Errf(c, "Failed to build docker image: %s", err) m.Lock() fails++ m.Unlock() } wg.Done() }) } // Number of failures. c.Put("ParallelBuild.failN", fails) return &wg, nil }
// Receive receives a Git repo. // This will only work for git-receive-pack. // // Params: // - operation (string): e.g. git-receive-pack // - repoName (string): The repository name, in the form '/REPO.git'. // - channel (ssh.Channel): The channel. // - request (*ssh.Request): The channel. // - gitHome (string): Defaults to /home/git. // - fingerprint (string): The fingerprint of the user's SSH key. // - user (string): The name of the Deis user. // // Returns: // - nothing func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok { return nil, fmt.Errorf("Missing requirements %q", z) } repoName := p.Get("repoName", "").(string) operation := p.Get("operation", "").(string) channel := p.Get("channel", nil).(ssh.Channel) gitHome := p.Get("gitHome", "/home/git").(string) fingerprint := p.Get("fingerprint", nil).(string) user := p.Get("user", "").(string) repo, err := cleanRepoName(repoName) if err != nil { log.Warnf(c, "Illegal repo name: %s.", err) channel.Stderr().Write([]byte("No repo given")) return nil, err } repo += ".git" if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil { log.Infof(c, "Did not create new repo: %s", err) } cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo)) log.Infof(c, strings.Join(cmd.Args, " ")) var errbuff bytes.Buffer cmd.Dir = gitHome cmd.Env = []string{ fmt.Sprintf("RECEIVE_USER=%s", user), fmt.Sprintf("RECEIVE_REPO=%s", repo), fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint), fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo), fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)), } cmd.Env = append(cmd.Env, os.Environ()...) done := plumbCommand(cmd, channel, &errbuff) if err := cmd.Start(); err != nil { log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes()) return nil, err } fmt.Printf("Waiting for git-receive to run.\n") done.Wait() fmt.Printf("Waiting for deploy.\n") if err := cmd.Wait(); err != nil { log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes()) return nil, err } if errbuff.Len() > 0 { log.Warnf(c, "Unreported error: %s", errbuff.Bytes()) } log.Infof(c, "Deploy complete.\n") return nil, nil }
// genSshKeys generates the default set of SSH host keys. func genSSHKeys(c cookoo.Context) error { // Generate a new key out, err := exec.Command("ssh-keygen", "-A").CombinedOutput() if err != nil { log.Infof(c, "ssh-keygen: %s", out) log.Errf(c, "Failed to generate SSH keys: %s", err) return err } return nil }
// Ping handles a simple test SSH exec. // // Returns the string PONG and exit status 0. // // Params: // - channel (ssh.Channel): The channel to respond on. // - request (*ssh.Request): The request. // func Ping(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { channel := p.Get("channel", nil).(ssh.Channel) req := p.Get("request", nil).(*ssh.Request) log.Info(c, "PING\n") if _, err := channel.Write([]byte("pong")); err != nil { log.Errf(c, "Failed to write to channel: %s", err) } sendExitStatus(0, channel) req.Reply(true, nil) return nil, nil }
// UpdateHostPort intermittently notifies etcd of the builder's address. // // If `port` is specified, this will notify etcd at 10 second intervals that // the builder is listening at $HOST:$PORT, setting the TTL to 20 seconds. // // This will notify etcd as long as the local sshd is running. // // Params: // - base (string): The base path to write the data: $base/host and $base/port. // - host (string): The hostname // - port (string): The port // - client (Setter): The client to use to write the data to etcd. // - sshPid (int): The PID for SSHD. If SSHD dies, this stops notifying. func UpdateHostPort(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { base := p.Get("base", "").(string) host := p.Get("host", "").(string) port := p.Get("port", "").(string) client := p.Get("client", nil).(Setter) sshd := p.Get("sshdPid", 0).(int) // If no port is specified, we don't do anything. if len(port) == 0 { log.Infof(c, "No external port provided. Not publishing details.") return false, nil } var ttl uint64 = 20 if err := setHostPort(client, base, host, port, ttl); err != nil { log.Errf(c, "Etcd error setting host/port: %s", err) return false, err } // Update etcd every ten seconds with this builder's host/port. safely.GoDo(c, func() { ticker := time.NewTicker(10 * time.Second) for range ticker.C { //log.Infof(c, "Setting SSHD host/port") if _, err := os.FindProcess(sshd); err != nil { log.Errf(c, "Lost SSHd process: %s", err) break } else { if err := setHostPort(client, base, host, port, ttl); err != nil { log.Errf(c, "Etcd error setting host/port: %s", err) break } } } ticker.Stop() }) return true, nil }
// iam injects info into the environment about a host's self. // // Sets the following environment variables. (Values represent the data format. // Instances will get its own values.) // // MY_NODEIP=10.245.1.3 // MY_SERVICE_IP=10.22.1.4 // MY_PORT_PEER=2380 // MY_PORT_CLIENT=2379 // MY_NAMESPACE=default // MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp // MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272 // MY_APISERVER=https://10.247.0.1:443 // MY_NAME=deis-etcd-1-336jp // MY_IP=10.246.44.7 // MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON // MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON // MY_PORT_CLIENT=4100 // MY_PORT_PEER=2380 func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { me, err := aboutme.FromEnv() if err != nil { log.Errf(c, "Failed aboutme.FromEnv: %s", err) log.Warn(c, "Attempting to recover.") } // This will try to recover whenever IP is not set. Only some fields // can be recovered. But all we really need is IP and Name. if strings.TrimSpace(me.IP) == "" { log.Warn(c, "No IP found by API query.") ip, err := aboutme.MyIP() if err != nil || ip == "" { // Force pod death. log.Errf(c, "Failed to get an IP address: %s", err) os.Exit(5) } me.IP = ip } if strings.TrimSpace(me.Name) == "" { // Try to set name from DAPI. me.Name = os.Getenv("POD_NAME") log.Warnf(c, "Setting name to %q", me.Name) } if strings.TrimSpace(me.Namespace) == "" { // Try to set namespace from DAPI. me.Namespace = os.Getenv("POD_NAMESPACE") log.Warnf(c, "Setting name to %q", me.Namespace) } me.ShuntEnv() os.Setenv("ETCD_NAME", me.Name) c.Put("ETCD_NAME", me.Name) passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT") passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER") return nil, nil }
// IsRunning checks to see if etcd is running. // // It will test `count` times before giving up. // // Params: // - client (EtcdGetter) // - count (int): Number of times to try before giving up. // // Returns: // boolean true if etcd is listening. func IsRunning(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { client := p.Get("client", nil).(Getter) count := p.Get("count", 20).(int) for i := 0; i < count; i++ { _, err := client.Get("/", false, false) if err == nil { return true, nil } log.Infof(c, "Waiting for etcd to come online.") time.Sleep(250 * time.Millisecond) } log.Errf(c, "Etcd is not answering after %d attempts.", count) return false, &cookoo.FatalError{"Could not connect to Etcd."} }
// keysToEtcd copies local keys into etcd. // // It only fails if it cannot copy ssh_host_key to sshHostKey. All other // abnormal conditions are logged, but not considered to be failures. func keysToEtcd(c cookoo.Context, client Setter, ciphers []string, etcdPath string) error { lpath := "/etc/ssh/ssh_host_%s_key" privkey := "%s/sshHost%sKey" for _, cipher := range ciphers { path := fmt.Sprintf(lpath, cipher) key := fmt.Sprintf(privkey, etcdPath, cipher) content, err := ioutil.ReadFile(path) if err != nil { log.Infof(c, "No key named %s", path) } else if _, err := client.Set(key, string(content), 0); err != nil { log.Errf(c, "Could not store ssh key in etcd: %s", err) } } // Now we set the generic key: if content, err := ioutil.ReadFile("/etc/ssh/ssh_host_key"); err != nil { log.Errf(c, "Could not read the ssh_host_key file.") return err } else if _, err := client.Set("sshHostKey", string(content), 0); err != nil { log.Errf(c, "Failed to set sshHostKey in etcd.") return err } return nil }
// iam injects info into the environment about a host's self. // // Sets the following environment variables. (Values represent the data format. // Instances will get its own values.) // // MY_NODEIP=10.245.1.3 // MY_SERVICE_IP=10.22.1.4 // MY_PORT_PEER=2380 // MY_PORT_CLIENT=2379 // MY_NAMESPACE=default // MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp // MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272 // MY_APISERVER=https://10.247.0.1:443 // MY_NAME=deis-etcd-1-336jp // MY_IP=10.246.44.7 // MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON // MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON // MY_PORT_CLIENT=4100 // MY_PORT_PEER=2380 func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { me, err := aboutme.FromEnv() if err != nil { log.Errf(c, "Failed aboutme.FromEnv: %s", err) } else { me.ShuntEnv() os.Setenv("ETCD_NAME", me.Name) c.Put("ETCD_NAME", me.Name) } passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT") passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER") return nil, nil }
// startEtcd starts a cluster member of a static etcd cluster. // // Params: // - discover (string): Value to pass to etcd --discovery. // - client (client.Client): A client to the discovery server. This will // periodically write data there to indicate liveness. func startEtcd(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { cli := p.Get("client", nil).(client.Client) // Use config from environment. cmd := exec.Command("etcd") cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout println(strings.Join(os.Environ(), "\n")) if err := cmd.Start(); err != nil { log.Errf(c, "Failed to start etcd: %s", err) return nil, err } // We need a way to tell starting members that there is an existing cluster, // and that that cluster has met the initial consensus requirements. This // basically stores a status record that indicates that it is an existing // and running etcd server. It allows for basically a two hour window // during which a cluster can be in an uncertain state before the entire // thing gives up and a new cluster is created. ticker := time.NewTicker(time.Minute) expires := time.Hour * 2 go func() { name := c.Get("ETCD_NAME", "").(string) tok := c.Get("DEIS_ETCD_DISCOVERY_TOKEN", "").(string) key := fmt.Sprintf(discovery.ClusterStatusKey, tok, name) for t := range ticker.C { etcd.SimpleSet(cli, key, t.String(), expires) } }() if err := cmd.Wait(); err != nil { ticker.Stop() log.Errf(c, "Etcd quit unexpectedly: %s", err) } return nil, nil }
// AddMember Add a new member to the cluster. // // Conceptually, this is equivalent to `etcdctl member add NAME IP`. // // Params: // - client(client.Client): An etcd client // - name (string): The name of the member to add. // - url (string): The peer ip:port or domain: port to use. // // Returns: // An etcd *client.Member. func AddMember(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { cli := p.Get("client", nil).(client.Client) name := p.Get("name", "default").(string) addr := p.Get("url", "127.0.0.1:2380").(string) mem := client.NewMembersAPI(cli) member, err := mem.Add(dctx(), addr) if err != nil { log.Errf(c, "Failed to add %s to cluster: %s", addr, err) return nil, err } log.Infof(c, "Added %s (%s) to cluster", addr, member.ID) member.Name = name return member, nil }
// setJoinMode determines what mode to start the etcd server in. // // In discovery mode, this will use the discovery URL to join a new cluster. // In "existing" mode, this will join to an existing cluster directly. // // Params: // - client (etcd.Getter): initialized etcd client // - path (string): path to get. This will go through os.ExpandEnv(). // - desiredLen (string): The number of nodes to expect in etcd. This is // usually stored as a string. // // Returns: // string "existing" or "new" func setJoinMode(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { cli := p.Get("client", nil).(client.Client) dlen := p.Get("desiredLen", "3").(string) path := p.Get("path", "").(string) path = os.ExpandEnv(path) state := "existing" dint, err := strconv.Atoi(dlen) if err != nil { log.Warnf(c, "Expected integer length, got '%s'. Defaulting to 3", dlen) dint = 3 } // Ideally, this should look in the /deis/status directory in the discovery // service. That will indicate how many members have been online in the // last few hours. This is a good indicator of whether a cluster exists, // even if not all the hosts are healthy. res, err := etcd.SimpleGet(cli, path, true) if err != nil { // This means that the discovery server is brand new, and nothing // has written to it yet. So we're new. if strings.Contains(err.Error(), "Key not found") { return "new", nil } log.Errf(c, "Failed simple get of %s: %s", path, err) return state, err } if !res.Node.Dir { //return state, errors.New("Expected a directory node in discovery service") log.Info(c, "No status information found in discovery service. Assuming new.") state = "new" } else if len(res.Node.Nodes) < dint { log.Info(c, "Cluster has not reached consensus number. Assuming new.") state = "new" } os.Setenv("ETCD_INITIAL_CLUSTER_STATE", state) return state, nil }
// Run starts confd and runs it in the background. // // If the command fails immediately on startup, an error is immediately // returned. But from that point, a goroutine watches the command and // reports if the command dies. // // Params: // - node (string): The etcd node to use. (Only etcd is currently supported) // - interval (int, default:5): The rebuilding interval. // // Returns // bool true if this succeeded. func Run(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { node := p.Get("node", defaultEtcd).(string) interval := strconv.Itoa(p.Get("interval", 5).(int)) cmd := exec.Command("confd", "-log-level", "error", "-node", node, "-interval", interval) if err := cmd.Start(); err != nil { return false, err } log.Infof(c, "Watching confd.") safely.Go(func() { if err := cmd.Wait(); err != nil { // If confd exits, builder will stop functioning as intended. So // we stop builder and let the environment restart. log.Errf(c, "Stopping builder. confd exited with error: %s", err) os.Exit(37) } }) return true, nil }
// Start starts a Docker daemon. // // This assumes the presence of the docker client on the host. It does not use // the API. func Start(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { // Allow insecure Docker registries on all private network ranges in RFC 1918 and RFC 6598. dargs := []string{ "-d", "--bip=172.19.42.1/16", "--insecure-registry", "10.0.0.0/8", "--insecure-registry", "172.16.0.0/12", "--insecure-registry", "192.168.0.0/16", "--insecure-registry", "100.64.0.0/10", "--exec-opt", "native.cgroupdriver=cgroupfs", } // For overlay-ish filesystems, force the overlay to kick in if it exists. // Then we can check the fstype. if err := os.MkdirAll("/", 0700); err == nil { cmd := exec.Command("findmnt", "--noheadings", "--output", "FSTYPE", "--target", "/") if out, err := cmd.Output(); err == nil && strings.TrimSpace(string(out)) == "overlay" { dargs = append(dargs, "--storage-driver=overlay") } else { log.Infof(c, "File system type: '%s' (%v)", out, err) } } log.Infof(c, "Starting docker with %s", strings.Join(dargs, " ")) cmd := exec.Command("docker", dargs...) if err := cmd.Start(); err != nil { log.Errf(c, "Failed to start Docker. %s", err) return -1, err } // Get the PID and return it. return cmd.Process.Pid, nil }
// Receive receives a Git repo. // This will only work for git-receive-pack. // // Params: // - operation (string): e.g. git-receive-pack // - repoName (string): The repository name, in the form '/REPO.git'. // - channel (ssh.Channel): The channel. // - request (*ssh.Request): The channel. // - gitHome (string): Defaults to /home/git. // - fingerprint (string): The fingerprint of the user's SSH key. // - user (string): The name of the Deis user. // // Returns: // - nothing func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok { return nil, fmt.Errorf("Missing requirements %q", z) } repoName := p.Get("repoName", "").(string) operation := p.Get("operation", "").(string) channel := p.Get("channel", nil).(ssh.Channel) gitHome := p.Get("gitHome", "/home/git").(string) fingerprint := p.Get("fingerprint", nil).(string) user := p.Get("user", "").(string) log.Debugf(c, "receiving git repo name: %s, operation: %s, fingerprint: %s, user: %s", repoName, operation, fingerprint, user) repo, err := cleanRepoName(repoName) if err != nil { log.Warnf(c, "Illegal repo name: %s.", err) channel.Stderr().Write([]byte("No repo given")) return nil, err } repo += ".git" repoPath := filepath.Join(gitHome, repo) log.Debugf(c, "creating repo directory %s", repoPath) if _, err := createRepo(c, repoPath); err != nil { err = fmt.Errorf("Did not create new repo (%s)", err) log.Warnf(c, err.Error()) return nil, err } log.Debugf(c, "writing pre-receive hook under %s", repoPath) if err := createPreReceiveHook(c, gitHome, repoPath); err != nil { err = fmt.Errorf("Did not write pre-receive hook (%s)", err) log.Warnf(c, err.Error()) return nil, err } cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo)) log.Infof(c, strings.Join(cmd.Args, " ")) var errbuff bytes.Buffer cmd.Dir = gitHome cmd.Env = []string{ fmt.Sprintf("RECEIVE_USER=%s", user), fmt.Sprintf("RECEIVE_REPO=%s", repo), fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint), fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo), fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)), } cmd.Env = append(cmd.Env, os.Environ()...) log.Debugf(c, "Working Dir: %s", cmd.Dir) log.Debugf(c, "Environment: %s", strings.Join(cmd.Env, ",")) inpipe, err := cmd.StdinPipe() if err != nil { return nil, err } cmd.Stdout = channel cmd.Stderr = io.MultiWriter(channel.Stderr(), &errbuff) if err := cmd.Start(); err != nil { err = fmt.Errorf("Failed to start git pre-receive hook: %s (%s)", err, errbuff.Bytes()) log.Warnf(c, err.Error()) return nil, err } if _, err := io.Copy(inpipe, channel); err != nil { err = fmt.Errorf("Failed to write git objects into the git pre-receive hook (%s)", err) log.Warnf(c, err.Error()) return nil, err } fmt.Println("Waiting for git-receive to run.") fmt.Println("Waiting for deploy.") if err := cmd.Wait(); err != nil { err = fmt.Errorf("Failed to run git pre-receive hook: %s (%s)", errbuff.Bytes(), err) log.Errf(c, err.Error()) return nil, err } if errbuff.Len() > 0 { log.Warnf(c, "Unreported error: %s", errbuff.Bytes()) } log.Infof(c, "Deploy complete.\n") return nil, nil }
// answer handles answering requests and channel requests // // Currently, an exec must be either "ping", "git-receive-pack" or // "git-upload-pack". Anything else will result in a failure response. Right // now, we leave the channel open on failure because it is unclear what the // correct behavior for a failed exec is. // // Support for setting environment variables via `env` has been disabled. func (s *server) answer(channel ssh.Channel, requests <-chan *ssh.Request, sshConn string) error { defer channel.Close() // Answer all the requests on this connection. for req := range requests { ok := false // I think that ideally what we want to do here is pass this on to // the Cookoo router and let it handle each Type on its own. switch req.Type { case "env": o := &EnvVar{} ssh.Unmarshal(req.Payload, o) fmt.Printf("Key='%s', Value='%s'\n", o.Name, o.Value) req.Reply(true, nil) case "exec": clean := cleanExec(req.Payload) parts := strings.SplitN(clean, " ", 2) router := s.c.Get("cookoo.Router", nil).(*cookoo.Router) // TODO: Should we unset the context value 'cookoo.Router'? // We need a shallow copy of the context to avoid race conditions. cxt := s.c.Copy() cxt.Put("SSH_CONNECTION", sshConn) // Only allow commands that we know about. switch parts[0] { case "ping": cxt.Put("channel", channel) cxt.Put("request", req) sshPing := cxt.Get("route.sshd.sshPing", "sshPing").(string) err := router.HandleRequest(sshPing, cxt, true) if err != nil { log.Warnf(s.c, "Error pinging: %s", err) } return err case "git-receive-pack", "git-upload-pack": if len(parts) < 2 { log.Warn(s.c, "Expected two-part command.\n") req.Reply(ok, nil) break } req.Reply(true, nil) // We processed. Yay. cxt.Put("channel", channel) cxt.Put("request", req) cxt.Put("operation", parts[0]) cxt.Put("repository", parts[1]) sshGitReceive := cxt.Get("route.sshd.sshGitReceive", "sshGitReceive").(string) err := router.HandleRequest(sshGitReceive, cxt, true) var xs uint32 if err != nil { log.Errf(s.c, "Failed git receive: %v", err) xs = 1 } sendExitStatus(xs, channel) return nil default: log.Warnf(s.c, "Illegal command is '%s'\n", clean) req.Reply(false, nil) return nil } if err := sendExitStatus(0, channel); err != nil { log.Errf(s.c, "Failed to write exit status: %s", err) } return nil default: // We simply ignore all of the other cases and leave the // channel open to take additional requests. log.Infof(s.c, "Received request of type %s\n", req.Type) req.Reply(false, nil) } } return nil }
// RemoveStaleMembers deletes cluster members whose pods are no longer running. // // This queries Kubernetes to determine what etcd pods are running, and then // compares that to the member list in the etcd cluster. It removes any // cluster members who are no longer in the pod list. // // The purpose of this is to keep the cluster membership from deadlocking // when inactive members prevent consensus building. // // Params: // - client (etcd/client.Client): The etcd client // - label (string): The pod label indicating an etcd node // - namespace (string): The namespace we're operating in func RemoveStaleMembers(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { eclient := p.Get("client", nil).(client.Client) label := p.Get("label", "name=deis-etcd-1").(string) ns := p.Get("namespace", "default").(string) // Should probably pass in the client from the context. klient, err := k8s.PodClient() if err != nil { log.Errf(c, "Could not create a Kubernetes client: %s", err) return nil, err } mapi := client.NewMembersAPI(eclient) members := map[string]bool{} idmap := map[string]string{} // Get members from etcd mm, err := mapi.List(dctx()) if err != nil { log.Warnf(c, "Could not get a list of etcd members: %s", err) return nil, err } for _, member := range mm { members[member.Name] = false idmap[member.Name] = member.ID } // Get the pods running with the given label labelSelector, err := labels.Parse(label) if err != nil { log.Errf(c, "Selector failed to parse: %s", err) return nil, err } pods, err := klient.Pods(ns).List(api.ListOptions{LabelSelector: labelSelector}) if err != nil { return nil, err } for _, item := range pods.Items { if _, ok := members[item.Name]; !ok { log.Infof(c, "Etcd pod %s is not in cluster yet.", item.Name) } else { members[item.Name] = true } } // Anything marked false in members should be removed from etcd. deleted := 0 for k, v := range members { if !v { log.Infof(c, "Deleting %s (%s) from etcd cluster members", k, idmap[k]) if err := mapi.Remove(dctx(), idmap[k]); err != nil { log.Errf(c, "Failed to remove %s from cluster. Skipping. %s", k, err) } else { deleted++ } } } return deleted, nil }
// Watch watches a given path, and executes a git check-repos for each event. // // It starts the watcher and then returns. The watcher runs on its own // goroutine. To stop the watching, send the returned channel a bool. // // Params: // - client (Watcher): An Etcd client. // - path (string): The path to watch // // Returns: // - chan bool: Send this a message to stop the watcher. func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) { // etcdctl -C $ETCD watch --recursive /deis/services path := p.Get("path", "/deis/services").(string) cli, ok := p.Has("client") if !ok { return nil, errors.New("No etcd client found.") } client := cli.(Watcher) // Stupid hack because etcd watch seems to be broken, constantly complaining // that the JSON it received is malformed. safely.GoDo(c, func() { for { response, err := client.Watch(path, 0, true, nil, nil) if err != nil { log.Errf(c, "Etcd Watch failed: %s", err) time.Sleep(50 * time.Millisecond) continue } if response.Node == nil { log.Infof(c, "Unexpected Etcd message: %v", response) } git := exec.Command("/home/git/check-repos") if out, err := git.CombinedOutput(); err != nil { log.Errf(c, "Failed git check-repos: %s", err) log.Infof(c, "Output: %s", out) } } }) return nil, nil /* Watch seems to be broken. So we do this stupid watch loop instead. receiver := make(chan *etcd.Response) stop := make(chan bool) // Buffer the channels so that we don't hang waiting for go-etcd to // read off the channel. stopetcd := make(chan bool, 1) stopwatch := make(chan bool, 1) // Watch for errors. safely.GoDo(c, func() { // When a receiver is passed in, no *Response is ever returned. Instead, // Watch acts like an error channel, and receiver gets all of the messages. _, err := client.Watch(path, 0, true, receiver, stopetcd) if err != nil { log.Infof(c, "Watcher stopped with error '%s'", err) stopwatch <- true //close(stopwatch) } }) // Watch for events safely.GoDo(c, func() { for { select { case msg := <-receiver: if msg.Node != nil { log.Infof(c, "Received notification %s for %s", msg.Action, msg.Node.Key) } else { log.Infof(c, "Received unexpected etcd message: %v", msg) } git := exec.Command("/home/git/check-repos") if out, err := git.CombinedOutput(); err != nil { log.Errf(c, "Failed git check-repos: %s", err) log.Infof(c, "Output: %s", out) } case <-stopwatch: c.Logf("debug", "Received signal to stop watching events.") return } } }) // Fan out stop requests. safely.GoDo(c, func() { <-stop stopwatch <- true stopetcd <- true close(stopwatch) close(stopetcd) }) return stop, nil */ }