// RemoveMemberByName removes a member whose name matches the given.
//
// Params:
// - client(client.Client): An etcd client
// - name (string): The name to remove
// Returns:
// true if the member was found, false otherwise.
func RemoveMemberByName(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
cli := p.Get("client", nil).(client.Client)
name := p.Get("name", "____").(string)
mem := client.NewMembersAPI(cli)
members, err := mem.List(dctx())
if err != nil {
log.Errf(c, "Could not get a list of members: %s", err)
return false, err
}
remIDs := []string{}
for _, member := range members {
if member.Name == name {
log.Infof(c, "Removing member %s (ID: %s)", name, member.ID)
// If this is synchronizable, we should do it in parallel.
if err := mem.Remove(dctx(), member.ID); err != nil {
log.Errf(c, "Failed to remove member: %s", err)
return len(remIDs) > 0, err
}
remIDs = append(remIDs, member.ID)
}
}
return len(remIDs) > 0, nil
}
// ParseHostKeys parses the host key files.
//
// By default it looks in /etc/ssh for host keys of the patterh ssh_host_{{TYPE}}_key.
//
// Params:
// - keytypes ([]string): Key types to parse. Defaults to []string{rsa, dsa, ecdsa}
// - enableV1 (bool): Allow V1 keys. By default this is disabled.
// - path (string): Override the lookup pattern. If %s, it will be replaced with the keytype.
//
// Returns:
// []ssh.Signer
func ParseHostKeys(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
log.Debugf(c, "Parsing ssh host keys")
hostKeyTypes := p.Get("keytypes", []string{"rsa", "dsa", "ecdsa"}).([]string)
pathTpl := p.Get("path", "/etc/ssh/ssh_host_%s_key").(string)
hostKeys := make([]ssh.Signer, 0, len(hostKeyTypes))
for _, t := range hostKeyTypes {
path := fmt.Sprintf(pathTpl, t)
if key, err := ioutil.ReadFile(path); err == nil {
if hk, err := ssh.ParsePrivateKey(key); err == nil {
log.Infof(c, "Parsed host key %s.", path)
hostKeys = append(hostKeys, hk)
} else {
log.Errf(c, "Failed to parse host key %s (skipping): %s", path, err)
}
}
}
if c.Get("enableV1", false).(bool) {
path := "/etc/ssh/ssh_host_key"
if key, err := ioutil.ReadFile(path); err != nil {
log.Errf(c, "Failed to read ssh_host_key")
} else if hk, err := ssh.ParsePrivateKey(key); err == nil {
log.Infof(c, "Parsed host key %s.", path)
hostKeys = append(hostKeys, hk)
} else {
log.Errf(c, "Failed to parse host key %s: %s", path, err)
}
}
return hostKeys, nil
}
// iam injects info into the environment about a host's self.
//
// Sets the following environment variables. (Values represent the data format.
// Instances will get its own values.)
//
// MY_NODEIP=10.245.1.3
// MY_SERVICE_IP=10.22.1.4
// MY_PORT_PEER=2380
// MY_PORT_CLIENT=2379
// MY_NAMESPACE=default
// MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp
// MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272
// MY_APISERVER=https://10.247.0.1:443
// MY_NAME=deis-etcd-1-336jp
// MY_IP=10.246.44.7
// MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON
// MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON
// MY_PORT_CLIENT=4100
// MY_PORT_PEER=2380
func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
me, err := aboutme.FromEnv()
if err != nil {
log.Errf(c, "Failed aboutme.FromEnv: %s", err)
} else {
if strings.TrimSpace(me.IP) == "" {
log.Warn(c, "No IP found by API query.")
ip, err := aboutme.MyIP()
if err != nil || ip == "" {
// Force pod death.
log.Errf(c, "Failed to get an IP address: %s", err)
os.Exit(5)
}
}
me.ShuntEnv()
os.Setenv("ETCD_NAME", me.Name)
c.Put("ETCD_NAME", me.Name)
}
passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT")
passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER")
return nil, nil
}
// Watch watches a given path, and executes a git check-repos for each event.
//
// It starts the watcher and then returns. The watcher runs on its own
// goroutine. To stop the watching, send the returned channel a bool.
//
// Params:
// - client (Watcher): An Etcd client.
// - path (string): The path to watch
func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
// etcdctl -C $ETCD watch --recursive /deis/services
path := p.Get("path", "/deis/services").(string)
cli := p.Get("client", nil).(client.Client)
k := client.NewKeysAPI(cli)
watcher := k.Watcher(path, &client.WatcherOptions{Recursive: true})
safely.GoDo(c, func() {
for {
// TODO: We should probably add cancellation support.
response, err := watcher.Next(dctx())
if err != nil {
log.Errf(c, "Etcd Watch failed: %s", err)
}
if response.Node == nil {
log.Infof(c, "Unexpected Etcd message: %v", response)
}
git := exec.Command("/home/git/check-repos")
if out, err := git.CombinedOutput(); err != nil {
log.Errf(c, "Failed git check-repos: %s", err)
log.Infof(c, "Output: %s", out)
}
}
})
return nil, nil
}
// keysToEtcd copies local keys into etcd.
//
// It only fails if it cannot copy ssh_host_key to sshHostKey. All other
// abnormal conditions are logged, but not considered to be failures.
func keysToEtcd(c cookoo.Context, client Setter, ciphers []string, etcdPath string) error {
firstpath := ""
lpath := "/etc/ssh/ssh_host_%s_key"
privkey := "%s/sshHost%sKey"
for _, cipher := range ciphers {
path := fmt.Sprintf(lpath, cipher)
key := fmt.Sprintf(privkey, etcdPath, cipher)
content, err := ioutil.ReadFile(path)
if err != nil {
log.Infof(c, "No key named %s", path)
} else if _, err := client.Set(key, string(content), 0); err != nil {
log.Errf(c, "Could not store ssh key in etcd: %s", err)
}
// Remember the first key's path in case the generic key is missing
if firstpath == "" {
firstpath = path
}
}
// Now we set the generic key:
keypath := "/etc/ssh/ssh_host_key"
if _, err := os.Stat(keypath); os.IsNotExist(err) && firstpath != "" {
// Use ssh_host_dsa_key if newer ssh-keygen didn't create ssh_host_key
keypath = firstpath
}
if content, err := ioutil.ReadFile(keypath); err != nil {
log.Errf(c, "Could not read the %s file.", keypath)
return err
} else if _, err := client.Set("sshHostKey", string(content), 0); err != nil {
log.Errf(c, "Failed to set sshHostKey in etcd.")
return err
}
return nil
}
// Watch watches a given path, and executes a git check-repos for each event.
//
// It starts the watcher and then returns. The watcher runs on its own
// goroutine. To stop the watching, send the returned channel a bool.
//
// Params:
// - client (Watcher): An Etcd client.
// - path (string): The path to watch
//
// Returns:
// - chan bool: Send this a message to stop the watcher.
func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
// etcdctl -C $ETCD watch --recursive /deis/services
path := p.Get("path", "/deis/services").(string)
cli, ok := p.Has("client")
if !ok {
return nil, errors.New("No etcd client found.")
}
client := cli.(Watcher)
// Stupid hack because etcd watch seems to be broken, constantly complaining
// that the JSON it received is malformed.
safely.GoDo(c, func() {
for {
response, err := client.Watch(path, 0, true, nil, nil)
if err != nil {
log.Errf(c, "Etcd Watch failed: %s", err)
time.Sleep(50 * time.Millisecond)
continue
}
if response.Node == nil {
log.Infof(c, "Unexpected Etcd message: %v", response)
}
git := exec.Command("/home/git/check-repos")
if out, err := git.CombinedOutput(); err != nil {
log.Errf(c, "Failed git check-repos: %s", err)
log.Infof(c, "Output: %s", out)
}
}
})
return nil, nil
}
// keysToLocal copies SSH host keys from etcd to the local file system.
//
// This only fails if the main key, sshHostKey cannot be stored or retrieved.
func keysToLocal(c cookoo.Context, client Getter, ciphers []string, etcdPath string) error {
lpath := "/etc/ssh/ssh_host_%s_key"
privkey := "%s/sshHost%sKey"
for _, cipher := range ciphers {
path := fmt.Sprintf(lpath, cipher)
key := fmt.Sprintf(privkey, etcdPath, cipher)
res, err := client.Get(key, false, false)
if err != nil || res.Node == nil {
continue
}
content := res.Node.Value
if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil {
log.Errf(c, "Error writing ssh host key file: %s", err)
}
}
// Now get generic key.
res, err := client.Get("sshHostKey", false, false)
if err != nil || res.Node == nil {
return fmt.Errorf("Failed to get sshHostKey from etcd. %v", err)
}
content := res.Node.Value
if err := ioutil.WriteFile("/etc/ssh/ssh_host_key", []byte(content), 0600); err != nil {
log.Errf(c, "Error writing ssh host key file: %s", err)
return err
}
return nil
}
// Run starts the Builder service.
//
// The Builder service is responsible for setting up the local container
// environment and then listening for new builds. The main listening service
// is SSH. Builder listens for new Git commands and then sends those on to
// Git.
//
// Run returns on of the Status* status code constants.
func Run(cmd string) int {
reg, router, ocxt := cookoo.Cookoo()
log.SetFlags(0) // Time is captured elsewhere.
// We layer the context to add better logging and also synchronize
// access so that goroutines don't get into race conditions.
cxt := cookoo.SyncContext(ocxt)
cxt.Put("cookoo.Router", router)
cxt.AddLogger("stdout", os.Stdout)
// Build the routes. See routes.go.
routes(reg)
// Bootstrap the background services. If this fails, we stop.
if err := router.HandleRequest("boot", cxt, false); err != nil {
clog.Errf(cxt, "Fatal errror on boot: %s", err)
return StatusLocalError
}
// Set up the SSH service.
ip := os.Getenv("SSH_HOST_IP")
if ip == "" {
ip = "0.0.0.0"
}
port := os.Getenv("SSH_HOST_PORT")
if port == "" {
port = "2223"
}
cxt.Put(sshd.Address, ip+":"+port)
// Supply route names for handling various internal routing. While this
// isn't necessary for Cookoo, it makes it easy for us to mock these
// routes in tests. c.f. sshd/server.go
cxt.Put("route.sshd.pubkeyAuth", "pubkeyAuth")
cxt.Put("route.sshd.sshPing", "sshPing")
cxt.Put("route.sshd.sshGitReceive", "sshGitReceive")
// Start the SSH service.
// TODO: We could refactor Serve to be a command, and then run this as
// a route.
if err := sshd.Serve(reg, router, cxt); err != nil {
clog.Errf(cxt, "SSH server failed: %s", err)
return StatusLocalError
}
return StatusOk
}
// handleConn handles an individual client connection.
//
// It manages the connection, but passes channels on to `answer()`.
func (s *server) handleConn(conn net.Conn, conf *ssh.ServerConfig) {
defer conn.Close()
log.Info(s.c, "Accepted connection.")
_, chans, reqs, err := ssh.NewServerConn(conn, conf)
if err != nil {
// Handshake failure.
log.Errf(s.c, "Failed handshake: %s (%v)", err, conn)
return
}
// Discard global requests. We're only concerned with channels.
safely.GoDo(s.c, func() { ssh.DiscardRequests(reqs) })
condata := sshConnection(conn)
// Now we handle the channels.
for incoming := range chans {
log.Infof(s.c, "Channel type: %s\n", incoming.ChannelType())
if incoming.ChannelType() != "session" {
incoming.Reject(ssh.UnknownChannelType, "Unknown channel type")
}
channel, req, err := incoming.Accept()
if err != nil {
// Should close request and move on.
panic(err)
}
safely.GoDo(s.c, func() { s.answer(channel, req, condata) })
}
conn.Close()
}
// ParallelBuild runs multiple docker builds at the same time.
//
// Params:
// -images ([]BuildImg): Images to build
// -alwaysFetch (bool): Default false. If set to true, this will always fetch
// the Docker image even if it already exists in the registry.
//
// Returns:
//
// - Waiter: A *sync.WaitGroup that is waiting for the docker downloads to finish.
//
// Context:
//
// This puts 'ParallelBuild.failN" (int) into the context to indicate how many failures
// occurred during fetches.
func ParallelBuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
images := p.Get("images", []BuildImg{}).([]BuildImg)
var wg sync.WaitGroup
var m sync.Mutex
var fails int
for _, img := range images {
img := img
wg.Add(1)
safely.GoDo(c, func() {
log.Infof(c, "Starting build for %s (tag: %s)", img.Path, img.Tag)
if _, err := buildImg(c, img.Path, img.Tag); err != nil {
log.Errf(c, "Failed to build docker image: %s", err)
m.Lock()
fails++
m.Unlock()
}
wg.Done()
})
}
// Number of failures.
c.Put("ParallelBuild.failN", fails)
return &wg, nil
}
// ParallelBuild runs multiple docker builds at the same time.
//
// Params:
// -images ([]BuildImg): Images to build
// -alwaysFetch (bool): Default false. If set to true, this will always fetch
// the Docker image even if it already exists in the registry.
//
// Returns:
//
// - Waiter: A *sync.WaitGroup that is waiting for the docker downloads to finish.
//
// Context:
//
// This puts 'ParallelBuild.failN" (int) into the context to indicate how many failures
// occurred during fetches.
func ParallelBuild(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
images := p.Get("images", []BuildImg{}).([]BuildImg)
var wg sync.WaitGroup
var m sync.Mutex
var fails int
for _, img := range images {
img := img
// HACK: ensure "docker build" is serialized by allowing only one entry in
// the WaitGroup. This works around the "simultaneous docker pull" bug.
wg.Wait()
wg.Add(1)
safely.GoDo(c, func() {
log.Infof(c, "Starting build for %s (tag: %s)", img.Path, img.Tag)
if _, err := buildImg(c, img.Path, img.Tag); err != nil {
log.Errf(c, "Failed to build docker image: %s", err)
m.Lock()
fails++
m.Unlock()
}
wg.Done()
})
}
// Number of failures.
c.Put("ParallelBuild.failN", fails)
return &wg, nil
}
// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// - operation (string): e.g. git-receive-pack
// - repoName (string): The repository name, in the form '/REPO.git'.
// - channel (ssh.Channel): The channel.
// - request (*ssh.Request): The channel.
// - gitHome (string): Defaults to /home/git.
// - fingerprint (string): The fingerprint of the user's SSH key.
// - user (string): The name of the Deis user.
//
// Returns:
// - nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
return nil, fmt.Errorf("Missing requirements %q", z)
}
repoName := p.Get("repoName", "").(string)
operation := p.Get("operation", "").(string)
channel := p.Get("channel", nil).(ssh.Channel)
gitHome := p.Get("gitHome", "/home/git").(string)
fingerprint := p.Get("fingerprint", nil).(string)
user := p.Get("user", "").(string)
repo, err := cleanRepoName(repoName)
if err != nil {
log.Warnf(c, "Illegal repo name: %s.", err)
channel.Stderr().Write([]byte("No repo given"))
return nil, err
}
repo += ".git"
if _, err := createRepo(c, filepath.Join(gitHome, repo), gitHome); err != nil {
log.Infof(c, "Did not create new repo: %s", err)
}
cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
log.Infof(c, strings.Join(cmd.Args, " "))
var errbuff bytes.Buffer
cmd.Dir = gitHome
cmd.Env = []string{
fmt.Sprintf("RECEIVE_USER=%s", user),
fmt.Sprintf("RECEIVE_REPO=%s", repo),
fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
}
cmd.Env = append(cmd.Env, os.Environ()...)
done := plumbCommand(cmd, channel, &errbuff)
if err := cmd.Start(); err != nil {
log.Warnf(c, "Failed git receive immediately: %s %s", err, errbuff.Bytes())
return nil, err
}
fmt.Printf("Waiting for git-receive to run.\n")
done.Wait()
fmt.Printf("Waiting for deploy.\n")
if err := cmd.Wait(); err != nil {
log.Errf(c, "Error on command: %s %s", err, errbuff.Bytes())
return nil, err
}
if errbuff.Len() > 0 {
log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
}
log.Infof(c, "Deploy complete.\n")
return nil, nil
}
// genSshKeys generates the default set of SSH host keys.
func genSSHKeys(c cookoo.Context) error {
// Generate a new key
out, err := exec.Command("ssh-keygen", "-A").CombinedOutput()
if err != nil {
log.Infof(c, "ssh-keygen: %s", out)
log.Errf(c, "Failed to generate SSH keys: %s", err)
return err
}
return nil
}
// Ping handles a simple test SSH exec.
//
// Returns the string PONG and exit status 0.
//
// Params:
// - channel (ssh.Channel): The channel to respond on.
// - request (*ssh.Request): The request.
//
func Ping(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
channel := p.Get("channel", nil).(ssh.Channel)
req := p.Get("request", nil).(*ssh.Request)
log.Info(c, "PING\n")
if _, err := channel.Write([]byte("pong")); err != nil {
log.Errf(c, "Failed to write to channel: %s", err)
}
sendExitStatus(0, channel)
req.Reply(true, nil)
return nil, nil
}
// UpdateHostPort intermittently notifies etcd of the builder's address.
//
// If `port` is specified, this will notify etcd at 10 second intervals that
// the builder is listening at $HOST:$PORT, setting the TTL to 20 seconds.
//
// This will notify etcd as long as the local sshd is running.
//
// Params:
// - base (string): The base path to write the data: $base/host and $base/port.
// - host (string): The hostname
// - port (string): The port
// - client (Setter): The client to use to write the data to etcd.
// - sshPid (int): The PID for SSHD. If SSHD dies, this stops notifying.
func UpdateHostPort(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
base := p.Get("base", "").(string)
host := p.Get("host", "").(string)
port := p.Get("port", "").(string)
client := p.Get("client", nil).(Setter)
sshd := p.Get("sshdPid", 0).(int)
// If no port is specified, we don't do anything.
if len(port) == 0 {
log.Infof(c, "No external port provided. Not publishing details.")
return false, nil
}
var ttl uint64 = 20
if err := setHostPort(client, base, host, port, ttl); err != nil {
log.Errf(c, "Etcd error setting host/port: %s", err)
return false, err
}
// Update etcd every ten seconds with this builder's host/port.
safely.GoDo(c, func() {
ticker := time.NewTicker(10 * time.Second)
for range ticker.C {
//log.Infof(c, "Setting SSHD host/port")
if _, err := os.FindProcess(sshd); err != nil {
log.Errf(c, "Lost SSHd process: %s", err)
break
} else {
if err := setHostPort(client, base, host, port, ttl); err != nil {
log.Errf(c, "Etcd error setting host/port: %s", err)
break
}
}
}
ticker.Stop()
})
return true, nil
}
// iam injects info into the environment about a host's self.
//
// Sets the following environment variables. (Values represent the data format.
// Instances will get its own values.)
//
// MY_NODEIP=10.245.1.3
// MY_SERVICE_IP=10.22.1.4
// MY_PORT_PEER=2380
// MY_PORT_CLIENT=2379
// MY_NAMESPACE=default
// MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp
// MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272
// MY_APISERVER=https://10.247.0.1:443
// MY_NAME=deis-etcd-1-336jp
// MY_IP=10.246.44.7
// MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON
// MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON
// MY_PORT_CLIENT=4100
// MY_PORT_PEER=2380
func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
me, err := aboutme.FromEnv()
if err != nil {
log.Errf(c, "Failed aboutme.FromEnv: %s", err)
log.Warn(c, "Attempting to recover.")
}
// This will try to recover whenever IP is not set. Only some fields
// can be recovered. But all we really need is IP and Name.
if strings.TrimSpace(me.IP) == "" {
log.Warn(c, "No IP found by API query.")
ip, err := aboutme.MyIP()
if err != nil || ip == "" {
// Force pod death.
log.Errf(c, "Failed to get an IP address: %s", err)
os.Exit(5)
}
me.IP = ip
}
if strings.TrimSpace(me.Name) == "" {
// Try to set name from DAPI.
me.Name = os.Getenv("POD_NAME")
log.Warnf(c, "Setting name to %q", me.Name)
}
if strings.TrimSpace(me.Namespace) == "" {
// Try to set namespace from DAPI.
me.Namespace = os.Getenv("POD_NAMESPACE")
log.Warnf(c, "Setting name to %q", me.Namespace)
}
me.ShuntEnv()
os.Setenv("ETCD_NAME", me.Name)
c.Put("ETCD_NAME", me.Name)
passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT")
passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER")
return nil, nil
}
// IsRunning checks to see if etcd is running.
//
// It will test `count` times before giving up.
//
// Params:
// - client (EtcdGetter)
// - count (int): Number of times to try before giving up.
//
// Returns:
// boolean true if etcd is listening.
func IsRunning(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
client := p.Get("client", nil).(Getter)
count := p.Get("count", 20).(int)
for i := 0; i < count; i++ {
_, err := client.Get("/", false, false)
if err == nil {
return true, nil
}
log.Infof(c, "Waiting for etcd to come online.")
time.Sleep(250 * time.Millisecond)
}
log.Errf(c, "Etcd is not answering after %d attempts.", count)
return false, &cookoo.FatalError{"Could not connect to Etcd."}
}
// keysToEtcd copies local keys into etcd.
//
// It only fails if it cannot copy ssh_host_key to sshHostKey. All other
// abnormal conditions are logged, but not considered to be failures.
func keysToEtcd(c cookoo.Context, client Setter, ciphers []string, etcdPath string) error {
lpath := "/etc/ssh/ssh_host_%s_key"
privkey := "%s/sshHost%sKey"
for _, cipher := range ciphers {
path := fmt.Sprintf(lpath, cipher)
key := fmt.Sprintf(privkey, etcdPath, cipher)
content, err := ioutil.ReadFile(path)
if err != nil {
log.Infof(c, "No key named %s", path)
} else if _, err := client.Set(key, string(content), 0); err != nil {
log.Errf(c, "Could not store ssh key in etcd: %s", err)
}
}
// Now we set the generic key:
if content, err := ioutil.ReadFile("/etc/ssh/ssh_host_key"); err != nil {
log.Errf(c, "Could not read the ssh_host_key file.")
return err
} else if _, err := client.Set("sshHostKey", string(content), 0); err != nil {
log.Errf(c, "Failed to set sshHostKey in etcd.")
return err
}
return nil
}
// iam injects info into the environment about a host's self.
//
// Sets the following environment variables. (Values represent the data format.
// Instances will get its own values.)
//
// MY_NODEIP=10.245.1.3
// MY_SERVICE_IP=10.22.1.4
// MY_PORT_PEER=2380
// MY_PORT_CLIENT=2379
// MY_NAMESPACE=default
// MY_SELFLINK=/api/v1/namespaces/default/pods/deis-etcd-1-336jp
// MY_UID=62a3b54a-6956-11e5-b8ab-0800279dd272
// MY_APISERVER=https://10.247.0.1:443
// MY_NAME=deis-etcd-1-336jp
// MY_IP=10.246.44.7
// MY_LABEL_NAME=deis-etcd-1 # One entry per label in the JSON
// MY_ANNOTATION_NAME=deis-etcd-1 # One entry per annitation in the JSON
// MY_PORT_CLIENT=4100
// MY_PORT_PEER=2380
func iam(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
me, err := aboutme.FromEnv()
if err != nil {
log.Errf(c, "Failed aboutme.FromEnv: %s", err)
} else {
me.ShuntEnv()
os.Setenv("ETCD_NAME", me.Name)
c.Put("ETCD_NAME", me.Name)
}
passEnv("MY_PORT_CLIENT", "$DEIS_ETCD_1_SERVICE_PORT_CLIENT")
passEnv("MY_PORT_PEER", "$DEIS_ETCD_1_SERVICE_PORT_PEER")
return nil, nil
}
// startEtcd starts a cluster member of a static etcd cluster.
//
// Params:
// - discover (string): Value to pass to etcd --discovery.
// - client (client.Client): A client to the discovery server. This will
// periodically write data there to indicate liveness.
func startEtcd(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
cli := p.Get("client", nil).(client.Client)
// Use config from environment.
cmd := exec.Command("etcd")
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
println(strings.Join(os.Environ(), "\n"))
if err := cmd.Start(); err != nil {
log.Errf(c, "Failed to start etcd: %s", err)
return nil, err
}
// We need a way to tell starting members that there is an existing cluster,
// and that that cluster has met the initial consensus requirements. This
// basically stores a status record that indicates that it is an existing
// and running etcd server. It allows for basically a two hour window
// during which a cluster can be in an uncertain state before the entire
// thing gives up and a new cluster is created.
ticker := time.NewTicker(time.Minute)
expires := time.Hour * 2
go func() {
name := c.Get("ETCD_NAME", "").(string)
tok := c.Get("DEIS_ETCD_DISCOVERY_TOKEN", "").(string)
key := fmt.Sprintf(discovery.ClusterStatusKey, tok, name)
for t := range ticker.C {
etcd.SimpleSet(cli, key, t.String(), expires)
}
}()
if err := cmd.Wait(); err != nil {
ticker.Stop()
log.Errf(c, "Etcd quit unexpectedly: %s", err)
}
return nil, nil
}
// AddMember Add a new member to the cluster.
//
// Conceptually, this is equivalent to `etcdctl member add NAME IP`.
//
// Params:
// - client(client.Client): An etcd client
// - name (string): The name of the member to add.
// - url (string): The peer ip:port or domain: port to use.
//
// Returns:
// An etcd *client.Member.
func AddMember(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
cli := p.Get("client", nil).(client.Client)
name := p.Get("name", "default").(string)
addr := p.Get("url", "127.0.0.1:2380").(string)
mem := client.NewMembersAPI(cli)
member, err := mem.Add(dctx(), addr)
if err != nil {
log.Errf(c, "Failed to add %s to cluster: %s", addr, err)
return nil, err
}
log.Infof(c, "Added %s (%s) to cluster", addr, member.ID)
member.Name = name
return member, nil
}
// setJoinMode determines what mode to start the etcd server in.
//
// In discovery mode, this will use the discovery URL to join a new cluster.
// In "existing" mode, this will join to an existing cluster directly.
//
// Params:
// - client (etcd.Getter): initialized etcd client
// - path (string): path to get. This will go through os.ExpandEnv().
// - desiredLen (string): The number of nodes to expect in etcd. This is
// usually stored as a string.
//
// Returns:
// string "existing" or "new"
func setJoinMode(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
cli := p.Get("client", nil).(client.Client)
dlen := p.Get("desiredLen", "3").(string)
path := p.Get("path", "").(string)
path = os.ExpandEnv(path)
state := "existing"
dint, err := strconv.Atoi(dlen)
if err != nil {
log.Warnf(c, "Expected integer length, got '%s'. Defaulting to 3", dlen)
dint = 3
}
// Ideally, this should look in the /deis/status directory in the discovery
// service. That will indicate how many members have been online in the
// last few hours. This is a good indicator of whether a cluster exists,
// even if not all the hosts are healthy.
res, err := etcd.SimpleGet(cli, path, true)
if err != nil {
// This means that the discovery server is brand new, and nothing
// has written to it yet. So we're new.
if strings.Contains(err.Error(), "Key not found") {
return "new", nil
}
log.Errf(c, "Failed simple get of %s: %s", path, err)
return state, err
}
if !res.Node.Dir {
//return state, errors.New("Expected a directory node in discovery service")
log.Info(c, "No status information found in discovery service. Assuming new.")
state = "new"
} else if len(res.Node.Nodes) < dint {
log.Info(c, "Cluster has not reached consensus number. Assuming new.")
state = "new"
}
os.Setenv("ETCD_INITIAL_CLUSTER_STATE", state)
return state, nil
}
// Run starts confd and runs it in the background.
//
// If the command fails immediately on startup, an error is immediately
// returned. But from that point, a goroutine watches the command and
// reports if the command dies.
//
// Params:
// - node (string): The etcd node to use. (Only etcd is currently supported)
// - interval (int, default:5): The rebuilding interval.
//
// Returns
// bool true if this succeeded.
func Run(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
node := p.Get("node", defaultEtcd).(string)
interval := strconv.Itoa(p.Get("interval", 5).(int))
cmd := exec.Command("confd", "-log-level", "error", "-node", node, "-interval", interval)
if err := cmd.Start(); err != nil {
return false, err
}
log.Infof(c, "Watching confd.")
safely.Go(func() {
if err := cmd.Wait(); err != nil {
// If confd exits, builder will stop functioning as intended. So
// we stop builder and let the environment restart.
log.Errf(c, "Stopping builder. confd exited with error: %s", err)
os.Exit(37)
}
})
return true, nil
}
// Start starts a Docker daemon.
//
// This assumes the presence of the docker client on the host. It does not use
// the API.
func Start(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
// Allow insecure Docker registries on all private network ranges in RFC 1918 and RFC 6598.
dargs := []string{
"-d",
"--bip=172.19.42.1/16",
"--insecure-registry",
"10.0.0.0/8",
"--insecure-registry",
"172.16.0.0/12",
"--insecure-registry",
"192.168.0.0/16",
"--insecure-registry",
"100.64.0.0/10",
"--exec-opt",
"native.cgroupdriver=cgroupfs",
}
// For overlay-ish filesystems, force the overlay to kick in if it exists.
// Then we can check the fstype.
if err := os.MkdirAll("/", 0700); err == nil {
cmd := exec.Command("findmnt", "--noheadings", "--output", "FSTYPE", "--target", "/")
if out, err := cmd.Output(); err == nil && strings.TrimSpace(string(out)) == "overlay" {
dargs = append(dargs, "--storage-driver=overlay")
} else {
log.Infof(c, "File system type: '%s' (%v)", out, err)
}
}
log.Infof(c, "Starting docker with %s", strings.Join(dargs, " "))
cmd := exec.Command("docker", dargs...)
if err := cmd.Start(); err != nil {
log.Errf(c, "Failed to start Docker. %s", err)
return -1, err
}
// Get the PID and return it.
return cmd.Process.Pid, nil
}
// Receive receives a Git repo.
// This will only work for git-receive-pack.
//
// Params:
// - operation (string): e.g. git-receive-pack
// - repoName (string): The repository name, in the form '/REPO.git'.
// - channel (ssh.Channel): The channel.
// - request (*ssh.Request): The channel.
// - gitHome (string): Defaults to /home/git.
// - fingerprint (string): The fingerprint of the user's SSH key.
// - user (string): The name of the Deis user.
//
// Returns:
// - nothing
func Receive(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
if ok, z := p.Requires("channel", "request", "fingerprint", "permissions"); !ok {
return nil, fmt.Errorf("Missing requirements %q", z)
}
repoName := p.Get("repoName", "").(string)
operation := p.Get("operation", "").(string)
channel := p.Get("channel", nil).(ssh.Channel)
gitHome := p.Get("gitHome", "/home/git").(string)
fingerprint := p.Get("fingerprint", nil).(string)
user := p.Get("user", "").(string)
log.Debugf(c, "receiving git repo name: %s, operation: %s, fingerprint: %s, user: %s", repoName, operation, fingerprint, user)
repo, err := cleanRepoName(repoName)
if err != nil {
log.Warnf(c, "Illegal repo name: %s.", err)
channel.Stderr().Write([]byte("No repo given"))
return nil, err
}
repo += ".git"
repoPath := filepath.Join(gitHome, repo)
log.Debugf(c, "creating repo directory %s", repoPath)
if _, err := createRepo(c, repoPath); err != nil {
err = fmt.Errorf("Did not create new repo (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
log.Debugf(c, "writing pre-receive hook under %s", repoPath)
if err := createPreReceiveHook(c, gitHome, repoPath); err != nil {
err = fmt.Errorf("Did not write pre-receive hook (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
cmd := exec.Command("git-shell", "-c", fmt.Sprintf("%s '%s'", operation, repo))
log.Infof(c, strings.Join(cmd.Args, " "))
var errbuff bytes.Buffer
cmd.Dir = gitHome
cmd.Env = []string{
fmt.Sprintf("RECEIVE_USER=%s", user),
fmt.Sprintf("RECEIVE_REPO=%s", repo),
fmt.Sprintf("RECEIVE_FINGERPRINT=%s", fingerprint),
fmt.Sprintf("SSH_ORIGINAL_COMMAND=%s '%s'", operation, repo),
fmt.Sprintf("SSH_CONNECTION=%s", c.Get("SSH_CONNECTION", "0 0 0 0").(string)),
}
cmd.Env = append(cmd.Env, os.Environ()...)
log.Debugf(c, "Working Dir: %s", cmd.Dir)
log.Debugf(c, "Environment: %s", strings.Join(cmd.Env, ","))
inpipe, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
cmd.Stdout = channel
cmd.Stderr = io.MultiWriter(channel.Stderr(), &errbuff)
if err := cmd.Start(); err != nil {
err = fmt.Errorf("Failed to start git pre-receive hook: %s (%s)", err, errbuff.Bytes())
log.Warnf(c, err.Error())
return nil, err
}
if _, err := io.Copy(inpipe, channel); err != nil {
err = fmt.Errorf("Failed to write git objects into the git pre-receive hook (%s)", err)
log.Warnf(c, err.Error())
return nil, err
}
fmt.Println("Waiting for git-receive to run.")
fmt.Println("Waiting for deploy.")
if err := cmd.Wait(); err != nil {
err = fmt.Errorf("Failed to run git pre-receive hook: %s (%s)", errbuff.Bytes(), err)
log.Errf(c, err.Error())
return nil, err
}
if errbuff.Len() > 0 {
log.Warnf(c, "Unreported error: %s", errbuff.Bytes())
}
log.Infof(c, "Deploy complete.\n")
return nil, nil
}
// answer handles answering requests and channel requests
//
// Currently, an exec must be either "ping", "git-receive-pack" or
// "git-upload-pack". Anything else will result in a failure response. Right
// now, we leave the channel open on failure because it is unclear what the
// correct behavior for a failed exec is.
//
// Support for setting environment variables via `env` has been disabled.
func (s *server) answer(channel ssh.Channel, requests <-chan *ssh.Request, sshConn string) error {
defer channel.Close()
// Answer all the requests on this connection.
for req := range requests {
ok := false
// I think that ideally what we want to do here is pass this on to
// the Cookoo router and let it handle each Type on its own.
switch req.Type {
case "env":
o := &EnvVar{}
ssh.Unmarshal(req.Payload, o)
fmt.Printf("Key='%s', Value='%s'\n", o.Name, o.Value)
req.Reply(true, nil)
case "exec":
clean := cleanExec(req.Payload)
parts := strings.SplitN(clean, " ", 2)
router := s.c.Get("cookoo.Router", nil).(*cookoo.Router)
// TODO: Should we unset the context value 'cookoo.Router'?
// We need a shallow copy of the context to avoid race conditions.
cxt := s.c.Copy()
cxt.Put("SSH_CONNECTION", sshConn)
// Only allow commands that we know about.
switch parts[0] {
case "ping":
cxt.Put("channel", channel)
cxt.Put("request", req)
sshPing := cxt.Get("route.sshd.sshPing", "sshPing").(string)
err := router.HandleRequest(sshPing, cxt, true)
if err != nil {
log.Warnf(s.c, "Error pinging: %s", err)
}
return err
case "git-receive-pack", "git-upload-pack":
if len(parts) < 2 {
log.Warn(s.c, "Expected two-part command.\n")
req.Reply(ok, nil)
break
}
req.Reply(true, nil) // We processed. Yay.
cxt.Put("channel", channel)
cxt.Put("request", req)
cxt.Put("operation", parts[0])
cxt.Put("repository", parts[1])
sshGitReceive := cxt.Get("route.sshd.sshGitReceive", "sshGitReceive").(string)
err := router.HandleRequest(sshGitReceive, cxt, true)
var xs uint32
if err != nil {
log.Errf(s.c, "Failed git receive: %v", err)
xs = 1
}
sendExitStatus(xs, channel)
return nil
default:
log.Warnf(s.c, "Illegal command is '%s'\n", clean)
req.Reply(false, nil)
return nil
}
if err := sendExitStatus(0, channel); err != nil {
log.Errf(s.c, "Failed to write exit status: %s", err)
}
return nil
default:
// We simply ignore all of the other cases and leave the
// channel open to take additional requests.
log.Infof(s.c, "Received request of type %s\n", req.Type)
req.Reply(false, nil)
}
}
return nil
}
// RemoveStaleMembers deletes cluster members whose pods are no longer running.
//
// This queries Kubernetes to determine what etcd pods are running, and then
// compares that to the member list in the etcd cluster. It removes any
// cluster members who are no longer in the pod list.
//
// The purpose of this is to keep the cluster membership from deadlocking
// when inactive members prevent consensus building.
//
// Params:
// - client (etcd/client.Client): The etcd client
// - label (string): The pod label indicating an etcd node
// - namespace (string): The namespace we're operating in
func RemoveStaleMembers(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
eclient := p.Get("client", nil).(client.Client)
label := p.Get("label", "name=deis-etcd-1").(string)
ns := p.Get("namespace", "default").(string)
// Should probably pass in the client from the context.
klient, err := k8s.PodClient()
if err != nil {
log.Errf(c, "Could not create a Kubernetes client: %s", err)
return nil, err
}
mapi := client.NewMembersAPI(eclient)
members := map[string]bool{}
idmap := map[string]string{}
// Get members from etcd
mm, err := mapi.List(dctx())
if err != nil {
log.Warnf(c, "Could not get a list of etcd members: %s", err)
return nil, err
}
for _, member := range mm {
members[member.Name] = false
idmap[member.Name] = member.ID
}
// Get the pods running with the given label
labelSelector, err := labels.Parse(label)
if err != nil {
log.Errf(c, "Selector failed to parse: %s", err)
return nil, err
}
pods, err := klient.Pods(ns).List(api.ListOptions{LabelSelector: labelSelector})
if err != nil {
return nil, err
}
for _, item := range pods.Items {
if _, ok := members[item.Name]; !ok {
log.Infof(c, "Etcd pod %s is not in cluster yet.", item.Name)
} else {
members[item.Name] = true
}
}
// Anything marked false in members should be removed from etcd.
deleted := 0
for k, v := range members {
if !v {
log.Infof(c, "Deleting %s (%s) from etcd cluster members", k, idmap[k])
if err := mapi.Remove(dctx(), idmap[k]); err != nil {
log.Errf(c, "Failed to remove %s from cluster. Skipping. %s", k, err)
} else {
deleted++
}
}
}
return deleted, nil
}
// Watch watches a given path, and executes a git check-repos for each event.
//
// It starts the watcher and then returns. The watcher runs on its own
// goroutine. To stop the watching, send the returned channel a bool.
//
// Params:
// - client (Watcher): An Etcd client.
// - path (string): The path to watch
//
// Returns:
// - chan bool: Send this a message to stop the watcher.
func Watch(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
// etcdctl -C $ETCD watch --recursive /deis/services
path := p.Get("path", "/deis/services").(string)
cli, ok := p.Has("client")
if !ok {
return nil, errors.New("No etcd client found.")
}
client := cli.(Watcher)
// Stupid hack because etcd watch seems to be broken, constantly complaining
// that the JSON it received is malformed.
safely.GoDo(c, func() {
for {
response, err := client.Watch(path, 0, true, nil, nil)
if err != nil {
log.Errf(c, "Etcd Watch failed: %s", err)
time.Sleep(50 * time.Millisecond)
continue
}
if response.Node == nil {
log.Infof(c, "Unexpected Etcd message: %v", response)
}
git := exec.Command("/home/git/check-repos")
if out, err := git.CombinedOutput(); err != nil {
log.Errf(c, "Failed git check-repos: %s", err)
log.Infof(c, "Output: %s", out)
}
}
})
return nil, nil
/* Watch seems to be broken. So we do this stupid watch loop instead.
receiver := make(chan *etcd.Response)
stop := make(chan bool)
// Buffer the channels so that we don't hang waiting for go-etcd to
// read off the channel.
stopetcd := make(chan bool, 1)
stopwatch := make(chan bool, 1)
// Watch for errors.
safely.GoDo(c, func() {
// When a receiver is passed in, no *Response is ever returned. Instead,
// Watch acts like an error channel, and receiver gets all of the messages.
_, err := client.Watch(path, 0, true, receiver, stopetcd)
if err != nil {
log.Infof(c, "Watcher stopped with error '%s'", err)
stopwatch <- true
//close(stopwatch)
}
})
// Watch for events
safely.GoDo(c, func() {
for {
select {
case msg := <-receiver:
if msg.Node != nil {
log.Infof(c, "Received notification %s for %s", msg.Action, msg.Node.Key)
} else {
log.Infof(c, "Received unexpected etcd message: %v", msg)
}
git := exec.Command("/home/git/check-repos")
if out, err := git.CombinedOutput(); err != nil {
log.Errf(c, "Failed git check-repos: %s", err)
log.Infof(c, "Output: %s", out)
}
case <-stopwatch:
c.Logf("debug", "Received signal to stop watching events.")
return
}
}
})
// Fan out stop requests.
safely.GoDo(c, func() {
<-stop
stopwatch <- true
stopetcd <- true
close(stopwatch)
close(stopetcd)
})
return stop, nil
*/
}