func (i *Instance) dialSSH(stderr io.Writer) error {
if i.ssh != nil {
return nil
}
i.sshMtx.RUnlock()
i.sshMtx.Lock()
defer i.sshMtx.RLock()
defer i.sshMtx.Unlock()
if i.ssh != nil {
return nil
}
var sc *ssh.Client
err := sshAttempts.Run(func() (err error) {
if stderr != nil {
fmt.Fprintf(stderr, "Attempting to ssh to %s:22...\n", i.IP)
}
sc, err = ssh.Dial("tcp", i.IP+":22", &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{ssh.Password("ubuntu")},
})
return
})
if sc != nil {
i.ssh = sc
}
return err
}
func (c *BaseCluster) instanceRunCmd(cmd string, sshConfig *ssh.ClientConfig, ipAddress string) error {
sshConn, err := ssh.Dial("tcp", ipAddress+":22", sshConfig)
if err != nil {
return err
}
defer sshConn.Close()
return c.instanceRunCmdWithClient(cmd, sshConn, sshConfig.User, ipAddress)
}
func (c *SSHCluster) testAndAddAuthentication(target *TargetServer, sshConfig *ssh.ClientConfig) bool {
conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", target.IP, target.Port), sshConfig)
if err != nil {
return false
}
target.SSHConfig = sshConfig
target.SSHClient = conn
return true
}
func (c *BaseCluster) uploadDebugInfo(t *TargetServer) {
cmd := "sudo flynn-host collect-debug-info"
if t.SSHClient == nil {
var err error
t.SSHClient, err = ssh.Dial("tcp", net.JoinHostPort(t.IP, t.Port), t.SSHConfig)
if err != nil {
c.SendLog(fmt.Sprintf("Error connecting to %s:%s: %s", t.IP, t.Port, err))
return
}
}
if err := c.instanceRunCmdWithClient(cmd, t.SSHClient, t.User, t.IP); err != nil {
c.SendLog(fmt.Sprintf("Error running %s: %s", cmd, err))
}
}
func (c *BaseCluster) attemptSSHConnectionForTarget(t *TargetServer) error {
attempts := 0
maxAttempts := 30
for {
var err error
t.SSHClient, err = ssh.Dial("tcp", net.JoinHostPort(t.IP, t.Port), t.SSHConfig)
if err != nil {
if attempts < maxAttempts {
attempts++
c.SendLog(err.Error())
time.Sleep(time.Second)
continue
}
return err
}
break
}
return nil
}
func (c *DigitalOceanCluster) instanceWaitForSSH(sshConfig *ssh.ClientConfig, ipAddress string) error {
c.base.SendLog(fmt.Sprintf("Waiting for ssh on %s", ipAddress))
timeout := time.After(5 * time.Minute)
for {
sshConn, err := ssh.Dial("tcp", ipAddress+":22", sshConfig)
if err != nil {
if _, ok := err.(*net.OpError); ok {
select {
case <-time.After(5 * time.Second):
continue
case <-timeout:
return err
}
}
return err
}
sshConn.Close()
return nil
}
}
func (c *BaseCluster) instanceRunCmd(cmd string, sshConfig *ssh.ClientConfig, ipAddress string) error {
c.SendLog(fmt.Sprintf("Running `%s` on %s", cmd, ipAddress))
sshConn, err := ssh.Dial("tcp", ipAddress+":22", sshConfig)
if err != nil {
return err
}
defer sshConn.Close()
sess, err := sshConn.NewSession()
if err != nil {
return err
}
stdout, err := sess.StdoutPipe()
if err != nil {
return err
}
stderr, err := sess.StderrPipe()
if err != nil {
return err
}
if err = sess.Start(cmd); err != nil {
return err
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
c.SendLog(scanner.Text())
}
}()
go func() {
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
c.SendLog(scanner.Text())
}
}()
return sess.Wait()
}
func (c *BaseCluster) bootstrap() error {
c.SendLog("Running bootstrap")
if c.SSHKey == nil {
return errors.New("No SSHKey found")
}
// bootstrap only needs to run on one instance
ipAddress := c.InstanceIPs[0]
sshConfig, err := c.sshConfig()
if err != nil {
return nil
}
attempts := 0
maxAttempts := 3
var sshConn *ssh.Client
for {
sshConn, err = ssh.Dial("tcp", ipAddress+":22", sshConfig)
if err != nil {
if attempts < maxAttempts {
attempts += 1
time.Sleep(time.Second)
continue
}
return err
}
break
}
defer sshConn.Close()
sess, err := sshConn.NewSession()
if err != nil {
return err
}
stdout, err := sess.StdoutPipe()
if err != nil {
return err
}
sess.Stderr = os.Stderr
if err := sess.Start(fmt.Sprintf("CLUSTER_DOMAIN=%s flynn-host bootstrap --json", c.Domain.Name)); err != nil {
c.uploadDebugInfo(sshConfig, ipAddress)
return err
}
var keyData struct {
Key string `json:"data"`
}
var loginTokenData struct {
Token string `json:"data"`
}
var controllerCertData struct {
Pin string `json:"pin"`
CACert string `json:"ca_cert"`
}
output := json.NewDecoder(stdout)
for {
var stepRaw json.RawMessage
if err := output.Decode(&stepRaw); err != nil {
if err == io.EOF {
break
}
return err
}
var step stepInfo
if err := json.Unmarshal(stepRaw, &step); err != nil {
return err
}
if step.State == "error" {
c.uploadDebugInfo(sshConfig, ipAddress)
return fmt.Errorf("bootstrap: %s %s error: %s", step.ID, step.Action, step.Error)
}
c.SendLog(fmt.Sprintf("%s: %s", step.ID, step.State))
if step.State != "done" {
continue
}
switch step.ID {
case "controller-key":
if err := json.Unmarshal(*step.Data, &keyData); err != nil {
return err
}
case "controller-cert":
if err := json.Unmarshal(*step.Data, &controllerCertData); err != nil {
return err
}
case "dashboard-login-token":
if err := json.Unmarshal(*step.Data, &loginTokenData); err != nil {
return err
}
case "log-complete":
break
}
}
if keyData.Key == "" || controllerCertData.Pin == "" {
return err
}
c.ControllerKey = keyData.Key
c.ControllerPin = controllerCertData.Pin
c.CACert = controllerCertData.CACert
c.DashboardLoginToken = loginTokenData.Token
if err := c.saveField("ControllerKey", c.ControllerKey); err != nil {
return err
}
if err := c.saveField("ControllerPin", c.ControllerPin); err != nil {
return err
}
if err := c.saveField("CACert", c.CACert); err != nil {
return err
}
if err := c.saveField("DashboardLoginToken", c.DashboardLoginToken); err != nil {
return err
}
if err := sess.Wait(); err != nil {
return err
}
return nil
}
func (c *BaseCluster) bootstrapTarget(t *TargetServer) error {
c.SendLog("Running bootstrap")
if t.SSHClient == nil {
attempts := 0
maxAttempts := 30
for {
var err error
t.SSHClient, err = ssh.Dial("tcp", net.JoinHostPort(t.IP, t.Port), t.SSHConfig)
if err != nil {
if attempts < maxAttempts {
attempts++
c.SendLog(err.Error())
time.Sleep(time.Second)
continue
}
return err
}
break
}
}
sess, err := t.SSHClient.NewSession()
if err != nil {
return err
}
stdout, err := sess.StdoutPipe()
if err != nil {
return err
}
sess.Stderr = os.Stderr
if err := sess.Start(fmt.Sprintf("CLUSTER_DOMAIN=%s flynn-host bootstrap --timeout 240 --min-hosts=%d --discovery=%s --json", c.Domain.Name, c.NumInstances, c.DiscoveryToken)); err != nil {
c.uploadDebugInfo(t)
return err
}
var keyData struct {
Key string `json:"data"`
}
var loginTokenData struct {
Token string `json:"data"`
}
var controllerCertData struct {
Pin string `json:"pin"`
CACert string `json:"ca_cert"`
}
output := json.NewDecoder(stdout)
for {
var stepRaw json.RawMessage
if err := output.Decode(&stepRaw); err != nil {
if err == io.EOF {
break
}
return err
}
var step stepInfo
if err := json.Unmarshal(stepRaw, &step); err != nil {
return err
}
if step.State == "error" {
c.uploadDebugInfo(t)
return fmt.Errorf("bootstrap: %s %s error: %s", step.ID, step.Action, step.Error)
}
c.SendLog(fmt.Sprintf("%s: %s", step.ID, step.State))
if step.State != "done" {
continue
}
switch step.ID {
case "controller-key":
if err := json.Unmarshal(*step.Data, &keyData); err != nil {
return err
}
case "controller-cert":
if err := json.Unmarshal(*step.Data, &controllerCertData); err != nil {
return err
}
case "dashboard-login-token":
if err := json.Unmarshal(*step.Data, &loginTokenData); err != nil {
return err
}
case "log-complete":
break
}
}
if keyData.Key == "" || controllerCertData.Pin == "" {
return err
}
c.ControllerKey = keyData.Key
c.ControllerPin = controllerCertData.Pin
c.CACert = controllerCertData.CACert
c.DashboardLoginToken = loginTokenData.Token
if err := c.saveField("ControllerKey", c.ControllerKey); err != nil {
return err
}
if err := c.saveField("ControllerPin", c.ControllerPin); err != nil {
return err
}
if err := c.saveField("CACert", c.CACert); err != nil {
return err
}
if err := c.saveField("DashboardLoginToken", c.DashboardLoginToken); err != nil {
return err
}
if err := sess.Wait(); err != nil {
return err
}
return nil
}
func (i *Instance) DialSSH() (*ssh.Client, error) {
return ssh.Dial("tcp", i.IP+":22", &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{ssh.Password("ubuntu")},
})
}