// Heal restarts the zookeeper using upstart. func (h zookeeperHealer) Heal() error { if h.needsHeal() { bootstrapMachine := getBootstrapMachine() log.Debug("Healing zookeeper") upStartCmd("stop", "zookeeper", bootstrapMachine.IPAddress) return upStartCmd("start", "zookeeper", bootstrapMachine.IPAddress) } log.Debug("Zookeeper needs no cure, skipping...") return nil }
// Heal executes the action for heal the bootstrap machine agent. func (h bootstrapMachineHealer) Heal() error { if h.needsHeal() { bootstrapMachine := getBootstrapMachine() log.Debug("Healing bootstrap juju-machine-agent") upStartCmd("stop", "juju-machine-agent", bootstrapMachine.IPAddress) return upStartCmd("start", "juju-machine-agent", bootstrapMachine.IPAddress) } log.Debug("Bootstrap juju-machine-agent needs no cure, skipping...") return nil }
func collect(ticker <-chan time.Time) { for _ = range ticker { log.Debug("Collecting status from provisioner") err := app.Provisioner.CollectStatus() if err != nil { log.Errorf("Failed to collect status within the provisioner: %s.", err) continue } log.Debug("Collecting status from provisioner finished") } }
func (h elbInstanceHealer) Heal() error { apps := h.getUnhealthyApps() if len(apps) == 0 { log.Debug("No app is down.") return nil } names := make([]string, len(apps)) i := 0 for n := range apps { names[i] = n i++ } if instances, err := h.checkInstances(names); err == nil && len(instances) > 0 { for _, instance := range instances { a := apps[instance.lb] if err := a.RemoveUnit(instance.id); err != nil { return err } if err := a.AddUnits(1); err != nil { return err } } } return nil }
func (c *Client) jsonFromResponse(resp *http.Response, v interface{}) error { log.Debug("Parsing response json...") defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Errorf("Got error while parsing json: %s", err) return err } return json.Unmarshal(body, &v) }
func (c *Client) Destroy(instance *ServiceInstance) error { log.Debug("Attempting to call destroy of service instance " + instance.Name + " at " + instance.ServiceName + " api") resp, err := c.issueRequest("/resources/"+instance.Name, "DELETE", nil) if err == nil && resp.StatusCode > 299 { msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp) log.Error(msg) return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg} } return err }
// HealTicker execute the registered healers registered by RegisterHealerTicker. func HealTicker(ticker <-chan time.Time) { log.Debug("running heal ticker") var wg sync.WaitGroup for _ = range ticker { healers := getHealers() wg.Add(len(healers)) for name, h := range healers { log.Debugf("running verification/heal for %s", name) go func(healer *healer) { err := healer.heal() if err != nil { log.Debug(err.Error()) } wg.Done() }(h) } wg.Wait() } }
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error { log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api") var resp *http.Response url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp() resp, err := c.issueRequest(url, "DELETE", nil) if err == nil && resp.StatusCode > 299 { msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp)) log.Error(msg) return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg} } return err }
// Info returns the additional info about a service instance. // The api should be prepared to receive the request, // like below: // GET /resources/<name> func (c *Client) Info(instance *ServiceInstance) ([]map[string]string, error) { log.Debug("Attempting to call info of service instance " + instance.Name + " at " + instance.ServiceName + " api") url := "/resources/" + instance.Name resp, err := c.issueRequest(url, "GET", nil) if err != nil || resp.StatusCode != 200 { return nil, err } result := []map[string]string{} err = c.jsonFromResponse(resp, &result) if err != nil { return nil, err } return result, nil }
// RegisterHealerTicker register healers from resource. func RegisterHealerTicker(ticker <-chan time.Time, endpoint string) { var registerHealer = func() { log.Debug("running register ticker") if healers, err := healersFromResource(endpoint); err == nil { setHealers(healers) } } registerHealer() go func() { for _ = range ticker { registerHealer() } }() }
func upStartCmd(cmd, daemon, machine string) error { args := []string{ "-o", "StrictHostKeyChecking no", "-q", "-l", "ubuntu", machine, "sudo", cmd, daemon, } log.Debug(strings.Join(args, " ")) return executor().Execute("ssh", args, nil, nil, nil) }
func (h bootstrapInstanceIDHealer) Heal() error { if h.needsHeal() { log.Debug("healing bootstrap instance id") jujuBucket, err := config.GetString("juju:bucket") if err != nil { return err } bucket := h.s3().Bucket(jujuBucket) ec2InstanceID, err := h.bootstrapInstanceID() if err != nil { return err } return bucket.Put("provider-state", []byte("zookeeper-instances: ["+ec2InstanceID+"]"), "binary/octet-stream", s3.BucketOwnerFull) } return nil }
func (c *Client) Create(instance *ServiceInstance) error { var err error log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api") var resp *http.Response params := map[string][]string{ "name": {instance.Name}, } if instance.PlanName != "" { params["plan"] = []string{instance.PlanName} } if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 { return nil } msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp) log.Error(msg) return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg} }
func (c *Client) issueRequest(path, method string, params map[string][]string) (*http.Response, error) { log.Debug("Issuing request...") v := url.Values(params) var suffix string var body io.Reader if method == "DELETE" || method == "GET" { suffix = "?" + v.Encode() } else { body = strings.NewReader(v.Encode()) } url := strings.TrimRight(c.endpoint, "/") + "/" + strings.Trim(path, "/") + suffix req, err := http.NewRequest(method, url, body) if err != nil { log.Errorf("Got error while creating request: %s", err) return nil, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Accept", "application/json") return http.DefaultClient.Do(req) }
func (p *dockerProvisioner) Restart(app provision.App) error { containers, err := listContainersByApp(app.GetName()) if err != nil { log.Errorf("Got error while getting app containers: %s", err) return err } var buf bytes.Buffer for _, c := range containers { err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart") if err != nil { log.Errorf("Failed to restart %q: %s.", app.GetName(), err) log.Debug("Command outputs:") log.Debugf("out: %s", &buf) log.Debugf("err: %s", &buf) return err } buf.Reset() } return nil }
func update(units []provision.Unit) { log.Debug("updating status from provisioner") var l AppList var err error for _, unit := range units { a, index := l.Search(unit.AppName) if index > -1 { a, err = app.GetByName(unit.AppName) if err != nil { log.Errorf("collector: app %q not found. Skipping.\n", unit.AppName) continue } a.Units = nil l.Add(a, index) } u := app.Unit{} u.Name = unit.Name u.Type = unit.Type u.Machine = unit.Machine u.InstanceId = unit.InstanceId u.Ip = unit.Ip if unit.Status == provision.StatusStarted && a.State == "" { a.State = "ready" } u.State = string(unit.Status) a.AddUnit(&u) } conn, err := db.Conn() if err != nil { log.Errorf("collector failed to connect to the database: %s", err) return } defer conn.Close() for _, a := range l { a.Ip, err = app.Provisioner.Addr(a) if err != nil { log.Errorf("collector failed to get app (%q) address: %s", a.Name, err) } conn.Apps().Update(bson.M{"name": a.Name}, a) } }
// Connects into service's api // The api should be prepared to receive the request, // like below: // GET /resources/<name>/status/ // The service host here is the private ip of the service instance // 204 means the service is up, 500 means the service is down func (c *Client) Status(instance *ServiceInstance) (string, error) { log.Debug("Attempting to call status of service instance " + instance.Name + " at " + instance.ServiceName + " api") var ( resp *http.Response err error ) url := "/resources/" + instance.Name + "/status" if resp, err = c.issueRequest(url, "GET", nil); err == nil { switch resp.StatusCode { case 202: return "pending", nil case 204: return "up", nil case 500: return "down", nil } } msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp) log.Error(msg) err = &errors.HTTP{Code: http.StatusInternalServerError, Message: msg} return "", err }
func (i *dockerMachineIaaS) DeleteMachine(m *iaas.Machine) error { buf := &bytes.Buffer{} debugConf, _ := i.base.GetConfigString("debug") if debugConf == "" { debugConf = "false" } isDebug, err := strconv.ParseBool(debugConf) if err != nil { return errors.Wrap(err, "failed to parse debug config") } dockerMachine, err := i.apiFactory(DockerMachineConfig{ OutWriter: buf, ErrWriter: buf, IsDebug: isDebug, }) if err != nil { return err } defer func() { dockerMachine.Close() log.Debug(buf.String()) }() return dockerMachine.DeleteMachine(m) }
func (t *ec2WaitTask) Run(job monsterqueue.Job) { params := job.Parameters() regionOrEndpoint := getRegionOrEndpoint(map[string]string{ "region": params["region"].(string), "endpoint": params["endpoint"].(string), }, true) machineId := params["machineId"].(string) var timeout int switch val := params["timeout"].(type) { case int: timeout = val case float64: timeout = int(val) } networkIdx := -1 if idx, ok := params["networkIndex"]; ok { switch val := idx.(type) { case int: networkIdx = val case float64: networkIdx = int(val) } } ec2Inst, err := t.iaas.createEC2Handler(regionOrEndpoint) if err != nil { job.Error(err) return } var dnsName string var notifiedSuccess bool t0 := time.Now() for { if time.Since(t0) > time.Duration(2*timeout)*time.Second { job.Error(errors.New("hard timeout")) break } log.Debugf("ec2: waiting for dnsname for instance %s", machineId) input := ec2.DescribeInstancesInput{ InstanceIds: []*string{aws.String(machineId)}, } resp, err := ec2Inst.DescribeInstances(&input) if err != nil { log.Debug("ec2: api error") time.Sleep(1000 * time.Millisecond) continue } if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 { job.Error(err) break } instance := resp.Reservations[0].Instances[0] if networkIdx < 0 { dnsName = aws.StringValue(instance.PublicDnsName) } else { if len(instance.NetworkInterfaces) < networkIdx { job.Error(errors.New("invalid network-index. ")) break } dnsName = aws.StringValue(instance.NetworkInterfaces[networkIdx].PrivateDnsName) } if dnsName != "" { notifiedSuccess, _ = job.Success(dnsName) break } time.Sleep(500 * time.Millisecond) } if !notifiedSuccess { input := ec2.TerminateInstancesInput{ InstanceIds: []*string{aws.String(machineId)}, } ec2Inst.TerminateInstances(&input) } }
// Heal starts the juju-provision-agent using upstart. func (h bootstrapProvisionHealer) Heal() error { bootstrapMachine := getBootstrapMachine() log.Debug("Healing bootstrap juju-provision-agent") return upStartCmd("start", "juju-provision-agent", bootstrapMachine.IPAddress) }
func (i *dockerMachineIaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) { caPath, _ := i.base.GetConfigString("ca-path") driverName, ok := params["driver"] if !ok { name, errConf := i.base.GetConfigString("driver:name") if errConf != nil { return nil, errDriverNotSet } driverName = name params["driver"] = driverName } dockerEngineInstallURL, _ := i.getParamOrConfigString("docker-install-url", params) insecureRegistry, _ := i.getParamOrConfigString("insecure-registry", params) var engineFlags []string if f, err := i.getParamOrConfigString("docker-flags", params); err == nil { engineFlags = strings.Split(f, ",") } machineName, ok := params["name"] if !ok { name, err := generateMachineName(params["pool"]) if err != nil { return nil, err } machineName = name } else { delete(params, "name") } userDataFileParam, err := i.base.GetConfigString("driver:user-data-file-param") if err == nil { f, errTemp := ioutil.TempFile("", "") if errTemp != nil { return nil, errors.Wrap(errTemp, "failed to create userdata file") } defer os.RemoveAll(f.Name()) userData, errData := i.base.ReadUserData() if errData != nil { return nil, errors.WithMessage(errData, "failed to read userdata") } _, errWrite := f.WriteString(userData) if errWrite != nil { return nil, errors.Wrap(errWrite, "failed to write local userdata file") } params[userDataFileParam] = f.Name() } driverOpts := i.buildDriverOpts(driverName, params) if userDataFileParam != "" { delete(params, userDataFileParam) } buf := &bytes.Buffer{} debugConf, _ := i.base.GetConfigString("debug") if debugConf == "" { debugConf = "false" } isDebug, err := strconv.ParseBool(debugConf) if err != nil { return nil, errors.Wrap(err, "failed to parse debug config") } dockerMachine, err := i.apiFactory(DockerMachineConfig{ CaPath: caPath, OutWriter: buf, ErrWriter: buf, IsDebug: isDebug, }) if err != nil { return nil, err } defer func() { dockerMachine.Close() log.Debug(buf.String()) }() m, err := dockerMachine.CreateMachine(CreateMachineOpts{ Name: machineName, DriverName: driverName, Params: driverOpts, InsecureRegistry: insecureRegistry, DockerEngineInstallURL: dockerEngineInstallURL, ArbitraryFlags: engineFlags, }) if err != nil { if m != nil { errRem := dockerMachine.DeleteMachine(m.Base) if errRem != nil { err = tsuruErrors.NewMultiError(err, errors.WithMessage(errRem, "failed to remove machine after error")) } } return nil, err } m.Base.CreationParams = params return m.Base, nil }