// Heal restarts the zookeeper using upstart.
func (h zookeeperHealer) Heal() error {
if h.needsHeal() {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing zookeeper")
upStartCmd("stop", "zookeeper", bootstrapMachine.IPAddress)
return upStartCmd("start", "zookeeper", bootstrapMachine.IPAddress)
}
log.Debug("Zookeeper needs no cure, skipping...")
return nil
}
// Heal executes the action for heal the bootstrap machine agent.
func (h bootstrapMachineHealer) Heal() error {
if h.needsHeal() {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing bootstrap juju-machine-agent")
upStartCmd("stop", "juju-machine-agent", bootstrapMachine.IPAddress)
return upStartCmd("start", "juju-machine-agent", bootstrapMachine.IPAddress)
}
log.Debug("Bootstrap juju-machine-agent needs no cure, skipping...")
return nil
}
func collect(ticker <-chan time.Time) {
for _ = range ticker {
log.Debug("Collecting status from provisioner")
units, err := app.Provisioner.CollectStatus()
if err != nil {
log.Errorf("Failed to collect status within the provisioner: %s.", err)
continue
}
update(units)
log.Debug("Collecting status from provisioner finished")
}
}
func (h elbInstanceHealer) Heal() error {
apps := h.getUnhealthyApps()
if len(apps) == 0 {
log.Debug("No app is down.")
return nil
}
names := make([]string, len(apps))
i := 0
for n := range apps {
names[i] = n
i++
}
if instances, err := h.checkInstances(names); err == nil && len(instances) > 0 {
for _, instance := range instances {
a := apps[instance.lb]
if err := a.RemoveUnit(instance.id); err != nil {
return err
}
if err := a.AddUnits(1); err != nil {
return err
}
}
}
return nil
}
func (c *Client) Bind(instance *ServiceInstance, app bind.App, unit bind.Unit) (map[string]string, error) {
log.Debug("Attempting to call bind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
var resp *http.Response
params := map[string][]string{
"unit-host": {unit.GetIp()},
"app-host": {app.GetIp()},
}
resp, err := c.issueRequest("/resources/"+instance.Name, "POST", params)
if err != nil {
if m, _ := regexp.MatchString("", err.Error()); m {
return nil, fmt.Errorf("%s api is down.", instance.Name)
}
return nil, err
}
if err == nil && resp.StatusCode < 300 {
var result map[string]string
err = c.jsonFromResponse(resp, &result)
if err != nil {
return nil, err
}
return result, nil
}
if resp.StatusCode == http.StatusPreconditionFailed {
return nil, &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
}
msg := "Failed to bind instance " + instance.Name + " to the unit " + unit.GetIp() + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return nil, &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
func (c *Client) Destroy(instance *ServiceInstance) error {
log.Debug("Attempting to call destroy of service instance " + instance.Name + " at " + instance.ServiceName + " api")
resp, err := c.issueRequest("/resources/"+instance.Name, "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
func (c *Client) jsonFromResponse(resp *http.Response, v interface{}) error {
log.Debug("Parsing response json...")
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Errorf("Got error while parsing json: %s", err)
return err
}
return json.Unmarshal(body, &v)
}
// HealTicker execute the registered healers registered by RegisterHealerTicker.
func HealTicker(ticker <-chan time.Time) {
log.Debug("running heal ticker")
var wg sync.WaitGroup
for _ = range ticker {
healers := getHealers()
wg.Add(len(healers))
for name, h := range healers {
log.Debugf("running verification/heal for %s", name)
go func(healer *healer) {
err := healer.heal()
if err != nil {
log.Debug(err.Error())
}
wg.Done()
}(h)
}
wg.Wait()
}
}
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error {
log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
var resp *http.Response
url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp()
resp, err := c.issueRequest(url, "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
func (c *Client) Create(instance *ServiceInstance) error {
var err error
log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api")
var resp *http.Response
params := map[string][]string{
"name": {instance.Name},
}
if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
return nil
}
msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
// Info returns the additional info about a service instance.
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>
func (c *Client) Info(instance *ServiceInstance) ([]map[string]string, error) {
log.Debug("Attempting to call info of service instance " + instance.Name + " at " + instance.ServiceName + " api")
url := "/resources/" + instance.Name
resp, err := c.issueRequest(url, "GET", nil)
if err != nil || resp.StatusCode != 200 {
return nil, err
}
result := []map[string]string{}
err = c.jsonFromResponse(resp, &result)
if err != nil {
return nil, err
}
return result, nil
}
// RegisterHealerTicker register healers from resource.
func RegisterHealerTicker(ticker <-chan time.Time, endpoint string) {
var registerHealer = func() {
log.Debug("running register ticker")
if healers, err := healersFromResource(endpoint); err == nil {
setHealers(healers)
}
}
registerHealer()
go func() {
for _ = range ticker {
registerHealer()
}
}()
}
func upStartCmd(cmd, daemon, machine string) error {
args := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
machine,
"sudo",
cmd,
daemon,
}
log.Debug(strings.Join(args, " "))
return executor().Execute("ssh", args, nil, nil, nil)
}
func (h bootstrapInstanceIDHealer) Heal() error {
if h.needsHeal() {
log.Debug("healing bootstrap instance id")
jujuBucket, err := config.GetString("juju:bucket")
if err != nil {
return err
}
bucket := h.s3().Bucket(jujuBucket)
ec2InstanceID, err := h.bootstrapInstanceID()
if err != nil {
return err
}
return bucket.Put("provider-state", []byte("zookeeper-instances: ["+ec2InstanceID+"]"), "binary/octet-stream", s3.BucketOwnerFull)
}
return nil
}
func (c *Client) issueRequest(path, method string, params map[string][]string) (*http.Response, error) {
log.Debug("Issuing request...")
v := url.Values(params)
var suffix string
var body io.Reader
if method == "DELETE" || method == "GET" {
suffix = "?" + v.Encode()
} else {
body = strings.NewReader(v.Encode())
}
url := strings.TrimRight(c.endpoint, "/") + "/" + strings.Trim(path, "/") + suffix
req, err := http.NewRequest(method, url, body)
if err != nil {
log.Errorf("Got error while creating request: %s", err)
return nil, err
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("Accept", "application/json")
return http.DefaultClient.Do(req)
}
func (p *dockerProvisioner) Restart(app provision.App) error {
containers, err := listAppContainers(app.GetName())
if err != nil {
log.Errorf("Got error while getting app containers: %s", err)
return err
}
var buf bytes.Buffer
for _, c := range containers {
err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
if err != nil {
log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
log.Debug("Command outputs:")
log.Debugf("out: %s", &buf)
log.Debugf("err: %s", &buf)
return err
}
buf.Reset()
}
return nil
}
func update(units []provision.Unit) {
log.Debug("updating status from provisioner")
var l AppList
var err error
for _, unit := range units {
a, index := l.Search(unit.AppName)
if index > -1 {
a, err = app.GetByName(unit.AppName)
if err != nil {
log.Errorf("collector: app %q not found. Skipping.\n", unit.AppName)
continue
}
a.Units = nil
l.Add(a, index)
}
u := app.Unit{}
u.Name = unit.Name
u.Type = unit.Type
u.Machine = unit.Machine
u.InstanceId = unit.InstanceId
u.Ip = unit.Ip
if unit.Status == provision.StatusStarted && a.State == "" {
a.State = "ready"
}
u.State = string(unit.Status)
a.AddUnit(&u)
}
conn, err := db.Conn()
if err != nil {
log.Errorf("collector failed to connect to the database: %s", err)
return
}
defer conn.Close()
for _, a := range l {
a.Ip, err = app.Provisioner.Addr(a)
if err != nil {
log.Errorf("collector failed to get app (%q) address: %s", a.Name, err)
}
conn.Apps().Update(bson.M{"name": a.Name}, a)
}
}
// Connects into service's api
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>/status/
// The service host here is the private ip of the service instance
// 204 means the service is up, 500 means the service is down
func (c *Client) Status(instance *ServiceInstance) (string, error) {
log.Debug("Attempting to call status of service instance " + instance.Name + " at " + instance.ServiceName + " api")
var (
resp *http.Response
err error
)
url := "/resources/" + instance.Name + "/status"
if resp, err = c.issueRequest(url, "GET", nil); err == nil {
switch resp.StatusCode {
case 202:
return "pending", nil
case 204:
return "up", nil
case 500:
return "down", nil
}
}
msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
err = &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
return "", err
}
// Heal starts the juju-provision-agent using upstart.
func (h bootstrapProvisionHealer) Heal() error {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing bootstrap juju-provision-agent")
return upStartCmd("start", "juju-provision-agent", bootstrapMachine.IPAddress)
}