Beispiel #1
0
// Heal restarts the zookeeper using upstart.
func (h zookeeperHealer) Heal() error {
	if h.needsHeal() {
		bootstrapMachine := getBootstrapMachine()
		log.Debug("Healing zookeeper")
		upStartCmd("stop", "zookeeper", bootstrapMachine.IPAddress)
		return upStartCmd("start", "zookeeper", bootstrapMachine.IPAddress)
	}
	log.Debug("Zookeeper needs no cure, skipping...")
	return nil
}
Beispiel #2
0
// Heal executes the action for heal the bootstrap machine agent.
func (h bootstrapMachineHealer) Heal() error {
	if h.needsHeal() {
		bootstrapMachine := getBootstrapMachine()
		log.Debug("Healing bootstrap juju-machine-agent")
		upStartCmd("stop", "juju-machine-agent", bootstrapMachine.IPAddress)
		return upStartCmd("start", "juju-machine-agent", bootstrapMachine.IPAddress)
	}
	log.Debug("Bootstrap juju-machine-agent needs no cure, skipping...")
	return nil
}
Beispiel #3
0
func collect(ticker <-chan time.Time) {
	for _ = range ticker {
		log.Debug("Collecting status from provisioner")
		units, err := app.Provisioner.CollectStatus()
		if err != nil {
			log.Errorf("Failed to collect status within the provisioner: %s.", err)
			continue
		}
		update(units)
		log.Debug("Collecting status from provisioner finished")
	}
}
Beispiel #4
0
func (h elbInstanceHealer) Heal() error {
	apps := h.getUnhealthyApps()
	if len(apps) == 0 {
		log.Debug("No app is down.")
		return nil
	}
	names := make([]string, len(apps))
	i := 0
	for n := range apps {
		names[i] = n
		i++
	}
	if instances, err := h.checkInstances(names); err == nil && len(instances) > 0 {
		for _, instance := range instances {
			a := apps[instance.lb]
			if err := a.RemoveUnit(instance.id); err != nil {
				return err
			}
			if err := a.AddUnits(1); err != nil {
				return err
			}
		}
	}
	return nil
}
Beispiel #5
0
func (c *Client) Bind(instance *ServiceInstance, app bind.App, unit bind.Unit) (map[string]string, error) {
	log.Debug("Attempting to call bind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
	var resp *http.Response
	params := map[string][]string{
		"unit-host": {unit.GetIp()},
		"app-host":  {app.GetIp()},
	}
	resp, err := c.issueRequest("/resources/"+instance.Name, "POST", params)
	if err != nil {
		if m, _ := regexp.MatchString("", err.Error()); m {
			return nil, fmt.Errorf("%s api is down.", instance.Name)
		}
		return nil, err
	}
	if err == nil && resp.StatusCode < 300 {
		var result map[string]string
		err = c.jsonFromResponse(resp, &result)
		if err != nil {
			return nil, err
		}
		return result, nil
	}
	if resp.StatusCode == http.StatusPreconditionFailed {
		return nil, &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
	}
	msg := "Failed to bind instance " + instance.Name + " to the unit " + unit.GetIp() + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	return nil, &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
Beispiel #6
0
func (c *Client) Destroy(instance *ServiceInstance) error {
	log.Debug("Attempting to call destroy of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	resp, err := c.issueRequest("/resources/"+instance.Name, "DELETE", nil)
	if err == nil && resp.StatusCode > 299 {
		msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return err
}
Beispiel #7
0
func (c *Client) jsonFromResponse(resp *http.Response, v interface{}) error {
	log.Debug("Parsing response json...")
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Got error while parsing json: %s", err)
		return err
	}
	return json.Unmarshal(body, &v)
}
// HealTicker execute the registered healers registered by RegisterHealerTicker.
func HealTicker(ticker <-chan time.Time) {
	log.Debug("running heal ticker")
	var wg sync.WaitGroup
	for _ = range ticker {
		healers := getHealers()
		wg.Add(len(healers))
		for name, h := range healers {
			log.Debugf("running verification/heal for %s", name)
			go func(healer *healer) {
				err := healer.heal()
				if err != nil {
					log.Debug(err.Error())
				}
				wg.Done()
			}(h)
		}
		wg.Wait()
	}
}
Beispiel #9
0
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error {
	log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
	var resp *http.Response
	url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp()
	resp, err := c.issueRequest(url, "DELETE", nil)
	if err == nil && resp.StatusCode > 299 {
		msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return err
}
Beispiel #10
0
func (c *Client) Create(instance *ServiceInstance) error {
	var err error
	log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	var resp *http.Response
	params := map[string][]string{
		"name": {instance.Name},
	}
	if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
		return nil
	}
	msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
Beispiel #11
0
// Info returns the additional info about a service instance.
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>
func (c *Client) Info(instance *ServiceInstance) ([]map[string]string, error) {
	log.Debug("Attempting to call info of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	url := "/resources/" + instance.Name
	resp, err := c.issueRequest(url, "GET", nil)
	if err != nil || resp.StatusCode != 200 {
		return nil, err
	}
	result := []map[string]string{}
	err = c.jsonFromResponse(resp, &result)
	if err != nil {
		return nil, err
	}
	return result, nil
}
Beispiel #12
0
// RegisterHealerTicker register healers from resource.
func RegisterHealerTicker(ticker <-chan time.Time, endpoint string) {
	var registerHealer = func() {
		log.Debug("running register ticker")
		if healers, err := healersFromResource(endpoint); err == nil {
			setHealers(healers)
		}
	}
	registerHealer()
	go func() {
		for _ = range ticker {
			registerHealer()
		}
	}()
}
Beispiel #13
0
func upStartCmd(cmd, daemon, machine string) error {
	args := []string{
		"-o",
		"StrictHostKeyChecking no",
		"-q",
		"-l",
		"ubuntu",
		machine,
		"sudo",
		cmd,
		daemon,
	}
	log.Debug(strings.Join(args, " "))
	return executor().Execute("ssh", args, nil, nil, nil)
}
Beispiel #14
0
func (h bootstrapInstanceIDHealer) Heal() error {
	if h.needsHeal() {
		log.Debug("healing bootstrap instance id")
		jujuBucket, err := config.GetString("juju:bucket")
		if err != nil {
			return err
		}
		bucket := h.s3().Bucket(jujuBucket)
		ec2InstanceID, err := h.bootstrapInstanceID()
		if err != nil {
			return err
		}
		return bucket.Put("provider-state", []byte("zookeeper-instances: ["+ec2InstanceID+"]"), "binary/octet-stream", s3.BucketOwnerFull)
	}
	return nil
}
Beispiel #15
0
func (c *Client) issueRequest(path, method string, params map[string][]string) (*http.Response, error) {
	log.Debug("Issuing request...")
	v := url.Values(params)
	var suffix string
	var body io.Reader
	if method == "DELETE" || method == "GET" {
		suffix = "?" + v.Encode()
	} else {
		body = strings.NewReader(v.Encode())
	}
	url := strings.TrimRight(c.endpoint, "/") + "/" + strings.Trim(path, "/") + suffix
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		log.Errorf("Got error while creating request: %s", err)
		return nil, err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")
	return http.DefaultClient.Do(req)
}
Beispiel #16
0
func (p *dockerProvisioner) Restart(app provision.App) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		log.Errorf("Got error while getting app containers: %s", err)
		return err
	}
	var buf bytes.Buffer
	for _, c := range containers {
		err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
		if err != nil {
			log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
			log.Debug("Command outputs:")
			log.Debugf("out: %s", &buf)
			log.Debugf("err: %s", &buf)
			return err
		}
		buf.Reset()
	}
	return nil
}
Beispiel #17
0
func update(units []provision.Unit) {
	log.Debug("updating status from provisioner")
	var l AppList
	var err error
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			a, err = app.GetByName(unit.AppName)
			if err != nil {
				log.Errorf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
			a.Units = nil
			l.Add(a, index)
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		if unit.Status == provision.StatusStarted && a.State == "" {
			a.State = "ready"
		}
		u.State = string(unit.Status)
		a.AddUnit(&u)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Errorf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, err = app.Provisioner.Addr(a)
		if err != nil {
			log.Errorf("collector failed to get app (%q) address: %s", a.Name, err)
		}
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
Beispiel #18
0
// Connects into service's api
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>/status/
// The service host here is the private ip of the service instance
// 204 means the service is up, 500 means the service is down
func (c *Client) Status(instance *ServiceInstance) (string, error) {
	log.Debug("Attempting to call status of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	var (
		resp *http.Response
		err  error
	)
	url := "/resources/" + instance.Name + "/status"
	if resp, err = c.issueRequest(url, "GET", nil); err == nil {
		switch resp.StatusCode {
		case 202:
			return "pending", nil
		case 204:
			return "up", nil
		case 500:
			return "down", nil
		}
	}
	msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	err = &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	return "", err
}
Beispiel #19
0
// Heal starts the juju-provision-agent using upstart.
func (h bootstrapProvisionHealer) Heal() error {
	bootstrapMachine := getBootstrapMachine()
	log.Debug("Healing bootstrap juju-provision-agent")
	return upStartCmd("start", "juju-provision-agent", bootstrapMachine.IPAddress)
}