// Heal restarts the zookeeper using upstart.
func (h zookeeperHealer) Heal() error {
if h.needsHeal() {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing zookeeper")
upStartCmd("stop", "zookeeper", bootstrapMachine.IPAddress)
return upStartCmd("start", "zookeeper", bootstrapMachine.IPAddress)
}
log.Debug("Zookeeper needs no cure, skipping...")
return nil
}
// Heal executes the action for heal the bootstrap machine agent.
func (h bootstrapMachineHealer) Heal() error {
if h.needsHeal() {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing bootstrap juju-machine-agent")
upStartCmd("stop", "juju-machine-agent", bootstrapMachine.IPAddress)
return upStartCmd("start", "juju-machine-agent", bootstrapMachine.IPAddress)
}
log.Debug("Bootstrap juju-machine-agent needs no cure, skipping...")
return nil
}
func collect(ticker <-chan time.Time) {
for _ = range ticker {
log.Debug("Collecting status from provisioner")
err := app.Provisioner.CollectStatus()
if err != nil {
log.Errorf("Failed to collect status within the provisioner: %s.", err)
continue
}
log.Debug("Collecting status from provisioner finished")
}
}
func (h elbInstanceHealer) Heal() error {
apps := h.getUnhealthyApps()
if len(apps) == 0 {
log.Debug("No app is down.")
return nil
}
names := make([]string, len(apps))
i := 0
for n := range apps {
names[i] = n
i++
}
if instances, err := h.checkInstances(names); err == nil && len(instances) > 0 {
for _, instance := range instances {
a := apps[instance.lb]
if err := a.RemoveUnit(instance.id); err != nil {
return err
}
if err := a.AddUnits(1); err != nil {
return err
}
}
}
return nil
}
func (c *Client) jsonFromResponse(resp *http.Response, v interface{}) error {
log.Debug("Parsing response json...")
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Errorf("Got error while parsing json: %s", err)
return err
}
return json.Unmarshal(body, &v)
}
func (c *Client) Destroy(instance *ServiceInstance) error {
log.Debug("Attempting to call destroy of service instance " + instance.Name + " at " + instance.ServiceName + " api")
resp, err := c.issueRequest("/resources/"+instance.Name, "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
// HealTicker execute the registered healers registered by RegisterHealerTicker.
func HealTicker(ticker <-chan time.Time) {
log.Debug("running heal ticker")
var wg sync.WaitGroup
for _ = range ticker {
healers := getHealers()
wg.Add(len(healers))
for name, h := range healers {
log.Debugf("running verification/heal for %s", name)
go func(healer *healer) {
err := healer.heal()
if err != nil {
log.Debug(err.Error())
}
wg.Done()
}(h)
}
wg.Wait()
}
}
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error {
log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
var resp *http.Response
url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp()
resp, err := c.issueRequest(url, "DELETE", nil)
if err == nil && resp.StatusCode > 299 {
msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
return err
}
// Info returns the additional info about a service instance.
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>
func (c *Client) Info(instance *ServiceInstance) ([]map[string]string, error) {
log.Debug("Attempting to call info of service instance " + instance.Name + " at " + instance.ServiceName + " api")
url := "/resources/" + instance.Name
resp, err := c.issueRequest(url, "GET", nil)
if err != nil || resp.StatusCode != 200 {
return nil, err
}
result := []map[string]string{}
err = c.jsonFromResponse(resp, &result)
if err != nil {
return nil, err
}
return result, nil
}
// RegisterHealerTicker register healers from resource.
func RegisterHealerTicker(ticker <-chan time.Time, endpoint string) {
var registerHealer = func() {
log.Debug("running register ticker")
if healers, err := healersFromResource(endpoint); err == nil {
setHealers(healers)
}
}
registerHealer()
go func() {
for _ = range ticker {
registerHealer()
}
}()
}
func upStartCmd(cmd, daemon, machine string) error {
args := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
machine,
"sudo",
cmd,
daemon,
}
log.Debug(strings.Join(args, " "))
return executor().Execute("ssh", args, nil, nil, nil)
}
func (h bootstrapInstanceIDHealer) Heal() error {
if h.needsHeal() {
log.Debug("healing bootstrap instance id")
jujuBucket, err := config.GetString("juju:bucket")
if err != nil {
return err
}
bucket := h.s3().Bucket(jujuBucket)
ec2InstanceID, err := h.bootstrapInstanceID()
if err != nil {
return err
}
return bucket.Put("provider-state", []byte("zookeeper-instances: ["+ec2InstanceID+"]"), "binary/octet-stream", s3.BucketOwnerFull)
}
return nil
}
func (c *Client) Create(instance *ServiceInstance) error {
var err error
log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api")
var resp *http.Response
params := map[string][]string{
"name": {instance.Name},
}
if instance.PlanName != "" {
params["plan"] = []string{instance.PlanName}
}
if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
return nil
}
msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
func (c *Client) issueRequest(path, method string, params map[string][]string) (*http.Response, error) {
log.Debug("Issuing request...")
v := url.Values(params)
var suffix string
var body io.Reader
if method == "DELETE" || method == "GET" {
suffix = "?" + v.Encode()
} else {
body = strings.NewReader(v.Encode())
}
url := strings.TrimRight(c.endpoint, "/") + "/" + strings.Trim(path, "/") + suffix
req, err := http.NewRequest(method, url, body)
if err != nil {
log.Errorf("Got error while creating request: %s", err)
return nil, err
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("Accept", "application/json")
return http.DefaultClient.Do(req)
}
func (p *dockerProvisioner) Restart(app provision.App) error {
containers, err := listContainersByApp(app.GetName())
if err != nil {
log.Errorf("Got error while getting app containers: %s", err)
return err
}
var buf bytes.Buffer
for _, c := range containers {
err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
if err != nil {
log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
log.Debug("Command outputs:")
log.Debugf("out: %s", &buf)
log.Debugf("err: %s", &buf)
return err
}
buf.Reset()
}
return nil
}
func update(units []provision.Unit) {
log.Debug("updating status from provisioner")
var l AppList
var err error
for _, unit := range units {
a, index := l.Search(unit.AppName)
if index > -1 {
a, err = app.GetByName(unit.AppName)
if err != nil {
log.Errorf("collector: app %q not found. Skipping.\n", unit.AppName)
continue
}
a.Units = nil
l.Add(a, index)
}
u := app.Unit{}
u.Name = unit.Name
u.Type = unit.Type
u.Machine = unit.Machine
u.InstanceId = unit.InstanceId
u.Ip = unit.Ip
if unit.Status == provision.StatusStarted && a.State == "" {
a.State = "ready"
}
u.State = string(unit.Status)
a.AddUnit(&u)
}
conn, err := db.Conn()
if err != nil {
log.Errorf("collector failed to connect to the database: %s", err)
return
}
defer conn.Close()
for _, a := range l {
a.Ip, err = app.Provisioner.Addr(a)
if err != nil {
log.Errorf("collector failed to get app (%q) address: %s", a.Name, err)
}
conn.Apps().Update(bson.M{"name": a.Name}, a)
}
}
// Connects into service's api
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>/status/
// The service host here is the private ip of the service instance
// 204 means the service is up, 500 means the service is down
func (c *Client) Status(instance *ServiceInstance) (string, error) {
log.Debug("Attempting to call status of service instance " + instance.Name + " at " + instance.ServiceName + " api")
var (
resp *http.Response
err error
)
url := "/resources/" + instance.Name + "/status"
if resp, err = c.issueRequest(url, "GET", nil); err == nil {
switch resp.StatusCode {
case 202:
return "pending", nil
case 204:
return "up", nil
case 500:
return "down", nil
}
}
msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
log.Error(msg)
err = &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
return "", err
}
func (i *dockerMachineIaaS) DeleteMachine(m *iaas.Machine) error {
buf := &bytes.Buffer{}
debugConf, _ := i.base.GetConfigString("debug")
if debugConf == "" {
debugConf = "false"
}
isDebug, err := strconv.ParseBool(debugConf)
if err != nil {
return errors.Wrap(err, "failed to parse debug config")
}
dockerMachine, err := i.apiFactory(DockerMachineConfig{
OutWriter: buf,
ErrWriter: buf,
IsDebug: isDebug,
})
if err != nil {
return err
}
defer func() {
dockerMachine.Close()
log.Debug(buf.String())
}()
return dockerMachine.DeleteMachine(m)
}
func (t *ec2WaitTask) Run(job monsterqueue.Job) {
params := job.Parameters()
regionOrEndpoint := getRegionOrEndpoint(map[string]string{
"region": params["region"].(string),
"endpoint": params["endpoint"].(string),
}, true)
machineId := params["machineId"].(string)
var timeout int
switch val := params["timeout"].(type) {
case int:
timeout = val
case float64:
timeout = int(val)
}
networkIdx := -1
if idx, ok := params["networkIndex"]; ok {
switch val := idx.(type) {
case int:
networkIdx = val
case float64:
networkIdx = int(val)
}
}
ec2Inst, err := t.iaas.createEC2Handler(regionOrEndpoint)
if err != nil {
job.Error(err)
return
}
var dnsName string
var notifiedSuccess bool
t0 := time.Now()
for {
if time.Since(t0) > time.Duration(2*timeout)*time.Second {
job.Error(errors.New("hard timeout"))
break
}
log.Debugf("ec2: waiting for dnsname for instance %s", machineId)
input := ec2.DescribeInstancesInput{
InstanceIds: []*string{aws.String(machineId)},
}
resp, err := ec2Inst.DescribeInstances(&input)
if err != nil {
log.Debug("ec2: api error")
time.Sleep(1000 * time.Millisecond)
continue
}
if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
job.Error(err)
break
}
instance := resp.Reservations[0].Instances[0]
if networkIdx < 0 {
dnsName = aws.StringValue(instance.PublicDnsName)
} else {
if len(instance.NetworkInterfaces) < networkIdx {
job.Error(errors.New("invalid network-index. "))
break
}
dnsName = aws.StringValue(instance.NetworkInterfaces[networkIdx].PrivateDnsName)
}
if dnsName != "" {
notifiedSuccess, _ = job.Success(dnsName)
break
}
time.Sleep(500 * time.Millisecond)
}
if !notifiedSuccess {
input := ec2.TerminateInstancesInput{
InstanceIds: []*string{aws.String(machineId)},
}
ec2Inst.TerminateInstances(&input)
}
}
// Heal starts the juju-provision-agent using upstart.
func (h bootstrapProvisionHealer) Heal() error {
bootstrapMachine := getBootstrapMachine()
log.Debug("Healing bootstrap juju-provision-agent")
return upStartCmd("start", "juju-provision-agent", bootstrapMachine.IPAddress)
}
func (i *dockerMachineIaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
caPath, _ := i.base.GetConfigString("ca-path")
driverName, ok := params["driver"]
if !ok {
name, errConf := i.base.GetConfigString("driver:name")
if errConf != nil {
return nil, errDriverNotSet
}
driverName = name
params["driver"] = driverName
}
dockerEngineInstallURL, _ := i.getParamOrConfigString("docker-install-url", params)
insecureRegistry, _ := i.getParamOrConfigString("insecure-registry", params)
var engineFlags []string
if f, err := i.getParamOrConfigString("docker-flags", params); err == nil {
engineFlags = strings.Split(f, ",")
}
machineName, ok := params["name"]
if !ok {
name, err := generateMachineName(params["pool"])
if err != nil {
return nil, err
}
machineName = name
} else {
delete(params, "name")
}
userDataFileParam, err := i.base.GetConfigString("driver:user-data-file-param")
if err == nil {
f, errTemp := ioutil.TempFile("", "")
if errTemp != nil {
return nil, errors.Wrap(errTemp, "failed to create userdata file")
}
defer os.RemoveAll(f.Name())
userData, errData := i.base.ReadUserData()
if errData != nil {
return nil, errors.WithMessage(errData, "failed to read userdata")
}
_, errWrite := f.WriteString(userData)
if errWrite != nil {
return nil, errors.Wrap(errWrite, "failed to write local userdata file")
}
params[userDataFileParam] = f.Name()
}
driverOpts := i.buildDriverOpts(driverName, params)
if userDataFileParam != "" {
delete(params, userDataFileParam)
}
buf := &bytes.Buffer{}
debugConf, _ := i.base.GetConfigString("debug")
if debugConf == "" {
debugConf = "false"
}
isDebug, err := strconv.ParseBool(debugConf)
if err != nil {
return nil, errors.Wrap(err, "failed to parse debug config")
}
dockerMachine, err := i.apiFactory(DockerMachineConfig{
CaPath: caPath,
OutWriter: buf,
ErrWriter: buf,
IsDebug: isDebug,
})
if err != nil {
return nil, err
}
defer func() {
dockerMachine.Close()
log.Debug(buf.String())
}()
m, err := dockerMachine.CreateMachine(CreateMachineOpts{
Name: machineName,
DriverName: driverName,
Params: driverOpts,
InsecureRegistry: insecureRegistry,
DockerEngineInstallURL: dockerEngineInstallURL,
ArbitraryFlags: engineFlags,
})
if err != nil {
if m != nil {
errRem := dockerMachine.DeleteMachine(m.Base)
if errRem != nil {
err = tsuruErrors.NewMultiError(err, errors.WithMessage(errRem, "failed to remove machine after error"))
}
}
return nil, err
}
m.Base.CreationParams = params
return m.Base, nil
}