Exemple #1
0
// Heal restarts the zookeeper using upstart.
func (h zookeeperHealer) Heal() error {
	if h.needsHeal() {
		bootstrapMachine := getBootstrapMachine()
		log.Debug("Healing zookeeper")
		upStartCmd("stop", "zookeeper", bootstrapMachine.IPAddress)
		return upStartCmd("start", "zookeeper", bootstrapMachine.IPAddress)
	}
	log.Debug("Zookeeper needs no cure, skipping...")
	return nil
}
Exemple #2
0
// Heal executes the action for heal the bootstrap machine agent.
func (h bootstrapMachineHealer) Heal() error {
	if h.needsHeal() {
		bootstrapMachine := getBootstrapMachine()
		log.Debug("Healing bootstrap juju-machine-agent")
		upStartCmd("stop", "juju-machine-agent", bootstrapMachine.IPAddress)
		return upStartCmd("start", "juju-machine-agent", bootstrapMachine.IPAddress)
	}
	log.Debug("Bootstrap juju-machine-agent needs no cure, skipping...")
	return nil
}
Exemple #3
0
func collect(ticker <-chan time.Time) {
	for _ = range ticker {
		log.Debug("Collecting status from provisioner")
		err := app.Provisioner.CollectStatus()
		if err != nil {
			log.Errorf("Failed to collect status within the provisioner: %s.", err)
			continue
		}
		log.Debug("Collecting status from provisioner finished")
	}
}
Exemple #4
0
func (h elbInstanceHealer) Heal() error {
	apps := h.getUnhealthyApps()
	if len(apps) == 0 {
		log.Debug("No app is down.")
		return nil
	}
	names := make([]string, len(apps))
	i := 0
	for n := range apps {
		names[i] = n
		i++
	}
	if instances, err := h.checkInstances(names); err == nil && len(instances) > 0 {
		for _, instance := range instances {
			a := apps[instance.lb]
			if err := a.RemoveUnit(instance.id); err != nil {
				return err
			}
			if err := a.AddUnits(1); err != nil {
				return err
			}
		}
	}
	return nil
}
func (c *Client) jsonFromResponse(resp *http.Response, v interface{}) error {
	log.Debug("Parsing response json...")
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("Got error while parsing json: %s", err)
		return err
	}
	return json.Unmarshal(body, &v)
}
func (c *Client) Destroy(instance *ServiceInstance) error {
	log.Debug("Attempting to call destroy of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	resp, err := c.issueRequest("/resources/"+instance.Name, "DELETE", nil)
	if err == nil && resp.StatusCode > 299 {
		msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return err
}
Exemple #7
0
// HealTicker execute the registered healers registered by RegisterHealerTicker.
func HealTicker(ticker <-chan time.Time) {
	log.Debug("running heal ticker")
	var wg sync.WaitGroup
	for _ = range ticker {
		healers := getHealers()
		wg.Add(len(healers))
		for name, h := range healers {
			log.Debugf("running verification/heal for %s", name)
			go func(healer *healer) {
				err := healer.heal()
				if err != nil {
					log.Debug(err.Error())
				}
				wg.Done()
			}(h)
		}
		wg.Wait()
	}
}
func (c *Client) Unbind(instance *ServiceInstance, unit bind.Unit) error {
	log.Debug("Attempting to call unbind of service instance " + instance.Name + " and unit " + unit.GetIp() + " at " + instance.ServiceName + " api")
	var resp *http.Response
	url := "/resources/" + instance.Name + "/hostname/" + unit.GetIp()
	resp, err := c.issueRequest(url, "DELETE", nil)
	if err == nil && resp.StatusCode > 299 {
		msg := fmt.Sprintf("Failed to unbind (%q): %s", url, c.buildErrorMessage(err, resp))
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return err
}
// Info returns the additional info about a service instance.
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>
func (c *Client) Info(instance *ServiceInstance) ([]map[string]string, error) {
	log.Debug("Attempting to call info of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	url := "/resources/" + instance.Name
	resp, err := c.issueRequest(url, "GET", nil)
	if err != nil || resp.StatusCode != 200 {
		return nil, err
	}
	result := []map[string]string{}
	err = c.jsonFromResponse(resp, &result)
	if err != nil {
		return nil, err
	}
	return result, nil
}
Exemple #10
0
// RegisterHealerTicker register healers from resource.
func RegisterHealerTicker(ticker <-chan time.Time, endpoint string) {
	var registerHealer = func() {
		log.Debug("running register ticker")
		if healers, err := healersFromResource(endpoint); err == nil {
			setHealers(healers)
		}
	}
	registerHealer()
	go func() {
		for _ = range ticker {
			registerHealer()
		}
	}()
}
Exemple #11
0
func upStartCmd(cmd, daemon, machine string) error {
	args := []string{
		"-o",
		"StrictHostKeyChecking no",
		"-q",
		"-l",
		"ubuntu",
		machine,
		"sudo",
		cmd,
		daemon,
	}
	log.Debug(strings.Join(args, " "))
	return executor().Execute("ssh", args, nil, nil, nil)
}
Exemple #12
0
func (h bootstrapInstanceIDHealer) Heal() error {
	if h.needsHeal() {
		log.Debug("healing bootstrap instance id")
		jujuBucket, err := config.GetString("juju:bucket")
		if err != nil {
			return err
		}
		bucket := h.s3().Bucket(jujuBucket)
		ec2InstanceID, err := h.bootstrapInstanceID()
		if err != nil {
			return err
		}
		return bucket.Put("provider-state", []byte("zookeeper-instances: ["+ec2InstanceID+"]"), "binary/octet-stream", s3.BucketOwnerFull)
	}
	return nil
}
Exemple #13
0
func (c *Client) Create(instance *ServiceInstance) error {
	var err error
	log.Debug("Attempting to call creation of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	var resp *http.Response
	params := map[string][]string{
		"name": {instance.Name},
	}
	if instance.PlanName != "" {
		params["plan"] = []string{instance.PlanName}
	}
	if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
		return nil
	}
	msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}
Exemple #14
0
func (c *Client) issueRequest(path, method string, params map[string][]string) (*http.Response, error) {
	log.Debug("Issuing request...")
	v := url.Values(params)
	var suffix string
	var body io.Reader
	if method == "DELETE" || method == "GET" {
		suffix = "?" + v.Encode()
	} else {
		body = strings.NewReader(v.Encode())
	}
	url := strings.TrimRight(c.endpoint, "/") + "/" + strings.Trim(path, "/") + suffix
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		log.Errorf("Got error while creating request: %s", err)
		return nil, err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")
	return http.DefaultClient.Do(req)
}
Exemple #15
0
func (p *dockerProvisioner) Restart(app provision.App) error {
	containers, err := listContainersByApp(app.GetName())
	if err != nil {
		log.Errorf("Got error while getting app containers: %s", err)
		return err
	}
	var buf bytes.Buffer
	for _, c := range containers {
		err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
		if err != nil {
			log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
			log.Debug("Command outputs:")
			log.Debugf("out: %s", &buf)
			log.Debugf("err: %s", &buf)
			return err
		}
		buf.Reset()
	}
	return nil
}
Exemple #16
0
func update(units []provision.Unit) {
	log.Debug("updating status from provisioner")
	var l AppList
	var err error
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			a, err = app.GetByName(unit.AppName)
			if err != nil {
				log.Errorf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
			a.Units = nil
			l.Add(a, index)
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		if unit.Status == provision.StatusStarted && a.State == "" {
			a.State = "ready"
		}
		u.State = string(unit.Status)
		a.AddUnit(&u)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Errorf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, err = app.Provisioner.Addr(a)
		if err != nil {
			log.Errorf("collector failed to get app (%q) address: %s", a.Name, err)
		}
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
// Connects into service's api
// The api should be prepared to receive the request,
// like below:
// GET /resources/<name>/status/
// The service host here is the private ip of the service instance
// 204 means the service is up, 500 means the service is down
func (c *Client) Status(instance *ServiceInstance) (string, error) {
	log.Debug("Attempting to call status of service instance " + instance.Name + " at " + instance.ServiceName + " api")
	var (
		resp *http.Response
		err  error
	)
	url := "/resources/" + instance.Name + "/status"
	if resp, err = c.issueRequest(url, "GET", nil); err == nil {
		switch resp.StatusCode {
		case 202:
			return "pending", nil
		case 204:
			return "up", nil
		case 500:
			return "down", nil
		}
	}
	msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	err = &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	return "", err
}
Exemple #18
0
func (i *dockerMachineIaaS) DeleteMachine(m *iaas.Machine) error {
	buf := &bytes.Buffer{}
	debugConf, _ := i.base.GetConfigString("debug")
	if debugConf == "" {
		debugConf = "false"
	}
	isDebug, err := strconv.ParseBool(debugConf)
	if err != nil {
		return errors.Wrap(err, "failed to parse debug config")
	}
	dockerMachine, err := i.apiFactory(DockerMachineConfig{
		OutWriter: buf,
		ErrWriter: buf,
		IsDebug:   isDebug,
	})
	if err != nil {
		return err
	}
	defer func() {
		dockerMachine.Close()
		log.Debug(buf.String())
	}()
	return dockerMachine.DeleteMachine(m)
}
Exemple #19
0
func (t *ec2WaitTask) Run(job monsterqueue.Job) {
	params := job.Parameters()
	regionOrEndpoint := getRegionOrEndpoint(map[string]string{
		"region":   params["region"].(string),
		"endpoint": params["endpoint"].(string),
	}, true)
	machineId := params["machineId"].(string)
	var timeout int
	switch val := params["timeout"].(type) {
	case int:
		timeout = val
	case float64:
		timeout = int(val)
	}
	networkIdx := -1
	if idx, ok := params["networkIndex"]; ok {
		switch val := idx.(type) {
		case int:
			networkIdx = val
		case float64:
			networkIdx = int(val)
		}
	}
	ec2Inst, err := t.iaas.createEC2Handler(regionOrEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	var dnsName string
	var notifiedSuccess bool
	t0 := time.Now()
	for {
		if time.Since(t0) > time.Duration(2*timeout)*time.Second {
			job.Error(errors.New("hard timeout"))
			break
		}
		log.Debugf("ec2: waiting for dnsname for instance %s", machineId)
		input := ec2.DescribeInstancesInput{
			InstanceIds: []*string{aws.String(machineId)},
		}
		resp, err := ec2Inst.DescribeInstances(&input)
		if err != nil {
			log.Debug("ec2: api error")
			time.Sleep(1000 * time.Millisecond)
			continue
		}
		if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
			job.Error(err)
			break
		}
		instance := resp.Reservations[0].Instances[0]
		if networkIdx < 0 {
			dnsName = aws.StringValue(instance.PublicDnsName)
		} else {
			if len(instance.NetworkInterfaces) < networkIdx {
				job.Error(errors.New("invalid network-index. "))
				break
			}
			dnsName = aws.StringValue(instance.NetworkInterfaces[networkIdx].PrivateDnsName)
		}
		if dnsName != "" {
			notifiedSuccess, _ = job.Success(dnsName)
			break
		}
		time.Sleep(500 * time.Millisecond)
	}
	if !notifiedSuccess {
		input := ec2.TerminateInstancesInput{
			InstanceIds: []*string{aws.String(machineId)},
		}
		ec2Inst.TerminateInstances(&input)
	}
}
Exemple #20
0
// Heal starts the juju-provision-agent using upstart.
func (h bootstrapProvisionHealer) Heal() error {
	bootstrapMachine := getBootstrapMachine()
	log.Debug("Healing bootstrap juju-provision-agent")
	return upStartCmd("start", "juju-provision-agent", bootstrapMachine.IPAddress)
}
Exemple #21
0
func (i *dockerMachineIaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
	caPath, _ := i.base.GetConfigString("ca-path")
	driverName, ok := params["driver"]
	if !ok {
		name, errConf := i.base.GetConfigString("driver:name")
		if errConf != nil {
			return nil, errDriverNotSet
		}
		driverName = name
		params["driver"] = driverName
	}
	dockerEngineInstallURL, _ := i.getParamOrConfigString("docker-install-url", params)
	insecureRegistry, _ := i.getParamOrConfigString("insecure-registry", params)
	var engineFlags []string
	if f, err := i.getParamOrConfigString("docker-flags", params); err == nil {
		engineFlags = strings.Split(f, ",")
	}
	machineName, ok := params["name"]
	if !ok {
		name, err := generateMachineName(params["pool"])
		if err != nil {
			return nil, err
		}
		machineName = name
	} else {
		delete(params, "name")
	}
	userDataFileParam, err := i.base.GetConfigString("driver:user-data-file-param")
	if err == nil {
		f, errTemp := ioutil.TempFile("", "")
		if errTemp != nil {
			return nil, errors.Wrap(errTemp, "failed to create userdata file")
		}
		defer os.RemoveAll(f.Name())
		userData, errData := i.base.ReadUserData()
		if errData != nil {
			return nil, errors.WithMessage(errData, "failed to read userdata")
		}
		_, errWrite := f.WriteString(userData)
		if errWrite != nil {
			return nil, errors.Wrap(errWrite, "failed to write local userdata file")
		}
		params[userDataFileParam] = f.Name()
	}
	driverOpts := i.buildDriverOpts(driverName, params)
	if userDataFileParam != "" {
		delete(params, userDataFileParam)
	}
	buf := &bytes.Buffer{}
	debugConf, _ := i.base.GetConfigString("debug")
	if debugConf == "" {
		debugConf = "false"
	}
	isDebug, err := strconv.ParseBool(debugConf)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse debug config")
	}
	dockerMachine, err := i.apiFactory(DockerMachineConfig{
		CaPath:    caPath,
		OutWriter: buf,
		ErrWriter: buf,
		IsDebug:   isDebug,
	})
	if err != nil {
		return nil, err
	}
	defer func() {
		dockerMachine.Close()
		log.Debug(buf.String())
	}()
	m, err := dockerMachine.CreateMachine(CreateMachineOpts{
		Name:                   machineName,
		DriverName:             driverName,
		Params:                 driverOpts,
		InsecureRegistry:       insecureRegistry,
		DockerEngineInstallURL: dockerEngineInstallURL,
		ArbitraryFlags:         engineFlags,
	})
	if err != nil {
		if m != nil {
			errRem := dockerMachine.DeleteMachine(m.Base)
			if errRem != nil {
				err = tsuruErrors.NewMultiError(err, errors.WithMessage(errRem, "failed to remove machine after error"))
			}
		}
		return nil, err
	}
	m.Base.CreationParams = params
	return m.Base, nil
}