func (i *EC2IaaS) waitForDnsName(ec2Inst *ec2.EC2, instance *ec2.Instance) (*ec2.Instance, error) {
rawWait, _ := i.base.GetConfigString("wait-timeout")
maxWaitTime, _ := strconv.Atoi(rawWait)
if maxWaitTime == 0 {
maxWaitTime = 300
}
q, err := queue.Queue()
if err != nil {
return nil, err
}
taskName := fmt.Sprintf("ec2-wait-machine-%s", i.base.IaaSName)
waitDuration := time.Duration(maxWaitTime) * time.Second
job, err := q.EnqueueWait(taskName, monsterqueue.JobParams{
"region": ec2Inst.Config.Region,
"endpoint": ec2Inst.Config.Endpoint,
"machineId": *instance.InstanceId,
"timeout": maxWaitTime,
}, waitDuration)
if err != nil {
if err == monsterqueue.ErrQueueWaitTimeout {
return nil, fmt.Errorf("ec2: time out after %v waiting for instance %s to start", waitDuration, *instance.InstanceId)
}
return nil, err
}
result, err := job.Result()
if err != nil {
return nil, err
}
instance.PublicDnsName = aws.String(result.(string))
return instance, nil
}
func (p *dockerProvisioner) addNodeForParams(params map[string]string, isRegister bool) (map[string]string, error) {
response := make(map[string]string)
var machineID string
var address string
if isRegister {
address, _ = params["address"]
delete(params, "address")
} else {
desc, _ := iaas.Describe(params["iaas"])
response["description"] = desc
m, err := iaas.CreateMachine(params)
if err != nil {
return response, err
}
address = m.FormatNodeAddress()
machineID = m.Id
}
err := validateNodeAddress(address)
if err != nil {
return response, err
}
node := cluster.Node{Address: address, Metadata: params, CreationStatus: cluster.NodeCreationStatusPending}
err = p.Cluster().Register(node)
if err != nil {
return response, err
}
q, err := queue.Queue()
if err != nil {
return response, err
}
jobParams := monsterqueue.JobParams{"endpoint": address, "machine": machineID, "metadata": params}
_, err = q.Enqueue(bs.QueueTaskName, jobParams)
return response, err
}
func (i *EC2IaaS) Initialize() error {
q, err := queue.Queue()
if err != nil {
return err
}
return q.RegisterTask(&ec2WaitTask{iaas: i})
}
// RegisterQueueTask registers the internal bs queue task for later execution.
func RegisterQueueTask(p DockerProvisioner) error {
q, err := queue.Queue()
if err != nil {
return err
}
return q.RegisterTask(&runBs{provisioner: p})
}
func registerRoutesRebuildTask() error {
q, err := queue.Queue()
if err != nil {
return err
}
return q.RegisterTask(&routesRebuildTask{})
}
func (i *CloudstackIaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
err := validateParams(params)
if err != nil {
return nil, err
}
userData, err := i.base.ReadUserData()
if err != nil {
return nil, err
}
q, err := queue.Queue()
if err != nil {
return nil, err
}
paramsCopy := make(map[string]string)
for k, v := range params {
paramsCopy[k] = v
}
if userData != "" {
paramsCopy["userdata"] = base64.StdEncoding.EncodeToString([]byte(userData))
}
var vmStatus DeployVirtualMachineResponse
err = i.do("deployVirtualMachine", paramsCopy, &vmStatus)
if err != nil {
return nil, err
}
rawWait, _ := i.base.GetConfigString("wait-timeout")
maxWaitTime, _ := strconv.Atoi(rawWait)
if maxWaitTime == 0 {
maxWaitTime = 300
}
waitDuration := time.Duration(maxWaitTime) * time.Second
jobParams := monsterqueue.JobParams{
"jobId": vmStatus.DeployVirtualMachineResponse.JobID,
"vmId": vmStatus.DeployVirtualMachineResponse.ID,
}
if tags, ok := params["tags"]; ok {
jobParams["tags"] = tags
}
if projectId, ok := params["projectid"]; ok {
jobParams["projectId"] = projectId
}
job, err := q.EnqueueWait(i.taskName(machineCreateTaskName), jobParams, waitDuration)
if err != nil {
if err == monsterqueue.ErrQueueWaitTimeout {
return nil, errors.Errorf("cloudstack: time out after %v waiting for instance %s to start", waitDuration, vmStatus.DeployVirtualMachineResponse.ID)
}
return nil, err
}
result, err := job.Result()
if err != nil {
return nil, err
}
ipAddress := result.(string)
m := &iaas.Machine{
Id: vmStatus.DeployVirtualMachineResponse.ID,
Address: ipAddress,
Status: "running",
}
return m, nil
}
func (i *CloudstackIaaS) DeleteMachine(machine *iaas.Machine) error {
q, err := queue.Queue()
if err != nil {
return err
}
rawWait, _ := i.base.GetConfigString("wait-timeout")
maxWaitTime, _ := strconv.Atoi(rawWait)
if maxWaitTime == 0 {
maxWaitTime = 300
}
waitDuration := time.Duration(maxWaitTime) * time.Second
jobParams := monsterqueue.JobParams{"vmId": machine.Id}
if projectId, ok := machine.CreationParams["projectid"]; ok {
jobParams["projectId"] = projectId
}
job, err := q.EnqueueWait(i.taskName(machineDeleteTaskName), jobParams, waitDuration)
if err != nil {
if err == monsterqueue.ErrQueueWaitTimeout {
return errors.Errorf("cloudstack: time out after %v waiting for instance %s to be destroyed", waitDuration, machine.Id)
}
return err
}
_, err = job.Result()
return err
}
func (p *dockerProvisioner) AddNode(opts provision.AddNodeOptions) error {
node := cluster.Node{
Address: opts.Address,
Metadata: opts.Metadata,
CreationStatus: cluster.NodeCreationStatusPending,
CaCert: opts.CaCert,
ClientCert: opts.ClientCert,
ClientKey: opts.ClientKey,
}
if len(opts.CaCert) == 0 && len(p.caCert) > 0 {
node.CaCert = p.caCert
node.ClientCert = p.clientCert
node.ClientKey = p.clientKey
}
err := p.Cluster().Register(node)
if err != nil {
return err
}
q, err := queue.Queue()
if err != nil {
return err
}
jobParams := monsterqueue.JobParams{"endpoint": opts.Address, "metadata": opts.Metadata}
var job monsterqueue.Job
if opts.WaitTO != 0 {
job, err = q.EnqueueWait(internalNodeContainer.QueueTaskName, jobParams, opts.WaitTO)
} else {
_, err = q.Enqueue(internalNodeContainer.QueueTaskName, jobParams)
}
if err == nil && job != nil {
_, err = job.Result()
}
return err
}
func RegisterTask(finder func(string) (RebuildApp, error)) error {
appFinder = finder
q, err := queue.Queue()
if err != nil {
return err
}
return q.RegisterTask(&routesRebuildTask{})
}
func (p *dockerProvisioner) Initialize() error {
q, err := queue.Queue()
if err != nil {
return err
}
err = q.RegisterTask(runBs{})
if err != nil {
return err
}
return p.initDockerCluster()
}
func (i *CloudstackIaaS) Initialize() error {
q, err := queue.Queue()
if err != nil {
return err
}
err = q.RegisterTask(&machineCreate{iaas: i})
if err != nil {
return err
}
return q.RegisterTask(&machineDelete{iaas: i})
}
func (a *autoScaleConfig) addNode(event *autoScaleEvent, modelNodes []*cluster.Node) (*cluster.Node, error) {
metadata, err := chooseMetadataFromNodes(modelNodes)
if err != nil {
return nil, err
}
_, hasIaas := metadata["iaas"]
if !hasIaas {
return nil, fmt.Errorf("no IaaS information in nodes metadata: %#v", metadata)
}
machine, err := iaas.CreateMachineForIaaS(metadata["iaas"], metadata)
if err != nil {
return nil, fmt.Errorf("unable to create machine: %s", err.Error())
}
newAddr := machine.FormatNodeAddress()
event.logMsg("new machine created: %s - Waiting for docker to start...", newAddr)
createdNode := cluster.Node{
Address: newAddr,
Metadata: metadata,
CreationStatus: cluster.NodeCreationStatusPending,
}
err = a.provisioner.Cluster().Register(createdNode)
if err != nil {
machine.Destroy()
return nil, fmt.Errorf("error registering new node %s: %s", newAddr, err.Error())
}
q, err := queue.Queue()
if err == nil {
jobParams := monsterqueue.JobParams{
"endpoint": createdNode.Address,
"machine": machine.Id,
"metadata": createdNode.Metadata,
}
var job monsterqueue.Job
job, err = q.EnqueueWait(bs.QueueTaskName, jobParams, a.WaitTimeNewMachine)
if err == nil {
_, err = job.Result()
}
}
if err != nil {
machine.Destroy()
a.provisioner.Cluster().Unregister(newAddr)
return nil, fmt.Errorf("error running bs task: %s", err)
}
event.logMsg("new machine created: %s - started!", newAddr)
return &createdNode, nil
}
func routesRebuildOrEnqueueOptionalLock(appName string, lock bool) {
if runRoutesRebuildOnce(appName, lock) {
return
}
q, err := queue.Queue()
if err != nil {
log.Errorf("unable to enqueue rebuild routes task: %s", err.Error())
return
}
_, err = q.Enqueue(routesRebuildTaskName, monsterqueue.JobParams{
"appName": appName,
})
if err != nil {
log.Errorf("unable to enqueue rebuild routes task: %s", err.Error())
return
}
}
func (i *EC2IaaS) waitForDnsName(ec2Inst *ec2.EC2, instanceID string, createParams map[string]string) (string, error) {
rawWait, _ := i.base.GetConfigString("wait-timeout")
maxWaitTime, _ := strconv.Atoi(rawWait)
if maxWaitTime == 0 {
maxWaitTime = 300
}
q, err := queue.Queue()
if err != nil {
return "", err
}
taskName := fmt.Sprintf("ec2-wait-machine-%s", i.base.IaaSName)
jobParams := monsterqueue.JobParams{
"region": ec2Inst.Config.Region,
"endpoint": ec2Inst.Config.Endpoint,
"machineId": instanceID,
"timeout": maxWaitTime,
}
if rawInterfaceIdx, ok := createParams["network-index"]; ok {
if interfaceIdx, atoiErr := strconv.Atoi(rawInterfaceIdx); atoiErr == nil {
jobParams["networkIndex"] = interfaceIdx
}
}
waitDuration := time.Duration(maxWaitTime) * time.Second
job, err := q.EnqueueWait(taskName, jobParams, waitDuration)
if err != nil {
if err == monsterqueue.ErrQueueWaitTimeout {
return "", fmt.Errorf("ec2: time out after %v waiting for instance %s to start", waitDuration, instanceID)
}
return "", err
}
result, err := job.Result()
if err != nil {
return "", err
}
return result.(string), nil
}
func (h *NodeHealer) healNode(node *cluster.Node) (cluster.Node, error) {
emptyNode := cluster.Node{}
failingAddr := node.Address
nodeMetadata := node.CleanMetadata()
failingHost := net.URLToHost(failingAddr)
failures := node.FailureCount()
machine, err := iaas.CreateMachineForIaaS(nodeMetadata["iaas"], nodeMetadata)
if err != nil {
node.ResetFailures()
return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error creating new machine: %s", failures, failingHost, err.Error())
}
err = h.provisioner.Cluster().Unregister(failingAddr)
if err != nil {
machine.Destroy()
return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error unregistering old node: %s", failures, failingHost, err.Error())
}
newAddr := machine.FormatNodeAddress()
log.Debugf("New machine created during healing process: %s - Waiting for docker to start...", newAddr)
createdNode := cluster.Node{
Address: newAddr,
Metadata: nodeMetadata,
CreationStatus: cluster.NodeCreationStatusPending,
}
err = h.provisioner.Cluster().Register(createdNode)
if err != nil {
node.ResetFailures()
h.provisioner.Cluster().Register(cluster.Node{Address: failingAddr, Metadata: nodeMetadata})
machine.Destroy()
return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error registering new node: %s", failures, failingHost, err.Error())
}
q, err := queue.Queue()
if err != nil {
return emptyNode, err
}
jobParams := monsterqueue.JobParams{
"endpoint": createdNode.Address,
"machine": machine.Id,
"metadata": createdNode.Metadata,
}
job, err := q.EnqueueWait(bs.QueueTaskName, jobParams, h.waitTimeNewMachine)
if err == nil {
_, err = job.Result()
}
if err != nil {
node.ResetFailures()
h.provisioner.Cluster().Register(cluster.Node{Address: failingAddr, Metadata: nodeMetadata})
return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error waiting for the bs task: %s", failures, failingHost, err.Error())
}
var buf bytes.Buffer
err = h.provisioner.MoveContainers(failingHost, "", &buf)
if err != nil {
log.Errorf("Unable to move containers, skipping containers healing %q -> %q: %s: %s", failingHost, machine.Address, err.Error(), buf.String())
}
failingMachine, err := iaas.FindMachineByIdOrAddress(node.Metadata["iaas-id"], failingHost)
if err != nil {
return createdNode, fmt.Errorf("Unable to find failing machine %s in IaaS: %s", failingHost, err.Error())
}
err = failingMachine.Destroy()
if err != nil {
return createdNode, fmt.Errorf("Unable to destroy machine %s from IaaS: %s", failingHost, err.Error())
}
log.Debugf("Done auto-healing node %q, node %q created in its place.", failingHost, machine.Address)
return createdNode, nil
}