func waitForState(ec2conn *ec2.EC2, originalInstance *ec2.Instance, pending []string, target string) (i *ec2.Instance, err error) {
log.Printf("Waiting for instance state to become: %s", target)
i = originalInstance
for i.State.Name != target {
found := false
for _, allowed := range pending {
if i.State.Name == allowed {
found = true
break
}
}
if !found {
fmt.Errorf("unexpected state '%s', wanted target '%s'", i.State.Name, target)
return
}
var resp *ec2.InstancesResp
resp, err = ec2conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
if err != nil {
return
}
i = &resp.Reservations[0].Instances[0]
time.Sleep(2 * time.Second)
}
return
}
// SSHAddress returns a function that can be given to the SSH communicator
// for determining the SSH address based on the instance DNS name.
func SSHAddress(e *ec2.EC2, port int) func(multistep.StateBag) (string, error) {
return func(state multistep.StateBag) (string, error) {
for j := 0; j < 2; j++ {
var host string
i := state.Get("instance").(*ec2.Instance)
if i.DNSName != "" {
host = i.DNSName
} else if i.VpcId != "" {
if i.PublicIpAddress != "" {
host = i.PublicIpAddress
} else {
host = i.PrivateIpAddress
}
}
if host != "" {
return fmt.Sprintf("%s:%d", host, port), nil
}
r, err := e.Instances([]string{i.InstanceId}, ec2.NewFilter())
if err != nil {
return "", err
}
if len(r.Reservations) == 0 || len(r.Reservations[0].Instances) == 0 {
return "", fmt.Errorf("instance not found: %s", i.InstanceId)
}
state.Put("instance", &r.Reservations[0].Instances[0])
time.Sleep(1 * time.Second)
}
return "", errors.New("couldn't determine IP address for instance")
}
}
// InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an EC2 instance.
func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
return func() (interface{}, string, error) {
resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
if err != nil {
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidInstanceID.NotFound" {
// Set this to nil as if we didn't find anything.
resp = nil
} else if isTransientNetworkError(err) {
// Transient network error, treat it as if we didn't find anything
resp = nil
} else {
log.Printf("Error on InstanceStateRefresh: %s", err)
return nil, "", err
}
}
if resp == nil || len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
// Sometimes AWS just has consistency issues and doesn't see
// our instance yet. Return an empty state.
return nil, "", nil
}
i = &resp.Reservations[0].Instances[0]
return i, i.State.Name, nil
}
}
// InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an EC2 instance.
func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
return func() (interface{}, string, error) {
resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
if err != nil {
return nil, "", err
}
i = &resp.Reservations[0].Instances[0]
return i, i.State.Name, nil
}
}
// findInstances searches for our instances
func findInstances(awsec2 *ec2.EC2, s *session) []ec2.Instance {
filter := ec2.NewFilter()
filter.Add("tag:Name", s.instanceNameTag)
resp, err := awsec2.Instances(nil, filter)
if err != nil {
s.fatal(fmt.Sprintf("EC2 API DescribeInstances failed: %s", err.Error()))
}
instances := []ec2.Instance{}
for _, reservation := range resp.Reservations {
for _, instance := range reservation.Instances {
instances = append(instances, instance)
}
}
return instances
}
// InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an EC2 instance.
func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
return func() (interface{}, string, error) {
resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter())
if err != nil {
log.Printf("Error on InstanceStateRefresh: %s", err)
return nil, "", err
}
if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
// Sometimes AWS just has consistency issues and doesn't see
// our instance yet. Return an empty state.
return nil, "", nil
}
i = &resp.Reservations[0].Instances[0]
return i, i.State.Name, nil
}
}
func (client *Client) ListRegion(c chan []types.Ship, parameters url.Values, region *ec2.EC2) {
log.Infof("List Instances for Region %s", region.Name)
var final []types.Ship
filter := ec2.NewFilter()
filter.Add("tag-key", "docker")
resp, err := region.Instances(nil, filter)
if err != nil {
c <- final
}
for _, pool := range resp.Reservations {
for _, ship := range pool.Instances {
if client.State(ship.State) != "end" {
final = append(final, types.Ship{ship.InstanceId, client.SearchTag(ship.Tags, "Name"), ship.DNSName, ship.PublicIpAddress, client.State(ship.State), "Ubuntu 14.04", ship.InstanceType, 27017, "http", 0, nil, false})
}
}
}
c <- final
}
//getInstanceInfo returns the full ec2 instance info for the given instance ID.
//Note that this is the *instance* id, not the spot request ID, which is different.
func getInstanceInfo(ec2Handle *ec2.EC2, instanceId string) (*ec2.Instance, error) {
amis := []string{instanceId}
resp, err := ec2Handle.Instances(amis, nil)
if err != nil {
return nil, err
}
reservation := resp.Reservations
if len(reservation) < 1 {
return nil, evergreen.Logger.Errorf(slogger.ERROR, "No reservation found for "+
"instance id: %v", instanceId)
}
instances := reservation[0].Instances
if len(instances) < 1 {
return nil, evergreen.Logger.Errorf(slogger.ERROR, "'%v' was not found in "+
"reservation '%v'", instanceId, resp.Reservations[0].ReservationId)
}
return &instances[0], nil
}
func startEC2Instance(ec2Handle *ec2.EC2, options *ec2.RunInstances,
intentHost *host.Host) (*host.Host, *ec2.RunInstancesResp, error) {
// start the instance
resp, err := ec2Handle.RunInstances(options)
if err != nil {
// remove the intent host document
rmErr := intentHost.Remove()
if rmErr != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Could not remove intent host "+
"“%v”: %v", intentHost.Id, rmErr)
}
return nil, nil, evergreen.Logger.Errorf(slogger.ERROR,
"EC2 RunInstances API call returned error: %v", err)
}
evergreen.Logger.Logf(slogger.DEBUG, "Spawned %v instance", len(resp.Instances))
// the instance should have been successfully spawned
instance := resp.Instances[0]
evergreen.Logger.Logf(slogger.DEBUG, "Started %v", instance.InstanceId)
evergreen.Logger.Logf(slogger.DEBUG, "Key name: %v", string(options.KeyName))
// find old intent host
host, err := host.FindOne(host.ById(intentHost.Id))
if host == nil {
return nil, nil, evergreen.Logger.Errorf(slogger.ERROR, "Can't locate "+
"record inserted for intended host “%v”", intentHost.Id)
}
if err != nil {
return nil, nil, evergreen.Logger.Errorf(slogger.ERROR, "Can't locate "+
"record inserted for intended host “%v” due to error: %v",
intentHost.Id, err)
}
// we found the old document now we can insert the new one
host.Id = instance.InstanceId
err = host.Insert()
if err != nil {
return nil, nil, evergreen.Logger.Errorf(slogger.ERROR, "Could not insert "+
"updated host information for “%v” with “%v”: %v", intentHost.Id,
host.Id, err)
}
// remove the intent host document
err = intentHost.Remove()
if err != nil {
return nil, nil, evergreen.Logger.Errorf(slogger.ERROR, "Could not remove "+
"insert host “%v” (replaced by “%v”): %v", intentHost.Id, host.Id,
err)
}
var infoResp *ec2.InstancesResp
instanceInfoRetryCount := 0
instanceInfoMaxRetries := 5
for {
infoResp, err = ec2Handle.Instances([]string{instance.InstanceId}, nil)
if err != nil {
instanceInfoRetryCount++
if instanceInfoRetryCount == instanceInfoMaxRetries {
evergreen.Logger.Errorf(slogger.ERROR, "There was an error querying for the "+
"instance's information and retries are exhausted. The insance may "+
"be up.")
return nil, resp, err
}
evergreen.Logger.Errorf(slogger.DEBUG, "There was an error querying for the "+
"instance's information. Retrying in 30 seconds. Error: %v", err)
time.Sleep(30 * time.Second)
continue
}
break
}
reservations := infoResp.Reservations
if len(reservations) < 1 {
return nil, resp, fmt.Errorf("Reservation was returned as nil, you " +
"may have to check manually")
}
instancesInfo := reservations[0].Instances
if len(instancesInfo) < 1 {
return nil, resp, fmt.Errorf("Reservation appears to have no " +
"associated instances")
}
return host, resp, nil
}