// AMIStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an AMI for state changes.
func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc {
return func() (interface{}, string, error) {
resp, err := conn.Images([]string{imageId}, ec2.NewFilter())
if err != nil {
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" {
// Set this to nil as if we didn't find anything.
resp = nil
} else if isTransientNetworkError(err) {
// Transient network error, treat it as if we didn't find anything
resp = nil
} else {
log.Printf("Error on AMIStateRefresh: %s", err)
return nil, "", err
}
}
if resp == nil || len(resp.Images) == 0 {
// Sometimes AWS has consistency issues and doesn't see the
// AMI. Return an empty state.
return nil, "", nil
}
i := resp.Images[0]
return i, i.State, nil
}
}
// findSnapshots returns a map of snapshots associated with an AMI
func findSnapshots(amiid string, awsec2 *ec2.EC2) (map[string]string, error) {
snaps := make(map[string]string)
resp, err := awsec2.Images([]string{amiid}, nil)
if err != nil {
return snaps, fmt.Errorf("EC2 API DescribeImages failed: %s", err.Error())
}
for _, image := range resp.Images {
for _, bd := range image.BlockDevices {
if len(bd.SnapshotId) > 0 {
snaps[bd.SnapshotId] = bd.DeviceName
}
}
}
return snaps, nil
}
// WaitForAMI waits for the given AMI ID to become ready.
func WaitForAMI(c *ec2.EC2, imageId string) error {
for {
imageResp, err := c.Images([]string{imageId}, ec2.NewFilter())
if err != nil {
return err
}
if imageResp.Images[0].State == "available" {
return nil
}
log.Printf("Image in state %s, sleeping 2s before checking again",
imageResp.Images[0].State)
time.Sleep(2 * time.Second)
}
}
// WaitForAMI waits for the given AMI ID to become ready.
func WaitForAMI(c *ec2.EC2, imageId string) error {
for {
imageResp, err := c.Images([]string{imageId}, ec2.NewFilter())
if err != nil {
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" {
log.Println("AMI not found, probably state issues on AWS side. Trying again.")
continue
}
return err
}
if imageResp.Images[0].State == "available" {
return nil
}
log.Printf("Image in state %s, sleeping 2s before checking again",
imageResp.Images[0].State)
time.Sleep(2 * time.Second)
}
}
// purgeAMIs purges AMIs based on specified windows
func purgeAMIs(awsec2 *ec2.EC2, instanceNameTag string, windows []window, s *session) error {
filter := ec2.NewFilter()
filter.Add("tag:hostname", instanceNameTag)
imageList, err := awsec2.Images(nil, filter)
if err != nil {
return fmt.Errorf("EC2 API Images failed: %s", err.Error())
}
s.debug(fmt.Sprintf("Found %d total images for %s in %s", len(imageList.Images), instanceNameTag, awsec2.Region.Name))
images := map[string]time.Time{}
for _, image := range imageList.Images {
timestampTag := ""
for _, tag := range image.Tags {
if tag.Key == "timestamp" {
timestampTag = tag.Value
}
}
if len(timestampTag) < 1 {
s.debug(fmt.Sprintf("AMI is missing timestamp tag - skipping: %s", image.Id))
continue
}
timestamp, err := strconv.ParseInt(timestampTag, 10, 64)
if err != nil {
s.debug(fmt.Sprintf("AMI timestamp tag is corrupt - skipping: %s", image.Id))
continue
}
images[image.Id] = time.Unix(timestamp, 0)
}
for _, window := range windows {
s.debug(fmt.Sprintf("Window: 1 per %s from %s-%s", window.interval.String(), window.start, window.stop))
for cursor := window.start; cursor.Before(window.stop); cursor = cursor.Add(window.interval) {
imagesInThisInterval := []string{}
imagesTimes := make(map[string]time.Time)
oldestImage := ""
oldestImageTime := time.Now()
for id, when := range images {
if when.After(cursor) && when.Before(cursor.Add(window.interval)) {
imagesInThisInterval = append(imagesInThisInterval, id)
imagesTimes[id] = when
if when.Before(oldestImageTime) {
oldestImageTime = when
oldestImage = id
}
}
}
if len(imagesInThisInterval) > 1 {
for _, id := range imagesInThisInterval {
if id == oldestImage { // keep the oldest one
s.debug(fmt.Sprintf("Keeping oldest AMI in this window: %s @ %s (%s->%s)", id, imagesTimes[id].Format(timeShortFormat), window.start.Format(timeShortFormat), window.stop.Format(timeShortFormat)))
continue
}
// find snapshots associated with this AMI.
snaps, err := findSnapshots(id, awsec2)
if err != nil {
return fmt.Errorf("EC2 API findSnapshots failed for %s: %s", id, err.Error())
}
// deregister the AMI.
resp, err := awsec2.DeregisterImage(id)
if err != nil {
return fmt.Errorf("EC2 API DeregisterImage failed for %s: %s", id, err.Error())
}
if resp.Return != true {
return fmt.Errorf("EC2 API DeregisterImage error for %s", id)
}
// delete snapshots associated with this AMI.
for snap, _ := range snaps {
if _, err := awsec2.DeleteSnapshots([]string{snap}); err != nil {
return fmt.Errorf("EC2 API DeleteSnapshot failed: %s", err.Error())
}
}
s.debug(fmt.Sprintf("Purged old AMI %s @ %s (%s->%s)", id, imagesTimes[id].Format(timeShortFormat), window.start.Format(timeShortFormat), window.stop.Format(timeShortFormat)))
}
}
}
}
return nil
}
// createAMIs actually creates the AMI(s)
func createAMIs(awsec2 *ec2.EC2, instances []ec2.Instance, s *session) map[string]string {
newAMIs := make(map[string]string)
pendingAMIs := make(map[string]bool)
for _, instance := range instances {
backupAmiName := fmt.Sprintf("%s-%s-%s", s.instanceNameTag, timeStamp, instance.InstanceId)
backupDesc := fmt.Sprintf("%s %s %s", s.instanceNameTag, timeString, instance.InstanceId)
blockDevices := []ec2.BlockDeviceMapping{}
for _, i := range s.ignoreVolumes {
blockDevices = append(blockDevices, ec2.BlockDeviceMapping{DeviceName: i, NoDevice: true})
}
createOpts := ec2.CreateImage{
InstanceId: instance.InstanceId,
Name: backupAmiName,
Description: backupDesc,
NoReboot: true,
BlockDevices: blockDevices,
}
resp, err := awsec2.CreateImage(&createOpts)
if err != nil {
s.fatal(fmt.Sprintf("Error creating new AMI: %s", err.Error()))
}
_, err = awsec2.CreateTags([]string{resp.ImageId}, []ec2.Tag{
{"hostname", s.instanceNameTag},
{"instance", instance.InstanceId},
{"date", timeString},
{"timestamp", timeSecs},
})
if err != nil {
s.fatal(fmt.Sprintf("Error tagging new AMI: %s", err.Error()))
}
newAMIs[resp.ImageId] = instance.InstanceId
pendingAMIs[resp.ImageId] = true
s.debug(fmt.Sprintf("Creating new AMI %s for %s (%s)", resp.ImageId, s.instanceNameTag, instance.InstanceId))
}
// wait for AMIs to be ready
done := make(chan bool)
go func() {
for len(pendingAMIs) > 0 {
s.debug(fmt.Sprintf("Sleeping for %d pending AMIs", len(pendingAMIs)))
time.Sleep(apiPollInterval)
list := []string{}
for k, _ := range pendingAMIs {
list = append(list, k)
}
images, err := awsec2.Images(list, nil)
if err != nil {
s.fatal("EC2 API Images failed")
}
for _, image := range images.Images {
if image.State == "available" {
delete(pendingAMIs, image.Id)
s.ok(fmt.Sprintf("Created new AMI %s", image.Id))
}
}
}
done <- true
}()
select {
case <-done:
case <-time.After(s.timeout):
list := []string{}
for k, _ := range pendingAMIs {
list = append(list, k)
}
s.fatal(fmt.Sprintf("Timeout waiting for AMIs in region %s: %s", s.sourceRegion.Name, strings.Join(list, " ,")))
}
return newAMIs
}