// AMIStateRefreshFunc returns a StateRefreshFunc that is used to watch // an AMI for state changes. func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc { return func() (interface{}, string, error) { resp, err := conn.Images([]string{imageId}, ec2.NewFilter()) if err != nil { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" { // Set this to nil as if we didn't find anything. resp = nil } else if isTransientNetworkError(err) { // Transient network error, treat it as if we didn't find anything resp = nil } else { log.Printf("Error on AMIStateRefresh: %s", err) return nil, "", err } } if resp == nil || len(resp.Images) == 0 { // Sometimes AWS has consistency issues and doesn't see the // AMI. Return an empty state. return nil, "", nil } i := resp.Images[0] return i, i.State, nil } }
// findSnapshots returns a map of snapshots associated with an AMI func findSnapshots(amiid string, awsec2 *ec2.EC2) (map[string]string, error) { snaps := make(map[string]string) resp, err := awsec2.Images([]string{amiid}, nil) if err != nil { return snaps, fmt.Errorf("EC2 API DescribeImages failed: %s", err.Error()) } for _, image := range resp.Images { for _, bd := range image.BlockDevices { if len(bd.SnapshotId) > 0 { snaps[bd.SnapshotId] = bd.DeviceName } } } return snaps, nil }
// WaitForAMI waits for the given AMI ID to become ready. func WaitForAMI(c *ec2.EC2, imageId string) error { for { imageResp, err := c.Images([]string{imageId}, ec2.NewFilter()) if err != nil { return err } if imageResp.Images[0].State == "available" { return nil } log.Printf("Image in state %s, sleeping 2s before checking again", imageResp.Images[0].State) time.Sleep(2 * time.Second) } }
// WaitForAMI waits for the given AMI ID to become ready. func WaitForAMI(c *ec2.EC2, imageId string) error { for { imageResp, err := c.Images([]string{imageId}, ec2.NewFilter()) if err != nil { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" { log.Println("AMI not found, probably state issues on AWS side. Trying again.") continue } return err } if imageResp.Images[0].State == "available" { return nil } log.Printf("Image in state %s, sleeping 2s before checking again", imageResp.Images[0].State) time.Sleep(2 * time.Second) } }
// purgeAMIs purges AMIs based on specified windows func purgeAMIs(awsec2 *ec2.EC2, instanceNameTag string, windows []window, s *session) error { filter := ec2.NewFilter() filter.Add("tag:hostname", instanceNameTag) imageList, err := awsec2.Images(nil, filter) if err != nil { return fmt.Errorf("EC2 API Images failed: %s", err.Error()) } s.debug(fmt.Sprintf("Found %d total images for %s in %s", len(imageList.Images), instanceNameTag, awsec2.Region.Name)) images := map[string]time.Time{} for _, image := range imageList.Images { timestampTag := "" for _, tag := range image.Tags { if tag.Key == "timestamp" { timestampTag = tag.Value } } if len(timestampTag) < 1 { s.debug(fmt.Sprintf("AMI is missing timestamp tag - skipping: %s", image.Id)) continue } timestamp, err := strconv.ParseInt(timestampTag, 10, 64) if err != nil { s.debug(fmt.Sprintf("AMI timestamp tag is corrupt - skipping: %s", image.Id)) continue } images[image.Id] = time.Unix(timestamp, 0) } for _, window := range windows { s.debug(fmt.Sprintf("Window: 1 per %s from %s-%s", window.interval.String(), window.start, window.stop)) for cursor := window.start; cursor.Before(window.stop); cursor = cursor.Add(window.interval) { imagesInThisInterval := []string{} imagesTimes := make(map[string]time.Time) oldestImage := "" oldestImageTime := time.Now() for id, when := range images { if when.After(cursor) && when.Before(cursor.Add(window.interval)) { imagesInThisInterval = append(imagesInThisInterval, id) imagesTimes[id] = when if when.Before(oldestImageTime) { oldestImageTime = when oldestImage = id } } } if len(imagesInThisInterval) > 1 { for _, id := range imagesInThisInterval { if id == oldestImage { // keep the oldest one s.debug(fmt.Sprintf("Keeping oldest AMI in this window: %s @ %s (%s->%s)", id, imagesTimes[id].Format(timeShortFormat), window.start.Format(timeShortFormat), window.stop.Format(timeShortFormat))) continue } // find snapshots associated with this AMI. snaps, err := findSnapshots(id, awsec2) if err != nil { return fmt.Errorf("EC2 API findSnapshots failed for %s: %s", id, err.Error()) } // deregister the AMI. resp, err := awsec2.DeregisterImage(id) if err != nil { return fmt.Errorf("EC2 API DeregisterImage failed for %s: %s", id, err.Error()) } if resp.Return != true { return fmt.Errorf("EC2 API DeregisterImage error for %s", id) } // delete snapshots associated with this AMI. for snap, _ := range snaps { if _, err := awsec2.DeleteSnapshots([]string{snap}); err != nil { return fmt.Errorf("EC2 API DeleteSnapshot failed: %s", err.Error()) } } s.debug(fmt.Sprintf("Purged old AMI %s @ %s (%s->%s)", id, imagesTimes[id].Format(timeShortFormat), window.start.Format(timeShortFormat), window.stop.Format(timeShortFormat))) } } } } return nil }
// createAMIs actually creates the AMI(s) func createAMIs(awsec2 *ec2.EC2, instances []ec2.Instance, s *session) map[string]string { newAMIs := make(map[string]string) pendingAMIs := make(map[string]bool) for _, instance := range instances { backupAmiName := fmt.Sprintf("%s-%s-%s", s.instanceNameTag, timeStamp, instance.InstanceId) backupDesc := fmt.Sprintf("%s %s %s", s.instanceNameTag, timeString, instance.InstanceId) blockDevices := []ec2.BlockDeviceMapping{} for _, i := range s.ignoreVolumes { blockDevices = append(blockDevices, ec2.BlockDeviceMapping{DeviceName: i, NoDevice: true}) } createOpts := ec2.CreateImage{ InstanceId: instance.InstanceId, Name: backupAmiName, Description: backupDesc, NoReboot: true, BlockDevices: blockDevices, } resp, err := awsec2.CreateImage(&createOpts) if err != nil { s.fatal(fmt.Sprintf("Error creating new AMI: %s", err.Error())) } _, err = awsec2.CreateTags([]string{resp.ImageId}, []ec2.Tag{ {"hostname", s.instanceNameTag}, {"instance", instance.InstanceId}, {"date", timeString}, {"timestamp", timeSecs}, }) if err != nil { s.fatal(fmt.Sprintf("Error tagging new AMI: %s", err.Error())) } newAMIs[resp.ImageId] = instance.InstanceId pendingAMIs[resp.ImageId] = true s.debug(fmt.Sprintf("Creating new AMI %s for %s (%s)", resp.ImageId, s.instanceNameTag, instance.InstanceId)) } // wait for AMIs to be ready done := make(chan bool) go func() { for len(pendingAMIs) > 0 { s.debug(fmt.Sprintf("Sleeping for %d pending AMIs", len(pendingAMIs))) time.Sleep(apiPollInterval) list := []string{} for k, _ := range pendingAMIs { list = append(list, k) } images, err := awsec2.Images(list, nil) if err != nil { s.fatal("EC2 API Images failed") } for _, image := range images.Images { if image.State == "available" { delete(pendingAMIs, image.Id) s.ok(fmt.Sprintf("Created new AMI %s", image.Id)) } } } done <- true }() select { case <-done: case <-time.After(s.timeout): list := []string{} for k, _ := range pendingAMIs { list = append(list, k) } s.fatal(fmt.Sprintf("Timeout waiting for AMIs in region %s: %s", s.sourceRegion.Name, strings.Join(list, " ,"))) } return newAMIs }