// deleteGroups ensures the given groups are deleted, by retrying
// until a timeout or all groups cannot be found anymore.
// This should be used to make sure tests leave no groups around.
func (s *ServerTests) deleteGroups(c *C, groups []ec2.SecurityGroup) {
testAttempt := aws.AttemptStrategy{
Total: 2 * time.Minute,
Delay: 5 * time.Second,
}
for a := testAttempt.Start(); a.Next(); {
deleted := 0
c.Logf("deleting groups %v", groups)
for _, group := range groups {
_, err := s.ec2.DeleteSecurityGroup(group)
if err == nil || errorCode(err) == "InvalidGroup.NotFound" {
c.Logf("group %v deleted", group)
deleted++
continue
}
if err != nil {
c.Logf("retrying; DeleteSecurityGroup returned: %v", err)
}
}
if deleted == len(groups) {
c.Logf("all groups deleted")
return
}
}
c.Fatalf("timeout while waiting %v groups to get deleted!", groups)
}
// deleteVPCs ensures the given VPCs are deleted, by retrying until a
// timeout or all VPC cannot be found anymore. This should be used to
// make sure tests leave no VPCs around.
func (s *ServerTests) deleteVPCs(c *C, ids []string) {
testAttempt := aws.AttemptStrategy{
Total: 2 * time.Minute,
Delay: 5 * time.Second,
}
for a := testAttempt.Start(); a.Next(); {
deleted := 0
c.Logf("deleting VPCs %v", ids)
for _, id := range ids {
_, err := s.ec2.DeleteVPC(id)
if err == nil || errorCode(err) == "InvalidVpcID.NotFound" {
c.Logf("VPC %s deleted", id)
deleted++
continue
}
if err != nil {
c.Logf("retrying; DeleteVPC returned: %v", err)
}
}
if deleted == len(ids) {
c.Logf("all VPCs deleted")
return
}
}
c.Fatalf("timeout while waiting %v VPCs to get deleted!", ids)
}
func (s *ServerTests) waitForAddresses(c *C, nicId string, ips []string, skipPrimary bool) []string {
// Wait for the given IPs to appear on the NIC, retrying if needed.
testAttempt := aws.AttemptStrategy{
Total: 5 * time.Minute,
Delay: 5 * time.Second,
}
for a := testAttempt.Start(); a.Next(); {
c.Logf("waiting for %v IPs on NIC %v", ips, nicId)
resp, err := s.ec2.NetworkInterfaces([]string{nicId}, nil)
if err != nil {
c.Logf("NetworkInterfaces returned: %v; retrying...", err)
continue
}
if len(resp.Interfaces) != 1 {
c.Logf("found %d NICs; retrying", len(resp.Interfaces))
continue
}
iface := resp.Interfaces[0]
if len(iface.PrivateIPs) != len(ips) {
c.Logf("addresses in %v: %v; still waiting", iface.Id, iface.PrivateIPs)
continue
}
var foundIPs []string
for i, ip := range iface.PrivateIPs {
if strings.HasSuffix(ips[i], ".*") {
c.Check(ip.Address, Matches, ips[i])
} else {
c.Check(ip.Address, Equals, ips[i])
}
if skipPrimary && ip.IsPrimary {
continue
}
foundIPs = append(foundIPs, ip.Address)
}
c.Logf("all addresses updated")
return foundIPs
}
c.Fatalf("timeout while waiting for the IPs to get updated")
return nil
}
func (S) TestAttemptTiming(c *C) {
testAttempt := aws.AttemptStrategy{
Total: 0.25e9,
Delay: 0.1e9,
}
want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
t0 := time.Now()
for a := testAttempt.Start(); a.Next(); {
got = append(got, time.Now().Sub(t0))
}
got = append(got, time.Now().Sub(t0))
c.Assert(got, HasLen, len(want))
const margin = 0.01e9
for i, got := range want {
lo := want[i] - margin
hi := want[i] + margin
if got < lo || got > hi {
c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
}
}
}
func terminateInstances(c *C, e *ec2.EC2, ids []string) {
_, err := e.TerminateInstances(ids)
c.Assert(err, IsNil, Commentf("%v INSTANCES LEFT RUNNING!!!", ids))
// We need to wait until the instances are really off, because
// entities that depend on them won't be deleted (i.e. groups,
// NICs, subnets, etc.)
testAttempt := aws.AttemptStrategy{
Total: 10 * time.Minute,
Delay: 5 * time.Second,
}
f := ec2.NewFilter()
f.Add("instance-state-name", "terminated")
idsLeft := make(map[string]bool)
for _, id := range ids {
idsLeft[id] = true
}
for a := testAttempt.Start(); a.Next(); {
c.Logf("waiting for %v to get terminated", ids)
resp, err := e.Instances(ids, f)
if err != nil {
c.Fatalf("not waiting for %v to terminate: %v", ids, err)
}
for _, r := range resp.Reservations {
for _, inst := range r.Instances {
delete(idsLeft, inst.InstanceId)
}
}
ids = []string{}
for id := range idsLeft {
ids = append(ids, id)
}
if len(ids) == 0 {
c.Logf("all instances terminated.")
return
}
}
c.Fatalf("%v INSTANCES LEFT RUNNING!!!", ids)
}
func (s *ServerTests) TestVPCs(c *C) {
resp1, err := s.ec2.CreateVPC("10.0.0.0/16", "")
c.Assert(err, IsNil)
assertVPC(c, resp1.VPC, "", "10.0.0.0/16")
id1 := resp1.VPC.Id
resp2, err := s.ec2.CreateVPC("10.1.0.0/16", "default")
c.Assert(err, IsNil)
assertVPC(c, resp2.VPC, "", "10.1.0.0/16")
id2 := resp2.VPC.Id
// We only check for the VPCs we just created, because the user
// might have others in his account (when testing against the EC2
// servers). In some cases it takes a short while until both VPCs
// are created, so we need to retry a few times to make sure.
var list *ec2.VPCsResp
done := false
testAttempt := aws.AttemptStrategy{
Total: 2 * time.Minute,
Delay: 5 * time.Second,
}
for a := testAttempt.Start(); a.Next(); {
c.Logf("waiting for %v to be created", []string{id1, id2})
list, err = s.ec2.VPCs(nil, nil)
if err != nil {
c.Logf("retrying; VPCs returned: %v", err)
continue
}
found := 0
for _, vpc := range list.VPCs {
c.Logf("found VPC %v", vpc)
switch vpc.Id {
case id1:
assertVPC(c, vpc, id1, resp1.VPC.CIDRBlock)
found++
case id2:
assertVPC(c, vpc, id2, resp2.VPC.CIDRBlock)
found++
}
if found == 2 {
done = true
break
}
}
if done {
c.Logf("all VPCs were created")
break
}
}
if !done {
c.Fatalf("timeout while waiting for VPCs %v", []string{id1, id2})
}
list, err = s.ec2.VPCs([]string{id1}, nil)
c.Assert(err, IsNil)
c.Assert(list.VPCs, HasLen, 1)
assertVPC(c, list.VPCs[0], id1, resp1.VPC.CIDRBlock)
f := ec2.NewFilter()
f.Add("cidr", resp2.VPC.CIDRBlock)
list, err = s.ec2.VPCs(nil, f)
c.Assert(err, IsNil)
c.Assert(list.VPCs, HasLen, 1)
assertVPC(c, list.VPCs[0], id2, resp2.VPC.CIDRBlock)
_, err = s.ec2.DeleteVPC(id1)
c.Assert(err, IsNil)
_, err = s.ec2.DeleteVPC(id2)
c.Assert(err, IsNil)
}
func (s *ServerTests) TestNetworkInterfaces(c *C) {
vpcResp, err := s.ec2.CreateVPC("10.3.0.0/16", "")
c.Assert(err, IsNil)
vpcId := vpcResp.VPC.Id
defer s.deleteVPCs(c, []string{vpcId})
subResp := s.createSubnet(c, vpcId, "10.3.1.0/24", "")
subId := subResp.Subnet.Id
defer s.deleteSubnets(c, []string{subId})
sg := s.makeTestGroupVPC(c, vpcId, "vpc-sg-1", "vpc test group1")
defer s.deleteGroups(c, []ec2.SecurityGroup{sg})
instList, err := s.ec2.RunInstances(&ec2.RunInstances{
ImageId: imageId,
InstanceType: "t1.micro",
SubnetId: subId,
})
c.Assert(err, IsNil)
inst := instList.Instances[0]
c.Assert(inst, NotNil)
instId := inst.InstanceId
defer terminateInstances(c, s.ec2, []string{instId})
ips1 := []ec2.PrivateIP{{Address: "10.3.1.10", IsPrimary: true}}
resp1, err := s.ec2.CreateNetworkInterface(ec2.CreateNetworkInterface{
SubnetId: subId,
PrivateIPs: ips1,
Description: "My first iface",
})
c.Assert(err, IsNil)
assertNetworkInterface(c, resp1.NetworkInterface, "", subId, ips1)
c.Check(resp1.NetworkInterface.Description, Equals, "My first iface")
id1 := resp1.NetworkInterface.Id
ips2 := []ec2.PrivateIP{
{Address: "10.3.1.20", IsPrimary: true},
{Address: "10.3.1.22", IsPrimary: false},
}
resp2, err := s.ec2.CreateNetworkInterface(ec2.CreateNetworkInterface{
SubnetId: subId,
PrivateIPs: ips2,
SecurityGroupIds: []string{sg.Id},
})
c.Assert(err, IsNil)
assertNetworkInterface(c, resp2.NetworkInterface, "", subId, ips2)
c.Assert(resp2.NetworkInterface.Groups, DeepEquals, []ec2.SecurityGroup{sg})
id2 := resp2.NetworkInterface.Id
// We only check for the network interfaces we just created,
// because the user might have others in his account (when testing
// against the EC2 servers). In some cases it takes a short while
// until both interfaces are created, so we need to retry a few
// times to make sure.
testAttempt := aws.AttemptStrategy{
Total: 5 * time.Minute,
Delay: 5 * time.Second,
}
var list *ec2.NetworkInterfacesResp
done := false
for a := testAttempt.Start(); a.Next(); {
c.Logf("waiting for %v to be created", []string{id1, id2})
list, err = s.ec2.NetworkInterfaces(nil, nil)
if err != nil {
c.Logf("retrying; NetworkInterfaces returned: %v", err)
continue
}
found := 0
for _, iface := range list.Interfaces {
c.Logf("found NIC %v", iface)
switch iface.Id {
case id1:
assertNetworkInterface(c, iface, id1, subId, ips1)
found++
case id2:
assertNetworkInterface(c, iface, id2, subId, ips2)
found++
}
if found == 2 {
done = true
break
}
}
if done {
c.Logf("all NICs were created")
break
}
}
if !done {
c.Fatalf("timeout while waiting for NICs %v", []string{id1, id2})
}
list, err = s.ec2.NetworkInterfaces([]string{id1}, nil)
c.Assert(err, IsNil)
c.Assert(list.Interfaces, HasLen, 1)
assertNetworkInterface(c, list.Interfaces[0], id1, subId, ips1)
f := ec2.NewFilter()
f.Add("network-interface-id", id2)
list, err = s.ec2.NetworkInterfaces(nil, f)
c.Assert(err, IsNil)
c.Assert(list.Interfaces, HasLen, 1)
assertNetworkInterface(c, list.Interfaces[0], id2, subId, ips2)
// Attachment might fail if the instance is not running yet,
// so we retry for a while until it succeeds.
var attResp *ec2.AttachNetworkInterfaceResp
for a := testAttempt.Start(); a.Next(); {
attResp, err = s.ec2.AttachNetworkInterface(id2, instId, 1)
if err != nil {
c.Logf("AttachNetworkInterface returned: %v; retrying...", err)
attResp = nil
continue
}
c.Logf("AttachNetworkInterface succeeded")
c.Check(attResp.AttachmentId, Not(Equals), "")
break
}
if attResp == nil {
c.Fatalf("timeout while waiting for AttachNetworkInterface to succeed")
}
list, err = s.ec2.NetworkInterfaces([]string{id2}, nil)
c.Assert(err, IsNil)
att := list.Interfaces[0].Attachment
c.Check(att.Id, Equals, attResp.AttachmentId)
c.Check(att.InstanceId, Equals, instId)
c.Check(att.DeviceIndex, Equals, 1)
c.Check(att.Status, Matches, "(attaching|in-use)")
_, err = s.ec2.DetachNetworkInterface(att.Id, true)
c.Check(err, IsNil)
_, err = s.ec2.DeleteNetworkInterface(id1)
c.Assert(err, IsNil)
// We might not be able to delete the interface until the
// detachment is completed, so we need to retry here as well.
for a := testAttempt.Start(); a.Next(); {
_, err = s.ec2.DeleteNetworkInterface(id2)
if err != nil {
c.Logf("DeleteNetworkInterface returned: %v; retrying...", err)
continue
}
c.Logf("DeleteNetworkInterface succeeded")
return
}
c.Fatalf("timeout while waiting for DeleteNetworkInterface to succeed")
}
func (s *ServerTests) TestAssignUnassignPrivateIPs(c *C) {
vpcResp, err := s.ec2.CreateVPC("10.7.0.0/16", "")
c.Assert(err, IsNil)
vpcId := vpcResp.VPC.Id
defer s.deleteVPCs(c, []string{vpcId})
subResp := s.createSubnet(c, vpcId, "10.7.1.0/24", "")
subId := subResp.Subnet.Id
defer s.deleteSubnets(c, []string{subId})
// Launch a m1.medium instance, so we can later assign up to 6
// private IPs per NIC.
instList, err := s.ec2.RunInstances(&ec2.RunInstances{
ImageId: imageId,
InstanceType: "m1.medium",
SubnetId: subId,
})
c.Assert(err, IsNil)
inst := instList.Instances[0]
c.Assert(inst, NotNil)
instId := inst.InstanceId
defer terminateInstances(c, s.ec2, []string{instId})
// We need to wait for the instance to change state to 'running',
// so its automatically created network interface on the created
// subnet will appear.
testAttempt := aws.AttemptStrategy{
Total: 5 * time.Minute,
Delay: 5 * time.Second,
}
var newNIC *ec2.NetworkInterface
f := ec2.NewFilter()
f.Add("subnet-id", subId)
for a := testAttempt.Start(); a.Next(); {
resp, err := s.ec2.NetworkInterfaces(nil, f)
if err != nil {
c.Logf("NetworkInterfaces returned: %v; retrying...", err)
continue
}
for _, iface := range resp.Interfaces {
c.Logf("found NIC %v", iface)
if iface.Attachment.InstanceId == instId {
c.Logf("found instance %v NIC", instId)
newNIC = &iface
break
}
}
if newNIC != nil {
break
}
}
if newNIC == nil {
c.Fatalf("timeout while waiting for the NIC to appear")
}
c.Check(newNIC.PrivateIPAddress, Matches, `^10\.7\.1\.\d+$`)
c.Check(newNIC.PrivateIPs, HasLen, 1)
// Now let's try assigning some more private IPs.
ips := []string{"10.7.1.25", "10.7.1.30"}
_, err = s.ec2.AssignPrivateIPAddresses(newNIC.Id, ips, 0, false)
c.Assert(err, IsNil)
expectIPs := append([]string{newNIC.PrivateIPAddress}, ips...)
s.waitForAddresses(c, newNIC.Id, expectIPs, false)
// Try using SecondaryPrivateIPCount.
_, err = s.ec2.AssignPrivateIPAddresses(newNIC.Id, nil, 2, false)
c.Assert(err, IsNil)
expectIPs = append(expectIPs, []string{"10.7.1.*", "10.7.1.*"}...)
ips = s.waitForAddresses(c, newNIC.Id, expectIPs, true)
// And finally, unassign them all, except the primary.
_, err = s.ec2.UnassignPrivateIPAddresses(newNIC.Id, ips)
c.Assert(err, IsNil)
expectIPs = []string{newNIC.PrivateIPAddress}
s.waitForAddresses(c, newNIC.Id, expectIPs, false)
}
func (s *AmazonServerSuite) TestRunInstancesVPC(c *C) {
vpcResp, err := s.ec2.CreateVPC("10.6.0.0/16", "")
c.Assert(err, IsNil)
vpcId := vpcResp.VPC.Id
defer s.deleteVPCs(c, []string{vpcId})
subResp := s.createSubnet(c, vpcId, "10.6.1.0/24", "")
subId := subResp.Subnet.Id
defer s.deleteSubnets(c, []string{subId})
groupResp, err := s.ec2.CreateSecurityGroupVPC(
vpcId,
sessionName("testgroup1 vpc"),
"testgroup description vpc",
)
c.Assert(err, IsNil)
group := groupResp.SecurityGroup
defer s.deleteGroups(c, []ec2.SecurityGroup{group})
// Run a single instance with a new network interface.
ips := []ec2.PrivateIP{
{Address: "10.6.1.10", IsPrimary: true},
{Address: "10.6.1.20", IsPrimary: false},
}
instResp, err := s.ec2.RunInstances(&ec2.RunInstances{
MinCount: 1,
ImageId: imageId,
InstanceType: "t1.micro",
NetworkInterfaces: []ec2.RunNetworkInterface{{
DeviceIndex: 0,
SubnetId: subId,
PrivateIPs: ips,
SecurityGroupIds: []string{group.Id},
DeleteOnTermination: true,
}},
})
c.Assert(err, IsNil)
inst := &instResp.Instances[0]
defer terminateInstances(c, s.ec2, []string{inst.InstanceId})
// Now list the network interfaces and find ours.
testAttempt := aws.AttemptStrategy{
Total: 5 * time.Minute,
Delay: 5 * time.Second,
}
f := ec2.NewFilter()
f.Add("subnet-id", subId)
var newNIC *ec2.NetworkInterface
for a := testAttempt.Start(); a.Next(); {
c.Logf("waiting for NIC to become available")
listNICs, err := s.ec2.NetworkInterfaces(nil, f)
if err != nil {
c.Logf("retrying; NetworkInterfaces returned: %v", err)
continue
}
for _, iface := range listNICs.Interfaces {
c.Logf("found NIC %v", iface)
if iface.Attachment.InstanceId == inst.InstanceId {
c.Logf("instance %v new NIC appeared", inst.InstanceId)
newNIC = &iface
break
}
}
if newNIC != nil {
break
}
}
if newNIC == nil {
c.Fatalf("timeout while waiting for NIC to appear.")
}
c.Check(newNIC.Id, Matches, `^eni-[0-9a-f]+$`)
c.Check(newNIC.SubnetId, Equals, subId)
c.Check(newNIC.VPCId, Equals, vpcId)
c.Check(newNIC.Status, Matches, `^(attaching|in-use)$`)
c.Check(newNIC.PrivateIPAddress, Equals, ips[0].Address)
c.Check(newNIC.PrivateIPs, DeepEquals, ips)
c.Check(newNIC.Groups, HasLen, 1)
c.Check(newNIC.Groups[0].Id, Equals, group.Id)
c.Check(newNIC.Attachment.Status, Matches, `^(attaching|attached)$`)
c.Check(newNIC.Attachment.DeviceIndex, Equals, 0)
c.Check(newNIC.Attachment.DeleteOnTermination, Equals, true)
}
func (S) TestAttemptNextHasNext(c *C) {
a := aws.AttemptStrategy{}.Start()
c.Assert(a.Next(), Equals, true)
c.Assert(a.Next(), Equals, false)
a = aws.AttemptStrategy{}.Start()
c.Assert(a.Next(), Equals, true)
c.Assert(a.HasNext(), Equals, false)
c.Assert(a.Next(), Equals, false)
a = aws.AttemptStrategy{Total: 2e8}.Start()
c.Assert(a.Next(), Equals, true)
c.Assert(a.HasNext(), Equals, true)
time.Sleep(2e8)
c.Assert(a.HasNext(), Equals, true)
c.Assert(a.Next(), Equals, true)
c.Assert(a.Next(), Equals, false)
a = aws.AttemptStrategy{Total: 1e8, Min: 2}.Start()
time.Sleep(1e8)
c.Assert(a.Next(), Equals, true)
c.Assert(a.HasNext(), Equals, true)
c.Assert(a.Next(), Equals, true)
c.Assert(a.HasNext(), Equals, false)
c.Assert(a.Next(), Equals, false)
}