func TestContainerStatsCollectionReconnection(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockDockerClient := ecsengine.NewMockDockerClient(ctrl)
dockerID := "container1"
ctx, cancel := context.WithCancel(context.TODO())
statChan := make(chan *docker.Stats)
statErr := fmt.Errorf("test error")
closedChan := make(chan *docker.Stats)
close(closedChan)
gomock.InOrder(
mockDockerClient.EXPECT().Stats(dockerID, ctx).Return(nil, statErr),
mockDockerClient.EXPECT().Stats(dockerID, ctx).Return(closedChan, nil),
mockDockerClient.EXPECT().Stats(dockerID, ctx).Return(statChan, nil),
)
container := &StatsContainer{
containerMetadata: &ContainerMetadata{
DockerID: dockerID,
},
ctx: ctx,
cancel: cancel,
client: mockDockerClient,
}
container.StartStatsCollection()
time.Sleep(checkPointSleep)
container.StopStatsCollection()
}
func TestStatsEngineClientErrorListingContainers(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
engine := NewDockerStatsEngine(&cfg)
mockDockerClient := ecsengine.NewMockDockerClient(mockCtrl)
// Mock client will return error while listing images.
mockDockerClient.EXPECT().ListContainers(false).Return(ecsengine.ListContainersResponse{DockerIds: nil, Error: fmt.Errorf("could not list containers")})
engine.client = mockDockerClient
mockChannel := make(chan ecsengine.DockerContainerChangeEvent)
mockDockerClient.EXPECT().ContainerEvents(gomock.Any()).Return(mockChannel, nil)
engine.client = mockDockerClient
engine.Init()
time.Sleep(waitForCleanupSleep)
// Make sure that the stats engine deregisters the event listener when it fails to
// list images.
if engine.ctx.Err() != context.Canceled {
t.Error("Engine context hasn't been canceled")
}
}
func TestContainerStatsCollectionStopsIfContainerIsTerminal(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockDockerClient := ecsengine.NewMockDockerClient(ctrl)
resolver := mock_resolver.NewMockContainerMetadataResolver(ctrl)
dockerID := "container1"
ctx, cancel := context.WithCancel(context.TODO())
closedChan := make(chan *docker.Stats)
close(closedChan)
statsErr := fmt.Errorf("test error")
mockContainer := &api.DockerContainer{
DockerId: dockerID,
Container: &api.Container{
KnownStatus: api.ContainerStopped,
},
}
gomock.InOrder(
mockDockerClient.EXPECT().Stats(dockerID, ctx).Return(closedChan, nil),
resolver.EXPECT().ResolveContainer(dockerID).Return(mockContainer, statsErr),
)
container := &StatsContainer{
containerMetadata: &ContainerMetadata{
DockerID: dockerID,
},
ctx: ctx,
cancel: cancel,
client: mockDockerClient,
resolver: resolver,
}
container.StartStatsCollection()
select {
case <-ctx.Done():
}
}
func TestStatsEngineAddRemoveContainers(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
resolver := mock_resolver.NewMockContainerMetadataResolver(ctrl)
mockDockerClient := ecsengine.NewMockDockerClient(ctrl)
t1 := &api.Task{Arn: "t1", Family: "f1"}
t2 := &api.Task{Arn: "t2", Family: "f2"}
t3 := &api.Task{Arn: "t3"}
resolver.EXPECT().ResolveTask("c1").AnyTimes().Return(t1, nil)
resolver.EXPECT().ResolveTask("c2").AnyTimes().Return(t1, nil)
resolver.EXPECT().ResolveTask("c3").AnyTimes().Return(t2, nil)
resolver.EXPECT().ResolveTask("c4").AnyTimes().Return(nil, fmt.Errorf("unmapped container"))
resolver.EXPECT().ResolveTask("c5").AnyTimes().Return(t2, nil)
resolver.EXPECT().ResolveTask("c6").AnyTimes().Return(t3, nil)
resolver.EXPECT().ResolveContainer(gomock.Any()).AnyTimes().Return(&api.DockerContainer{
Container: &api.Container{},
}, nil)
mockStatsChannel := make(chan *docker.Stats)
defer close(mockStatsChannel)
mockDockerClient.EXPECT().Stats(gomock.Any(), gomock.Any()).Return(mockStatsChannel, nil).AnyTimes()
engine := NewDockerStatsEngine(&cfg, nil, eventStream("TestStatsEngineAddRemoveContainers"))
engine.resolver = resolver
engine.client = mockDockerClient
engine.cluster = defaultCluster
engine.containerInstanceArn = defaultContainerInstance
defer engine.removeAll()
engine.addContainer("c1")
engine.addContainer("c1")
if len(engine.tasksToContainers) != 1 {
t.Errorf("Adding containers failed. Expected num tasks = 1, got: %d", len(engine.tasksToContainers))
}
containers, _ := engine.tasksToContainers["t1"]
if len(containers) != 1 {
t.Error("Adding duplicate containers failed.")
}
_, exists := containers["c1"]
if !exists {
t.Error("Container c1 not found in engine")
}
engine.addContainer("c2")
containers, _ = engine.tasksToContainers["t1"]
_, exists = containers["c2"]
if !exists {
t.Error("Container c2 not found in engine")
}
for _, statsContainer := range containers {
for _, fakeContainerStats := range createFakeContainerStats() {
statsContainer.statsQueue.Add(fakeContainerStats)
}
}
// Ensure task shows up in metrics.
containerMetrics, err := engine.getContainerMetricsForTask("t1")
if err != nil {
t.Errorf("Error getting container metrics: %v", err)
}
err = validateContainerMetrics(containerMetrics, 2)
if err != nil {
t.Errorf("Error validating container metrics: %v", err)
}
metadata, taskMetrics, err := engine.GetInstanceMetrics()
if err != nil {
t.Errorf("Error gettting instance metrics: %v", err)
}
err = validateMetricsMetadata(metadata)
if err != nil {
t.Errorf("Error validating metadata: %v", err)
}
if len(taskMetrics) != 1 {
t.Errorf("Incorrect number of tasks. Expected: 1, got: %d", len(taskMetrics))
}
err = validateContainerMetrics(taskMetrics[0].ContainerMetrics, 2)
if err != nil {
t.Errorf("Error validating container metrics: %v", err)
}
if *taskMetrics[0].TaskArn != "t1" {
t.Errorf("Incorrect task arn. Expected: t1, got: %s", *taskMetrics[0].TaskArn)
}
// Ensure that only valid task shows up in metrics.
_, err = engine.getContainerMetricsForTask("t2")
if err == nil {
t.Error("Expected non-empty error for non existent task")
}
engine.removeContainer("c1")
containers, _ = engine.tasksToContainers["t1"]
_, exists = containers["c1"]
if exists {
t.Error("Container c1 not removed from engine")
}
engine.removeContainer("c2")
containers, _ = engine.tasksToContainers["t1"]
_, exists = containers["c2"]
if exists {
t.Error("Container c2 not removed from engine")
}
engine.addContainer("c3")
containers, _ = engine.tasksToContainers["t2"]
_, exists = containers["c3"]
if !exists {
t.Error("Container c3 not found in engine")
}
_, _, err = engine.GetInstanceMetrics()
if err == nil {
t.Error("Expected non-empty error for empty stats.")
}
engine.removeContainer("c3")
// Should get an error while adding this container due to unmapped
// container to task.
engine.addContainer("c4")
err = validateIdleContainerMetrics(engine)
if err != nil {
t.Fatalf("Error validating metadata: %v", err)
}
// Should get an error while adding this container due to unmapped
// task arn to task definition family.
engine.addContainer("c6")
err = validateIdleContainerMetrics(engine)
if err != nil {
t.Fatalf("Error validating metadata: %v", err)
}
}
func TestContainerStatsCollection(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockDockerClient := ecsengine.NewMockDockerClient(ctrl)
dockerID := "container1"
ctx, cancel := context.WithCancel(context.TODO())
statChan := make(chan *docker.Stats)
mockDockerClient.EXPECT().Stats(dockerID, ctx).Return(statChan, nil)
go func() {
for _, stat := range statsData {
// doing this with json makes me sad, but is the easiest way to
// deal with the docker.Stats.MemoryStats inner struct
jsonStat := fmt.Sprintf(`
{
"memory_stats": {"usage":%d},
"cpu_stats":{
"cpu_usage":{
"percpu_usage":[%d],
"total_usage":%d
}
}
}`, stat.memBytes, stat.cpuTime, stat.cpuTime)
dockerStat := &docker.Stats{}
json.Unmarshal([]byte(jsonStat), dockerStat)
dockerStat.Read = stat.timestamp
statChan <- dockerStat
}
}()
container := &StatsContainer{
containerMetadata: &ContainerMetadata{
DockerID: dockerID,
},
ctx: ctx,
cancel: cancel,
client: mockDockerClient,
}
container.StartStatsCollection()
time.Sleep(checkPointSleep)
container.StopStatsCollection()
cpuStatsSet, err := container.statsQueue.GetCPUStatsSet()
if err != nil {
t.Fatal("Error gettting cpu stats set:", err)
}
if *cpuStatsSet.Min == math.MaxFloat64 || math.IsNaN(*cpuStatsSet.Min) {
t.Error("Min value incorrectly set: ", *cpuStatsSet.Min)
}
if *cpuStatsSet.Max == -math.MaxFloat64 || math.IsNaN(*cpuStatsSet.Max) {
t.Error("Max value incorrectly set: ", *cpuStatsSet.Max)
}
if *cpuStatsSet.SampleCount == 0 {
t.Error("Samplecount is 0")
}
if *cpuStatsSet.Sum == 0 {
t.Error("Sum value incorrectly set: ", *cpuStatsSet.Sum)
}
memStatsSet, err := container.statsQueue.GetMemoryStatsSet()
if err != nil {
t.Error("Error gettting cpu stats set:", err)
}
if *memStatsSet.Min == math.MaxFloat64 {
t.Error("Min value incorrectly set: ", *memStatsSet.Min)
}
if *memStatsSet.Max == 0 {
t.Error("Max value incorrectly set: ", *memStatsSet.Max)
}
if *memStatsSet.SampleCount == 0 {
t.Error("Samplecount is 0")
}
if *memStatsSet.Sum == 0 {
t.Error("Sum value incorrectly set: ", *memStatsSet.Sum)
}
}