func NewManager(sources []source_api.Source, sinkManager sinks.ExternalSinkManager, res, bufferDuration time.Duration, useModel bool, modelRes time.Duration, align bool) (Manager, error) {
// TimeStore constructor passed to the cluster implementation.
tsConstructor := func() store.TimeStore {
// TODO(afein): determine default analogy of cache duration to Timestore durations.
return store.NewGCStore(store.NewCMAStore(), 5*bufferDuration)
}
var newCluster model.Cluster = nil
if useModel {
newCluster = model.NewCluster(tsConstructor, modelRes)
}
firstSync := time.Now()
if align {
firstSync = firstSync.Truncate(res).Add(res)
}
return &realManager{
sources: sources,
sinkManager: sinkManager,
cache: cache.NewCache(bufferDuration),
model: newCluster,
lastSync: firstSync,
resolution: res,
align: align,
decoder: sink_api.NewDecoder(),
}, nil
}
func doWork() ([]source_api.Source, sinks.ExternalSinkManager, manager.Manager, error) {
c := cache.NewCache(*argCacheDuration, time.Minute)
sources, err := newSources(c)
if err != nil {
return nil, nil, nil, err
}
sinkManager, err := sinks.NewExternalSinkManager(nil)
if err != nil {
return nil, nil, nil, err
}
manager, err := manager.NewManager(sources, sinkManager, *argStatsResolution, *argCacheDuration, c, *argUseModel, *argModelResolution, *argAlignStats)
if err != nil {
return nil, nil, nil, err
}
if err := manager.SetSinkUris(argSinks); err != nil {
return nil, nil, nil, err
}
// Spawn the Model Housekeeping goroutine even if the model is not enabled.
// This will allow the model to be activated/deactivated in runtime.
modelDuration := 2 * *argModelResolution
if (*argCacheDuration).Nanoseconds() < modelDuration.Nanoseconds() {
modelDuration = *argCacheDuration
}
go util.Until(manager.HousekeepModel, modelDuration, util.NeverStop)
go util.Until(manager.Housekeep, *argPollDuration, util.NeverStop)
return sources, sinkManager, manager, nil
}
func NewManager(sources []source_api.Source, sinkManager sinks.ExternalSinkManager, res, bufferDuration time.Duration) (Manager, error) {
return &realManager{
sources: sources,
sinkManager: sinkManager,
cache: cache.NewCache(bufferDuration),
lastSync: time.Now(),
resolution: res,
decoder: sink_api.NewDecoder(),
}, nil
}
// TestUpdate tests the normal flows of Update.
// TestUpdate performs consecutive calls to Update with both empty and non-empty caches
func TestUpdate(t *testing.T) {
var (
cluster = newRealCluster(newTimeStore, time.Minute)
source_cache = cacheFactory()
assert = assert.New(t)
empty_cache = cache.NewCache(24*time.Hour, time.Hour)
zeroTime = time.Time{}
)
// Invocation with empty cache
assert.NoError(cluster.Update(empty_cache))
assert.Empty(cluster.Nodes)
assert.Empty(cluster.Namespaces)
assert.Empty(cluster.Metrics)
// Invocation with regular parameters
assert.NoError(cluster.Update(source_cache))
verifyCacheFactoryCluster(&cluster.ClusterInfo, t)
// Assert Node Metric aggregation
assert.NotEmpty(cluster.Nodes)
assert.NotEmpty(cluster.Metrics)
assert.NotNil(cluster.Metrics[memWorking])
mem_work_ts := *(cluster.Metrics[memWorking])
actual := mem_work_ts.Get(zeroTime, zeroTime)
assert.Len(actual, 2)
// Datapoint present in both nodes, added up to 1024
assert.Equal(actual[1].Value.(uint64), uint64(1204))
// Datapoint present in only one node
assert.Equal(actual[0].Value.(uint64), uint64(602))
assert.NotNil(cluster.Metrics[memUsage])
mem_usage_ts := *(cluster.Metrics[memUsage])
actual = mem_usage_ts.Get(zeroTime, zeroTime)
assert.Len(actual, 2)
// Datapoint present in both nodes, added up to 10000
assert.Equal(actual[1].Value.(uint64), uint64(10000))
// Datapoint present in only one node
assert.Equal(actual[0].Value.(uint64), uint64(5000))
// Assert Kubernetes Metric aggregation up to namespaces
ns := cluster.Namespaces["test"]
mem_work_ts = *(ns.Metrics[memWorking])
actual = mem_work_ts.Get(zeroTime, zeroTime)
assert.Len(actual, 1)
assert.Equal(actual[0].Value.(uint64), uint64(2408))
// Invocation with no fresh data - expect no change in cluster
assert.NoError(cluster.Update(source_cache))
verifyCacheFactoryCluster(&cluster.ClusterInfo, t)
// Invocation with empty cache - expect no change in cluster
assert.NoError(cluster.Update(empty_cache))
verifyCacheFactoryCluster(&cluster.ClusterInfo, t)
}
func TestEventsBasic(t *testing.T) {
handler := util.FakeHandler{
StatusCode: 200,
RequestBody: "something",
ResponseBody: body(&kube_api.EventList{}),
T: t,
}
server := httptest.NewServer(&handler)
defer server.Close()
client := client.NewOrDie(&client.Config{Host: server.URL, Version: testapi.Version()})
cache := cache.NewCache(time.Hour, time.Hour)
source := NewKubeEvents(client, cache)
_, err := source.GetInfo(time.Now(), time.Now().Add(time.Minute), time.Second, false)
require.NoError(t, err)
require.NotEmpty(t, source.DebugInfo())
}
// cacheFactory generates a cache with a predetermined structure.
// The cache contains two pods, one with two containers and one without any containers.
// The cache also contains a free container and a "machine"-tagged container.
func cacheFactory() cache.Cache {
source_cache := cache.NewCache(24*time.Hour, time.Hour)
// Generate Container CMEs - same timestamp for aggregation
cme_1 := cmeFactory()
cme_2 := cmeFactory()
cme_2.Stats.Timestamp = cme_1.Stats.Timestamp
// Genete Machine CMEs - same timestamp for aggregation
cme_3 := cmeFactory()
cme_4 := cmeFactory()
cme_4.Stats.Timestamp = cme_3.Stats.Timestamp
cme_5 := cmeFactory()
cme_5.Stats.Timestamp = cme_4.Stats.Timestamp.Add(time.Hour)
cme_5.Stats.Cpu.Usage.Total = cme_4.Stats.Cpu.Usage.Total + uint64(3600000000000)
// Generate a pod with two containers, and a pod without any containers
container1 := source_api.Container{
Name: "container1",
Hostname: "hostname2",
Spec: *cme_1.Spec,
Stats: []*cadvisor.ContainerStats{cme_1.Stats},
}
container2 := source_api.Container{
Name: "container2",
Hostname: "hostname3",
Spec: *cme_2.Spec,
Stats: []*cadvisor.ContainerStats{cme_2.Stats},
}
containers := []source_api.Container{container1, container2}
pods := []source_api.Pod{
{
PodMetadata: source_api.PodMetadata{
Name: "pod1",
ID: "123",
Namespace: "test",
Hostname: "hostname2",
Status: "Running",
},
Containers: containers,
},
{
PodMetadata: source_api.PodMetadata{
Name: "pod2",
ID: "1234",
Namespace: "test",
Hostname: "hostname3",
Status: "Running",
},
Containers: containers,
},
}
// Generate two machine containers
machine_container := source_api.Container{
Name: "/",
Hostname: "hostname2",
Spec: *cme_3.Spec,
Stats: []*cadvisor.ContainerStats{cme_3.Stats},
}
machine_container2 := source_api.Container{
Name: "/",
Hostname: "hostname3",
Spec: *cme_4.Spec,
Stats: []*cadvisor.ContainerStats{cme_5.Stats, cme_4.Stats},
}
// Generate a free container
free_container := source_api.Container{
Name: "free_container1",
Hostname: "hostname2",
Spec: *cme_5.Spec,
Stats: []*cadvisor.ContainerStats{cme_5.Stats},
}
other_containers := []source_api.Container{
machine_container,
machine_container2,
free_container,
}
// Enter everything in the cache
source_cache.StorePods(pods)
source_cache.StoreContainers(other_containers)
return source_cache
}
// cacheFactory generates a cache with a predetermined structure.
// The cache contains two pods, one with two containers and one without any containers.
// The cache also contains a free container and a "machine"-tagged container.
func cacheFactory() cache.Cache {
source_cache := cache.NewCache(time.Hour)
// Generate 4 ContainerMetricElements
cme_1 := cmeFactory()
cme_2 := cmeFactory()
cme_3 := cmeFactory()
cme_4 := cmeFactory()
// Generate a pod with two containers, and a pod without any containers
container1 := source_api.Container{
Name: "container1",
Hostname: "hostname2",
Spec: *cme_1.Spec,
Stats: []*cadvisor.ContainerStats{cme_1.Stats},
}
container2 := source_api.Container{
Name: "container2",
Hostname: "hostname3",
Spec: *cme_2.Spec,
Stats: []*cadvisor.ContainerStats{cme_2.Stats},
}
containers := []source_api.Container{container1, container2}
pods := []source_api.Pod{
{
PodMetadata: source_api.PodMetadata{
Name: "pod1",
ID: "123",
Namespace: "test",
Hostname: "hostname2",
Status: "Running",
},
Containers: containers,
},
{
PodMetadata: source_api.PodMetadata{
Name: "pod2",
ID: "1234",
Namespace: "test",
Hostname: "hostname3",
Status: "Running",
},
Containers: []source_api.Container{},
},
}
// Generate a machine container
machine_container := source_api.Container{
Name: "/",
Hostname: "hostname2",
Spec: *cme_3.Spec,
Stats: []*cadvisor.ContainerStats{cme_3.Stats},
}
// Generate a free container
free_container := source_api.Container{
Name: "free_container1",
Hostname: "hostname2",
Spec: *cme_4.Spec,
Stats: []*cadvisor.ContainerStats{cme_4.Stats},
}
other_containers := []source_api.Container{machine_container, free_container}
// Enter everything in the cache
source_cache.StorePods(pods)
source_cache.StoreContainers(other_containers)
return source_cache
}
