func TestSyncLastUpdated(t *testing.T) {
as := assert.New(t)
s1 := &DummySink{}
c := cache.NewCache(time.Hour, time.Minute)
m, err := newExternalSinkManager([]sink_api.ExternalSink{s1}, c, time.Microsecond)
as.Nil(err)
var (
pods []source_api.Pod
containers []source_api.Container
events []*cache.Event
expectedESync, expectedPSync, expectedNSync time.Time
)
f := fuzz.New().NumElements(10, 10).NilChance(0)
f.Fuzz(&pods)
now := time.Now()
for pidx := range pods {
for cidx := range pods[pidx].Containers {
for sidx := range pods[pidx].Containers[cidx].Stats {
ts := now.Add(time.Duration(sidx) * time.Minute)
pods[pidx].Containers[cidx].Stats[sidx].Timestamp = ts
expectedPSync = hUtil.GetLatest(expectedPSync, ts)
}
}
}
f.Fuzz(&containers)
for cidx := range containers {
for sidx := range containers[cidx].Stats {
ts := now.Add(time.Duration(sidx) * time.Minute)
containers[cidx].Stats[sidx].Timestamp = ts
expectedNSync = hUtil.GetLatest(expectedNSync, ts)
}
}
f.Fuzz(&events)
for eidx := range events {
ts := now.Add(time.Duration(eidx) * time.Minute)
events[eidx].LastUpdate = ts
events[eidx].UID = fmt.Sprintf("id:%d", eidx)
expectedESync = hUtil.GetLatest(expectedESync, ts)
}
err = c.StorePods(pods)
if err != nil {
glog.Fatalf("Failed to store pods %v", err)
}
err = c.StoreContainers(containers)
if err != nil {
glog.Fatalf("Failed to store containers %v", err)
}
err = c.StoreEvents(events)
if err != nil {
glog.Fatalf("Failed to store events %v", err)
}
m.store()
as.Equal(m.lastSync.eventsSync, expectedESync, "Event now: %v, eventSync: %v, expected: %v", now, m.lastSync.eventsSync, expectedESync)
as.Equal(m.lastSync.podSync, expectedPSync, "Pod now: %v, podSync: %v, expected: %v", now, m.lastSync.podSync, expectedPSync)
as.Equal(m.lastSync.nodeSync, expectedNSync, "Node now: %v, nodeSync: %v, expected: %v", now, m.lastSync.nodeSync, expectedNSync)
}
func storeSpecAndStats(ce *containerElement, c *source_api.Container) time.Time {
if ce == nil || c == nil {
return time.Time{}
}
latestTimestamp := time.Time{}
for idx := range c.Stats {
if c.Stats[idx] == nil {
continue
}
cme := &ContainerMetricElement{
Spec: &c.Spec,
Stats: c.Stats[idx],
}
ce.metrics.Put(store.TimePoint{
Timestamp: c.Stats[idx].Timestamp,
Value: cme,
})
latestTimestamp = hUtil.GetLatest(latestTimestamp, c.Stats[idx].Timestamp)
}
return latestTimestamp
}
func (rc *realCache) StorePods(pods []source_api.Pod) error {
rc.Lock()
defer rc.Unlock()
now := time.Now()
for _, pod := range pods {
pe, ok := rc.pods[pod.ID]
if !ok {
pe = rc.newPodElement()
pe.Metadata = Metadata{
Name: pod.Name,
Namespace: pod.Namespace,
NamespaceUID: pod.NamespaceUID,
UID: pod.ID,
Hostname: pod.Hostname,
Labels: pod.Labels,
ExternalID: pod.ExternalID,
}
rc.pods[pod.ID] = pe
}
for idx := range pod.Containers {
cont := &pod.Containers[idx]
ce, ok := pe.containers[cont.Name]
if !ok {
ce = rc.newContainerElement()
pe.containers[cont.Name] = ce
}
ce.Metadata = Metadata{
Name: cont.Name,
Hostname: cont.Hostname,
Labels: cont.Spec.Labels,
}
ce.Image = cont.Image
ce.Metadata.LastUpdate = storeSpecAndStats(ce, cont)
ce.lastUpdated = now
pe.LastUpdate = hUtil.GetLatest(pe.LastUpdate, ce.Metadata.LastUpdate)
}
pe.lastUpdated = now
}
return nil
}
// TODO(vmarmol): Paralellize this.
func (esm *externalSinkManager) store() error {
pods := esm.cache.GetPods(esm.lastSync.podSync, zeroTime)
for _, pod := range pods {
esm.lastSync.podSync = hUtil.GetLatest(esm.lastSync.podSync, pod.LastUpdate)
}
containers := esm.cache.GetNodes(esm.lastSync.nodeSync, zeroTime)
containers = append(containers, esm.cache.GetFreeContainers(esm.lastSync.nodeSync, zeroTime)...)
for _, c := range containers {
esm.lastSync.nodeSync = hUtil.GetLatest(esm.lastSync.nodeSync, c.LastUpdate)
}
// TODO: Store data in cache.
timeseries, err := esm.decoder.TimeseriesFromPods(pods)
if err != nil {
return err
}
containerTimeseries, err := esm.decoder.TimeseriesFromContainers(containers)
if err != nil {
return err
}
timeseries = append(timeseries, containerTimeseries...)
if len(timeseries) == 0 {
glog.V(3).Info("no timeseries data between %v and %v", esm.lastSync.nodeSync, zeroTime)
// Continue here to push events data.
}
events := esm.cache.GetEvents(esm.lastSync.eventsSync, zeroTime)
var kEvents []kube_api.Event
for _, event := range events {
kEvents = append(kEvents, event.Raw)
esm.lastSync.eventsSync = hUtil.GetLatest(esm.lastSync.eventsSync, event.LastUpdate)
}
if len(timeseries) == 0 && len(events) == 0 {
glog.V(5).Infof("Skipping sync loop")
return nil
}
// Format metrics and push them.
esm.RLock()
defer esm.RUnlock()
errorsLen := 2 * len(esm.externalSinks)
errorsChan := make(chan error, errorsLen)
for idx := range esm.externalSinks {
sink := esm.externalSinks[idx]
go func(sink sink_api.ExternalSink) {
glog.V(2).Infof("Storing Timeseries to %q", sink.Name())
errorsChan <- sink.StoreTimeseries(timeseries)
}(sink)
go func(sink sink_api.ExternalSink) {
glog.V(2).Infof("Storing Events to %q", sink.Name())
errorsChan <- sink.StoreEvents(kEvents)
}(sink)
}
var errors []string
for i := 0; i < errorsLen; i++ {
if err := <-errorsChan; err != nil {
errors = append(errors, fmt.Sprintf("%v ", err))
}
}
if len(errors) > 0 {
return fmt.Errorf("encountered the following errors: %s", strings.Join(errors, ";\n"))
}
return nil
}
