// EventAggregate identifies similar events and groups into a common event if required func (e *EventAggregator) EventAggregate(newEvent *v1.Event) (*v1.Event, error) { aggregateKey, localKey := e.keyFunc(newEvent) now := unversioned.NewTime(e.clock.Now()) record := aggregateRecord{localKeys: sets.NewString(), lastTimestamp: now} e.Lock() defer e.Unlock() value, found := e.cache.Get(aggregateKey) if found { record = value.(aggregateRecord) } // if the last event was far enough in the past, it is not aggregated, and we must reset state maxInterval := time.Duration(e.maxIntervalInSeconds) * time.Second interval := now.Time.Sub(record.lastTimestamp.Time) if interval > maxInterval { record = aggregateRecord{localKeys: sets.NewString()} } record.localKeys.Insert(localKey) record.lastTimestamp = now e.cache.Add(aggregateKey, record) if record.localKeys.Len() < e.maxEvents { return newEvent, nil } // do not grow our local key set any larger than max record.localKeys.PopAny() // create a new aggregate event eventCopy := &v1.Event{ ObjectMeta: v1.ObjectMeta{ Name: fmt.Sprintf("%v.%x", newEvent.InvolvedObject.Name, now.UnixNano()), Namespace: newEvent.Namespace, }, Count: 1, FirstTimestamp: now, InvolvedObject: newEvent.InvolvedObject, LastTimestamp: now, Message: e.messageFunc(newEvent), Type: newEvent.Type, Reason: newEvent.Reason, Source: newEvent.Source, } return eventCopy, nil }
// eventObserve records the event, and determines if its frequency should update func (e *eventLogger) eventObserve(newEvent *v1.Event) (*v1.Event, []byte, error) { var ( patch []byte err error ) key := getEventKey(newEvent) eventCopy := *newEvent event := &eventCopy e.Lock() defer e.Unlock() lastObservation := e.lastEventObservationFromCache(key) // we have seen this event before, so we must prepare a patch if lastObservation.count > 0 { // update the event based on the last observation so patch will work as desired event.Name = lastObservation.name event.ResourceVersion = lastObservation.resourceVersion event.FirstTimestamp = lastObservation.firstTimestamp event.Count = int32(lastObservation.count) + 1 eventCopy2 := *event eventCopy2.Count = 0 eventCopy2.LastTimestamp = unversioned.NewTime(time.Unix(0, 0)) newData, _ := json.Marshal(event) oldData, _ := json.Marshal(eventCopy2) patch, err = strategicpatch.CreateStrategicMergePatch(oldData, newData, event) } // record our new observation e.cache.Add( key, eventLog{ count: int(event.Count), firstTimestamp: event.FirstTimestamp, name: event.Name, resourceVersion: event.ResourceVersion, }, ) return event, patch, err }
// TestEventCorrelator validates proper counting, aggregation of events func TestEventCorrelator(t *testing.T) { firstEvent := makeEvent("first", "i am first", makeObjectReference("Pod", "my-pod", "my-ns")) duplicateEvent := makeEvent("duplicate", "me again", makeObjectReference("Pod", "my-pod", "my-ns")) uniqueEvent := makeEvent("unique", "snowflake", makeObjectReference("Pod", "my-pod", "my-ns")) similarEvent := makeEvent("similar", "similar message", makeObjectReference("Pod", "my-pod", "my-ns")) aggregateEvent := makeEvent(similarEvent.Reason, EventAggregatorByReasonMessageFunc(&similarEvent), similarEvent.InvolvedObject) scenario := map[string]struct { previousEvents []v1.Event newEvent v1.Event expectedEvent v1.Event intervalSeconds int }{ "create-a-single-event": { previousEvents: []v1.Event{}, newEvent: firstEvent, expectedEvent: setCount(firstEvent, 1), intervalSeconds: 5, }, "the-same-event-should-just-count": { previousEvents: makeEvents(1, duplicateEvent), newEvent: duplicateEvent, expectedEvent: setCount(duplicateEvent, 2), intervalSeconds: 5, }, "the-same-event-should-just-count-even-if-more-than-aggregate": { previousEvents: makeEvents(defaultAggregateMaxEvents, duplicateEvent), newEvent: duplicateEvent, expectedEvent: setCount(duplicateEvent, defaultAggregateMaxEvents+1), intervalSeconds: 5, }, "create-many-unique-events": { previousEvents: makeUniqueEvents(30), newEvent: uniqueEvent, expectedEvent: setCount(uniqueEvent, 1), intervalSeconds: 5, }, "similar-events-should-aggregate-event": { previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message), newEvent: similarEvent, expectedEvent: setCount(aggregateEvent, 1), intervalSeconds: 5, }, "similar-events-many-times-should-count-the-aggregate": { previousEvents: makeSimilarEvents(defaultAggregateMaxEvents, similarEvent, similarEvent.Message), newEvent: similarEvent, expectedEvent: setCount(aggregateEvent, 2), intervalSeconds: 5, }, "similar-events-whose-interval-is-greater-than-aggregate-interval-do-not-aggregate": { previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message), newEvent: similarEvent, expectedEvent: setCount(similarEvent, 1), intervalSeconds: defaultAggregateIntervalInSeconds, }, } for testScenario, testInput := range scenario { eventInterval := time.Duration(testInput.intervalSeconds) * time.Second clock := clock.IntervalClock{Time: time.Now(), Duration: eventInterval} correlator := NewEventCorrelator(&clock) for i := range testInput.previousEvents { event := testInput.previousEvents[i] now := unversioned.NewTime(clock.Now()) event.FirstTimestamp = now event.LastTimestamp = now result, err := correlator.EventCorrelate(&event) if err != nil { t.Errorf("scenario %v: unexpected error playing back prevEvents %v", testScenario, err) } correlator.UpdateState(result.Event) } // update the input to current clock value now := unversioned.NewTime(clock.Now()) testInput.newEvent.FirstTimestamp = now testInput.newEvent.LastTimestamp = now result, err := correlator.EventCorrelate(&testInput.newEvent) if err != nil { t.Errorf("scenario %v: unexpected error correlating input event %v", testScenario, err) } _, err = validateEvent(testScenario, result.Event, &testInput.expectedEvent, t) if err != nil { t.Errorf("scenario %v: unexpected error validating result %v", testScenario, err) } } }