func eventIngestBulkMeta(
index outil.Selector,
pipelineSel *outil.Selector,
data outputs.Data,
) interface{} {
type bulkMetaIndex struct {
Index string `json:"_index"`
DocType string `json:"_type"`
Pipeline string `json:"pipeline"`
}
type bulkMeta struct {
Index bulkMetaIndex `json:"index"`
}
event := data.Event
pipeline, _ := pipelineSel.Select(event)
if pipeline == "" {
return eventBulkMeta(index, nil, data)
}
return bulkMeta{
Index: bulkMetaIndex{
Index: getIndex(event, index),
Pipeline: pipeline,
DocType: event["type"].(string),
},
}
}
func publishEventsBulk(conn redis.Conn, key outil.Selector, command string) publishFn {
// XXX: requires key.IsConst() == true
dest, _ := key.Select(common.MapStr{})
return func(_ outil.Selector, data []outputs.Data) ([]outputs.Data, error) {
args := make([]interface{}, 1, len(data)+1)
args[0] = dest
data, args = serializeEvents(args, 1, data)
if (len(args) - 1) == 0 {
return nil, nil
}
// RPUSH returns total length of list -> fail and retry all on error
_, err := conn.Do(command, args...)
if err != nil {
logp.Err("Failed to %v to redis list (%v) with %v", command, err)
return data, err
}
return nil, nil
}
}
// getIndex returns the full index name
// Index is either defined in the config as part of the output
// or can be overload by the event through setting index
func getIndex(event common.MapStr, index outil.Selector) string {
ts := time.Time(event["@timestamp"].(common.Time)).UTC()
// Check for dynamic index
// XXX: is this used/needed?
if _, ok := event["beat"]; ok {
beatMeta, ok := event["beat"].(common.MapStr)
if ok {
// Check if index is set dynamically
if dynamicIndex, ok := beatMeta["index"]; ok {
if dynamicIndexValue, ok := dynamicIndex.(string); ok {
return fmt.Sprintf("%s-%d.%02d.%02d",
dynamicIndexValue, ts.Year(), ts.Month(), ts.Day())
}
}
}
}
str, _ := index.Select(event)
return str
}