Beispiel #1
0
func (out *redisOutput) BulkPublish(
	signal outputs.Signaler,
	ts time.Time,
	events []common.MapStr,
) error {
	if !out.connected {
		logp.Debug("output_redis", "Droping pkt ...")
		return errors.New("Not connected")
	}

	command := "RPUSH"
	if out.DataType == RedisChannelType {
		command = "PUBLISH"
	}

	if len(events) == 1 { // single event
		event := events[0]
		jsonEvent, err := json.Marshal(event)
		if err != nil {
			logp.Err("Fail to convert the event to JSON: %s", err)
			outputs.SignalCompleted(signal)
			return err
		}

		_, err = out.Conn.Do(command, out.Index, string(jsonEvent))
		outputs.Signal(signal, err)
		out.onFail(err)
		return err
	}

	for _, event := range events {
		jsonEvent, err := json.Marshal(event)
		if err != nil {
			logp.Err("Fail to convert the event to JSON: %s", err)
			continue
		}
		err = out.Conn.Send(command, out.Index, string(jsonEvent))
		if err != nil {
			outputs.SignalFailed(signal, err)
			out.onFail(err)
			return err
		}
	}
	if err := out.Conn.Flush(); err != nil {
		outputs.Signal(signal, err)
		out.onFail(err)
		return err
	}
	_, err := out.Conn.Receive()
	outputs.Signal(signal, err)
	out.onFail(err)
	return err
}
Beispiel #2
0
func (out *elasticsearchOutput) BulkPublish(
	trans outputs.Signaler,
	ts time.Time,
	events []common.MapStr,
) error {
	go func() {
		request, err := out.Conn.startBulkRequest("", "", nil)
		if err != nil {
			logp.Err("Failed to perform many index operations in a single API call: %s", err)
			outputs.Signal(trans, err)
			return
		}

		for _, event := range events {
			ts := time.Time(event["timestamp"].(common.Time)).UTC()
			index := fmt.Sprintf("%s-%d.%02d.%02d",
				out.Index, ts.Year(), ts.Month(), ts.Day())
			meta := common.MapStr{
				"index": map[string]interface{}{
					"_index": index,
					"_type":  event["type"].(string),
				},
			}
			err := request.Send(meta, event)
			if err != nil {
				logp.Err("Fail to encode event: %s", err)
			}
		}

		_, err = request.Flush()
		outputs.Signal(trans, err)
		if err != nil {
			logp.Err("Failed to perform many index operations in a single API call: %s",
				err)
		}
	}()
	return nil
}
Beispiel #3
0
func (out *fileOutput) PublishEvent(
	trans outputs.Signaler,
	ts time.Time,
	event common.MapStr,
) error {
	jsonEvent, err := json.Marshal(event)
	if err != nil {
		// mark as success so event is not send again.
		outputs.SignalCompleted(trans)

		logp.Err("Fail to convert the event to JSON: %s", err)
		return err
	}

	err = out.rotator.WriteLine(jsonEvent)
	outputs.Signal(trans, err)
	return err
}
Beispiel #4
0
// Publish an event by adding it to the queue of events.
func (out *elasticsearchOutput) PublishEvent(
	signaler outputs.Signaler,
	ts time.Time,
	event common.MapStr,
) error {
	index := fmt.Sprintf("%s-%d.%02d.%02d",
		out.Index, ts.Year(), ts.Month(), ts.Day())

	logp.Debug("output_elasticsearch", "Publish event: %s", event)

	// insert the events one by one
	_, err := out.Conn.Index(index, event["type"].(string), "", nil, event)
	outputs.Signal(signaler, err)
	if err != nil {
		logp.Err("Fail to insert a single event: %s", err)
	}

	return nil
}