Beispiel #1
0
func (mc *Memcache) correlateTCP(conn *connection) error {
	// merge requests with responses into transactions
	for !conn.responses.empty() {
		var requ *message
		resp := conn.responses.pop()

		for !conn.requests.empty() {
			requ = conn.requests.pop()
			if requ.isBinary != resp.isBinary {
				err := ErrMixOfBinaryAndText
				logp.Warn("%v", err)
				return err
			}

			// If requ and response belong to the same transaction, continue
			// merging them into one transaction
			sameTransaction := !requ.isBinary ||
				(requ.opaque == resp.opaque &&
					requ.opcode == resp.opcode)
			if sameTransaction {
				break
			}

			// check if we are missing a response or quiet message.
			// Quiet message only MAY get a response -> so we need
			// to clear message list from all quiet messages not having
			// received a response
			if requ.isBinary && !requ.isQuiet {
				note := NoteNonQuietResponseOnly
				logp.Warn("%s", note)
				requ.AddNotes(note)
			}

			// send request
			debug("send single request=%p", requ)
			err := mc.onTCPTrans(requ, nil)
			if err != nil {
				logp.Warn("error processing memcache transaction: %s", err)
			}
			requ = nil
		}

		// Check if response without request. This should only happen when a TCP
		// stream is found (or after message gap) when we receive a response
		// without having seen a request.
		if requ == nil {
			debug("found orphan memcached response=%p", resp)
			resp.AddNotes(NoteTransactionNoRequ)
		}

		debug("merge request=%p and response=%p", requ, resp)
		err := mc.onTCPTrans(requ, resp)
		if err != nil {
			logp.Warn("error processing memcache transaction: %s", err)
		}
		// continue processing more transactions (reporting error only)
	}

	return nil
}
Beispiel #2
0
func connectionStartMethod(m *AmqpMessage, args []byte) (bool, bool) {
	major := args[0]
	minor := args[1]
	properties := make(common.MapStr)
	next, err, exists := getTable(properties, args, 2)
	if err {
		//failed to get de peer-properties, size may be wrong, let's quit
		logp.Warn("Failed to parse server properties in connection.start method")
		return false, false
	}
	mechanisms, next, err := getShortString(args, next+4, binary.BigEndian.Uint32(args[next:next+4]))
	if err {
		logp.Warn("Failed to get connection mechanisms")
		return false, false
	}
	locales, _, err := getShortString(args, next+4, binary.BigEndian.Uint32(args[next:next+4]))
	if err {
		logp.Warn("Failed to get connection locales")
		return false, false
	}
	m.Method = "connection.start"
	m.IsRequest = true
	m.Fields = common.MapStr{
		"version-major": major,
		"version-minor": minor,
		"mechanisms":    mechanisms,
		"locales":       locales,
	}
	//if there is a server properties table, add it
	if exists {
		m.Fields["server-properties"] = properties
	}
	return true, true
}
Beispiel #3
0
func (config *harvesterConfig) Validate() error {

	// DEPRECATED: remove in 6.0
	if config.ForceCloseFiles {
		config.CloseRemoved = true
		config.CloseRenamed = true
		logp.Warn("DEPRECATED: force_close_files was set to true. Use close_removed + close_rename")
	}

	// DEPRECATED: remove in 6.0
	if config.CloseOlder > 0 {
		config.CloseInactive = config.CloseOlder
		logp.Warn("DEPRECATED: close_older is deprecated. Use close_inactive")
	}

	// Check input type
	if _, ok := cfg.ValidInputType[config.InputType]; !ok {
		return fmt.Errorf("Invalid input type: %v", config.InputType)
	}

	if config.JSON != nil && len(config.JSON.MessageKey) == 0 &&
		config.Multiline != nil {
		return fmt.Errorf("When using the JSON decoder and multiline together, you need to specify a message_key value")
	}

	if config.JSON != nil && len(config.JSON.MessageKey) == 0 &&
		(len(config.IncludeLines) > 0 || len(config.ExcludeLines) > 0) {
		return fmt.Errorf("When using the JSON decoder and line filtering together, you need to specify a message_key value")
	}

	return nil
}
Beispiel #4
0
func basicConsumeMethod(m *AmqpMessage, args []byte) (bool, bool) {
	queue, offset, err := getShortString(args, 3, uint32(args[2]))
	if err {
		logp.Warn("Error getting name of queue in basic consume")
		return false, false
	}
	consumerTag, offset, err := getShortString(args, offset+1, uint32(args[offset]))
	if err {
		logp.Warn("Error getting name of consumer tag in basic consume")
		return false, false
	}
	params := getBitParams(args[offset])
	m.Method = "basic.consume"
	m.IsRequest = true
	m.Request = queue
	m.Fields = common.MapStr{
		"queue":        queue,
		"consumer-tag": consumerTag,
		"no-local":     params[0],
		"no-ack":       params[1],
		"exclusive":    params[2],
		"no-wait":      params[3],
	}
	if args[offset+1] != frameEndOctet && m.ParseArguments {
		arguments := make(common.MapStr)
		_, err, exists := getTable(arguments, args, offset+1)
		if !err && exists {
			m.Fields["arguments"] = arguments
		} else if err {
			m.Notes = append(m.Notes, "Failed to parse additional arguments")
		}
	}
	return true, true
}
Beispiel #5
0
func connectionStartOkMethod(m *AmqpMessage, args []byte) (bool, bool) {
	properties := make(common.MapStr)
	next, err, exists := getTable(properties, args, 0)
	if err {
		//failed to get de peer-properties, size may be wrong, let's quit
		logp.Warn("Failed to parse server properties in connection.start method")
		return false, false
	}
	mechanism, next, err := getShortString(args, next+1, uint32(args[next]))
	if err {
		logp.Warn("Failed to get connection mechanism from client")
		return false, false
	}
	_, next, err = getShortString(args, next+4, binary.BigEndian.Uint32(args[next:next+4]))
	if err {
		logp.Warn("Failed to get connection response from client")
		return false, false
	}
	locale, _, err := getShortString(args, next+1, uint32(args[next]))
	if err {
		logp.Warn("Failed to get connection locale from client")
		return false, false
	}
	m.IsRequest = false
	m.Fields = common.MapStr{
		"mechanism": mechanism,
		"locale":    locale,
	}
	//if there is a client properties table, add it
	if exists {
		m.Fields["client-properties"] = properties
	}
	return true, true
}
Beispiel #6
0
func basicReturnMethod(m *AmqpMessage, args []byte) (bool, bool) {
	code := binary.BigEndian.Uint16(args[0:2])
	if code < 300 {
		//not an error or exception ? not interesting
		return true, false
	}
	replyText, nextOffset, err := getShortString(args, 3, uint32(args[2]))
	if err {
		logp.Warn("Error getting name of reply text in basic return")
		return false, false
	}
	exchange, nextOffset, err := getShortString(args, nextOffset+1, uint32(args[nextOffset]))
	if err {
		logp.Warn("Error getting name of exchange in basic return")
		return false, false
	}
	routingKey, _, err := getShortString(args, nextOffset+1, uint32(args[nextOffset]))
	if err {
		logp.Warn("Error getting name of routing key in basic return")
		return false, false
	}
	m.Method = "basic.return"
	m.Fields = common.MapStr{
		"exchange":    exchange,
		"routing-key": routingKey,
		"reply-code":  code,
		"reply-text":  replyText,
	}
	return true, false
}
Beispiel #7
0
func (msw *metricSetWrapper) multiEventFetch(fetcher mb.EventsFetcher, host string) error {
	start := time.Now()
	events, err := fetcher.Fetch(host)
	elapsed := time.Since(start)
	if err == nil {
		msw.stats.Add(successesKey, 1)

		for _, event := range events {
			event, err = createEvent(msw, host, event, nil, start, elapsed)
			if err != nil {
				logp.Warn("createEvent error: %v", err)
			}

			if event != nil {
				msw.module.pubClient.PublishEvent(event)
				msw.stats.Add(eventsKey, 1)
			}
		}
	} else {
		msw.stats.Add(failuresKey, 1)

		event, err := createEvent(msw, host, nil, err, start, elapsed)
		if err != nil {
			logp.Warn("createEvent error: %v", err)
		}

		if event != nil {
			msw.module.pubClient.PublishEvent(event)
			msw.stats.Add(eventsKey, 1)
		}
	}

	return nil
}
Beispiel #8
0
func (*parser) parseBodyChunkedStart(s *stream, m *message) (cont, ok, complete bool) {
	// read hexa length
	i := bytes.Index(s.data[s.parseOffset:], []byte("\r\n"))
	if i == -1 {
		return false, true, false
	}
	line := string(s.data[s.parseOffset : s.parseOffset+i])
	_, err := fmt.Sscanf(line, "%x", &m.chunkedLength)
	if err != nil {
		logp.Warn("Failed to understand chunked body start line")
		return false, false, false
	}

	s.parseOffset += i + 2 //+ \r\n
	if m.chunkedLength == 0 {
		if len(s.data[s.parseOffset:]) < 2 {
			s.parseState = stateBodyChunkedWaitFinalCRLF
			return false, true, false
		}
		if s.data[s.parseOffset] != '\r' || s.data[s.parseOffset+1] != '\n' {
			logp.Warn("Expected CRLF sequence at end of message")
			return false, false, false
		}
		s.parseOffset += 2 // skip final CRLF

		m.end = s.parseOffset
		m.Size = uint64(m.end - m.start)
		return false, true, true
	}
	s.bodyReceived = 0
	s.parseState = stateBodyChunked

	return true, true, false
}
Beispiel #9
0
func basicDeliverMethod(m *AmqpMessage, args []byte) (bool, bool) {
	consumerTag, offset, err := getShortString(args, 1, uint32(args[0]))
	if err {
		logp.Warn("Failed to get consumer tag in basic deliver")
		return false, false
	}
	deliveryTag := binary.BigEndian.Uint64(args[offset : offset+8])
	params := getBitParams(args[offset+8])
	offset = offset + 9
	exchange, offset, err := getShortString(args, offset+1, uint32(args[offset]))
	if err {
		logp.Warn("Failed to get exchange in basic deliver")
		return false, false
	}
	routingKey, _, err := getShortString(args, offset+1, uint32(args[offset]))
	if err {
		logp.Warn("Failed to get routing key in basic deliver")
		return false, false
	}
	m.Method = "basic.deliver"
	m.Fields = common.MapStr{
		"consumer-tag": consumerTag,
		"delivery-tag": deliveryTag,
		"redelivered":  params[0],
		"routing-key":  routingKey,
	}
	// is exchange not default exchange ?
	if len(exchange) > 0 {
		m.Fields["exchange"] = exchange
	}
	return true, false
}
Beispiel #10
0
func (t *Topbeat) exportSystemStats() error {
	load_stat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return err
	}
	cpu_stat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return err
	}

	t.addCpuPercentage(cpu_stat)

	cpu_core_stat, err := GetCpuTimesList()
	if err != nil {
		logp.Warn("Getting cpu core times: %v", err)
		return err
	}
	t.addCpuPercentageList(cpu_core_stat)

	mem_stat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return err
	}
	t.addMemPercentage(mem_stat)

	swap_stat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return err
	}
	t.addSwapPercentage(swap_stat)

	event := common.MapStr{
		"@timestamp": common.Time(time.Now()),
		"type":       "system",
		"load":       load_stat,
		"cpu":        cpu_stat,
		"mem":        mem_stat,
		"swap":       swap_stat,
		"count":      1,
	}

	if t.cpuPerCore {

		cpus := common.MapStr{}

		for coreNumber, stat := range cpu_core_stat {
			cpus["cpu"+strconv.Itoa(coreNumber)] = stat
		}
		event["cpus"] = cpus
	}

	t.events.PublishEvent(event)

	return nil
}
Beispiel #11
0
Datei: http.go Projekt: tsg/beats
func (http *HTTP) receivedHTTPResponse(msg *message) {

	// we need to search the request first.
	tuple := msg.TCPTuple

	debugf("Received response with tuple: %s", tuple)

	trans := http.getTransaction(tuple.Hashable())
	if trans == nil {
		logp.Warn("Response from unknown transaction. Ignoring: %v", tuple)
		return
	}

	if trans.HTTP == nil {
		logp.Warn("Response without a known request. Ignoring.")
		return
	}

	response := common.MapStr{
		"phrase":         msg.StatusPhrase,
		"code":           msg.StatusCode,
		"content_length": msg.ContentLength,
	}

	if http.parserConfig.SendHeaders {
		if !http.SplitCookie {
			response["response_headers"] = msg.Headers
		} else {
			hdrs := common.MapStr{}
			for name, value := range msg.Headers {
				if name == "set-cookie" {
					hdrs[name] = splitCookiesHeader(value)
				} else {
					hdrs[name] = value
				}
			}

			response["response_headers"] = hdrs
		}
	}

	trans.BytesOut = msg.Size
	trans.HTTP.Update(response)
	trans.Notes = append(trans.Notes, msg.Notes...)

	trans.ResponseTime = int32(msg.Ts.Sub(trans.ts).Nanoseconds() / 1e6) // resp_time in milliseconds

	// save Raw message
	if http.SendResponse {
		trans.ResponseRaw = string(http.cutMessageBody(msg))
	}

	http.publishTransaction(trans)
	http.transactions.Delete(trans.tuple.Hashable())

	debugf("HTTP transaction completed: %s\n", trans.HTTP)
}
Beispiel #12
0
func (cpu *CPU) GetSystemStats() (common.MapStr, error) {
	loadStat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return nil, err
	}
	cpuStat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return nil, err
	}

	cpu.AddCpuPercentage(cpuStat)

	memStat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return nil, err
	}
	AddMemPercentage(memStat)

	swapStat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return nil, err
	}
	AddSwapPercentage(swapStat)

	event := common.MapStr{
		"@timestamp": common.Time(time.Now()),
		"type":       "system",
		"load":       loadStat,
		"cpu":        GetCpuStatEvent(cpuStat),
		"mem":        GetMemoryEvent(memStat),
		"swap":       GetSwapEvent(swapStat),
	}

	if cpu.CpuPerCore {

		cpuCoreStat, err := GetCpuTimesList()
		if err != nil {
			logp.Warn("Getting cpu core times: %v", err)
			return nil, err
		}
		cpu.AddCpuPercentageList(cpuCoreStat)

		cpus := common.MapStr{}

		for coreNumber, stat := range cpuCoreStat {
			cpus["cpu"+strconv.Itoa(coreNumber)] = GetCpuStatEvent(&stat)
		}
		event["cpus"] = cpus
	}

	return event, nil
}
Beispiel #13
0
func LoadGeoIPData(config Geoip) *libgeo.GeoIP {

	geoipPaths := []string{}

	if config.Paths != nil {
		geoipPaths = *config.Paths
	}
	if len(geoipPaths) == 0 {
		logp.Info("GeoIP disabled: No paths were set under output.geoip.paths")
		// disabled
		return nil
	}

	// look for the first existing path
	var geoipPath string
	for _, path := range geoipPaths {
		fi, err := os.Lstat(path)
		if err != nil {
			logp.Err("GeoIP path could not be loaded: %s", path)
			continue
		}

		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
			// follow symlink
			geoipPath, err = filepath.EvalSymlinks(path)
			if err != nil {
				logp.Warn("Could not load GeoIP data: %s", err.Error())
				return nil
			}
		} else {
			geoipPath = path
		}
		break
	}

	if len(geoipPath) == 0 {
		logp.Warn("Couldn't load GeoIP database")
		return nil
	}

	geoLite, err := libgeo.Load(geoipPath)
	if err != nil {
		logp.Warn("Could not load GeoIP data: %s", err.Error())
	}

	logp.Info("Loaded GeoIP data from: %s", geoipPath)
	return geoLite
}
Beispiel #14
0
func (amqp *Amqp) amqpMessageParser(s *AmqpStream) (ok bool, complete bool) {
	for s.parseOffset < len(s.data) {

		if len(s.data[s.parseOffset:]) < 8 {
			logp.Warn("AMQP message smaller than a frame, waiting for more data")
			return true, false
		}

		yes, version := isProtocolHeader(s.data[s.parseOffset:])
		if yes {
			debugf("Client header detected, version %d.%d.%d",
				version[0], version[1], version[2])
			s.parseOffset += 8
		}

		f, err := readFrameHeader(s.data[s.parseOffset:])
		if err {
			//incorrect header
			return false, false
		} else if f == nil {
			//header not complete
			return true, false
		}

		switch f.Type {
		case methodType:
			ok, complete = amqp.decodeMethodFrame(s, f.content)
		case headerType:
			ok = amqp.decodeHeaderFrame(s, f.content)
		case bodyType:
			ok, complete = s.decodeBodyFrame(f.content)
		case heartbeatType:
			detailedf("Heartbeat frame received")
		default:
			logp.Warn("Received unknown AMQP frame")
			return false, false
		}

		// cast should be safe because f.size should not be bigger than tcp.TCP_MAX_DATA_IN_STREAM
		s.parseOffset += 8 + int(f.size)
		if !ok {
			return false, false
		} else if complete {
			return true, true
		}
	}
	return ok, complete
}
Beispiel #15
0
func (f *FileEvent) ToMapStr() common.MapStr {
	event := common.MapStr{
		"@timestamp": common.Time(f.ReadTime),
		"source":     f.Source,
		"offset":     f.Offset, // Offset here is the offset before the starting char.
		"message":    f.Text,
		"type":       f.DocumentType,
		"input_type": f.InputType,
		"count":      1,
	}

	if f.Fields != nil {
		if f.fieldsUnderRoot {
			for key, value := range *f.Fields {
				// in case of conflicts, overwrite
				_, found := event[key]
				if found {
					logp.Warn("Overwriting %s key", key)
				}
				event[key] = value
			}
		} else {
			event["fields"] = f.Fields
		}
	}

	return event
}
Beispiel #16
0
func (c *Condition) CheckEquals(event common.MapStr) bool {

	for field, equalValue := range c.Equals {

		value, err := event.GetValue(field)
		if err != nil {
			logp.Debug("filter", "unavailable field %s: %v", field, err)
			return false
		}

		switch value.(type) {
		case uint8, uint16, uint32, uint64, int8, int16, int32, int64, int, uint:
			return value == equalValue.Int
		case string:
			return value == equalValue.Str
		default:
			logp.Warn("unexpected type %T in equals condition as it accepts only integers and strings. ", value)
			return false
		}

	}

	return true

}
Beispiel #17
0
func queueDeclareMethod(m *AmqpMessage, args []byte) (bool, bool) {
	name, offset, err := getShortString(args, 3, uint32(args[2]))
	if err {
		logp.Warn("Error getting name of queue in queue declaration")
		return false, false
	}
	m.IsRequest = true
	m.Method = "queue.declare"
	params := getBitParams(args[offset])
	m.Request = name
	m.Fields = common.MapStr{
		"queue":       name,
		"passive":     params[0],
		"durable":     params[1],
		"exclusive":   params[2],
		"auto-delete": params[3],
		"no-wait":     params[4],
	}
	if args[offset+1] != frameEndOctet && m.ParseArguments {
		arguments := make(common.MapStr)
		_, err, exists := getTable(arguments, args, offset+1)
		if !err && exists {
			m.Fields["arguments"] = arguments
		} else if err {
			m.Notes = append(m.Notes, "Failed to parse additional arguments")
		}
	}
	return true, true
}
Beispiel #18
0
func (eb *Winlogbeat) Setup(b *beat.Beat) error {
	eb.beat = b
	eb.client = b.Events
	eb.done = make(chan struct{})

	var err error
	eb.checkpoint, err = checkpoint.NewCheckpoint(
		eb.config.Winlogbeat.RegistryFile, 10, 5*time.Second)
	if err != nil {
		return err
	}

	if eb.config.Winlogbeat.Metrics.BindAddress != "" {
		bindAddress := eb.config.Winlogbeat.Metrics.BindAddress
		sock, err := net.Listen("tcp", bindAddress)
		if err != nil {
			return err
		}
		go func() {
			logp.Info("Metrics hosted at http://%s/debug/vars", bindAddress)
			err := http.Serve(sock, nil)
			if err != nil {
				logp.Warn("Unable to launch HTTP service for metrics. %v", err)
				return
			}
		}()
	}

	return nil
}
Beispiel #19
0
func exportContainerStats(client *docker.Client, container *docker.APIContainers) Stat {
	var wg sync.WaitGroup
	var event Stat

	statsC := make(chan *docker.Stats)
	errC := make(chan error, 1)
	statsOptions := docker.StatsOptions{
		ID:      container.ID,
		Stats:   statsC,
		Stream:  false,
		Timeout: -1,
	}

	wg.Add(2)
	go func() {
		defer wg.Done()
		errC <- client.Stats(statsOptions)
		close(errC)
	}()
	go func() {
		defer wg.Done()
		stats := <-statsC
		err := <-errC
		if stats != nil && err == nil {
			event.Stats = *stats
			event.Container = *container
		} else if err == nil && stats == nil {
			logp.Warn("Container stopped when recovering stats: %v", container.ID)
		} else {
			logp.Err("An error occurred while getting docker stats: %v", err)
		}
	}()
	wg.Wait()
	return event
}
Beispiel #20
0
// Setup uses the loaded config and creates necessary markers and environment
// settings to allow the beat to be used.
func (eb *Winlogbeat) setup(b *beat.Beat) error {
	config := &eb.config.Winlogbeat

	eb.client = b.Publisher.Connect()

	var err error
	eb.checkpoint, err = checkpoint.NewCheckpoint(config.RegistryFile, 10, 5*time.Second)
	if err != nil {
		return err
	}

	if config.Metrics.BindAddress != "" {
		bindAddress := config.Metrics.BindAddress
		sock, err := net.Listen("tcp", bindAddress)
		if err != nil {
			return err
		}
		go func() {
			logp.Info("Metrics hosted at http://%s/debug/vars", bindAddress)
			err := http.Serve(sock, nil)
			if err != nil {
				logp.Warn("Unable to launch HTTP service for metrics. %v", err)
			}
		}()
	}

	return nil
}
Beispiel #21
0
func (d *Dockerbeat) Run(b *beat.Beat) error {
	var err error

	ticker := time.NewTicker(d.period)
	defer ticker.Stop()

	for {
		select {
		case <-d.done:
			return nil
		case <-ticker.C:
		}

		// check prerequisites
		err = d.checkPrerequisites()
		if err != nil {
			logp.Err("Unable to collect metrics: %v", err)
			continue
		}

		timerStart := time.Now()
		d.RunOneTime(b)
		timerEnd := time.Now()

		duration := timerEnd.Sub(timerStart)
		if duration.Nanoseconds() > d.period.Nanoseconds() {
			logp.Warn("Ignoring tick(s) due to processing taking longer than one period")
		}
	}

	return err
}
Beispiel #22
0
// bulkCollectPublishFails checks per item errors returning all events
// to be tried again due to error code returned for that items. If indexing an
// event failed due to some error in the event itself (e.g. does not respect mapping),
// the event will be dropped.
func bulkCollectPublishFails(
	reader *jsonReader,
	events []common.MapStr,
) []common.MapStr {
	if err := reader.expectDict(); err != nil {
		logp.Err("Failed to parse bulk respose: expected JSON object")
		return nil
	}

	// find 'items' field in response
	for {
		kind, name, err := reader.nextFieldName()
		if err != nil {
			logp.Err("Failed to parse bulk response")
			return nil
		}

		if kind == dictEnd {
			logp.Err("Failed to parse bulk response: no 'items' field in response")
			return nil
		}

		// found items array -> continue
		if bytes.Equal(name, nameItems) {
			break
		}

		reader.ignoreNext()
	}

	// check items field is an array
	if err := reader.expectArray(); err != nil {
		logp.Err("Failed to parse bulk respose: expected items array")
		return nil
	}

	count := len(events)
	failed := events[:0]
	for i := 0; i < count; i++ {
		status, msg, err := itemStatus(reader)
		if err != nil {
			return nil
		}

		if status < 300 {
			continue // ok value
		}

		if status < 500 && status != 429 {
			// hard failure, don't collect
			logp.Warn("Can not index event (status=%v): %s", status, msg)
			continue
		}

		logp.Info("Bulk item insert failed (i=%v, status=%v): %s", i, status, msg)
		failed = append(failed, events[i])
	}

	return failed
}
Beispiel #23
0
func (client *Client) PublishEvent(event common.MapStr) error {
	if !client.connected {
		return ErrNotConnected
	}

	index := getIndex(event, client.index)
	logp.Debug("output_elasticsearch", "Publish event: %s", event)

	// insert the events one by one
	status, _, err := client.Index(
		index, event["type"].(string), "", client.params, event)
	if err != nil {
		logp.Warn("Fail to insert a single event: %s", err)
		if err == ErrJSONEncodeFailed {
			// don't retry unencodable values
			return nil
		}
	}
	switch {
	case status == 0: // event was not send yet
		return nil
	case status >= 500 || status == 429: // server error, retry
		return err
	case status >= 300 && status < 500:
		// won't be able to index event in Elasticsearch => don't retry
		return nil
	}

	return nil
}
Beispiel #24
0
func eventMapping(response io.Reader) common.MapStr {
	fullEvent := map[string]interface{}{}
	scanner := bufio.NewScanner(response)

	// Iterate through all events to gather data
	for scanner.Scan() {
		if match := paramMatcher.FindStringSubmatch(scanner.Text()); len(match) == 3 {
			fullEvent[match[1]] = match[2]
		} else {
			logp.Warn("Unexpected line in mntr output: %s", scanner.Text())
		}
	}

	event := schema_.Apply(fullEvent)

	// only exposed by the Leader
	if _, ok := fullEvent["zk_followers"]; ok {
		schemaLeader.ApplyTo(event, fullEvent)
	}

	// only available on Unix platforms
	if _, ok := fullEvent["open_file_descriptor_count"]; ok {
		schemaUnix.ApplyTo(event, fullEvent)
	}

	return event
}
Beispiel #25
0
func (amqp *Amqp) handleAmqpResponse(msg *AmqpMessage) {
	tuple := msg.TcpTuple
	trans := amqp.getTransaction(tuple.Hashable())
	if trans == nil || trans.Amqp == nil {
		logp.Warn("Response from unknown transaction. Ignoring.")
		return
	}

	//length = message + 4 bytes class/method + frame end octet + header
	trans.BytesOut = msg.Body_size + 12
	//merge the both fields from request and response
	trans.Amqp.Update(msg.Fields)
	trans.Response = common.OK_STATUS

	if msg.Method == "basic.get-empty" {
		trans.Method = "basic.get-empty"
	}

	trans.ResponseTime = int32(msg.Ts.Sub(trans.ts).Nanoseconds() / 1e6)
	trans.Notes = msg.Notes

	amqp.publishTransaction(trans)

	debugf("Amqp transaction completed")

	// remove from map
	amqp.transactions.Delete(trans.tuple.Hashable())
	if trans.timer != nil {
		trans.timer.Stop()
	}
}
Beispiel #26
0
func (conn *Connection) execRequest(
	method, url string,
	body io.Reader,
) (int, []byte, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		logp.Warn("Failed to create request", err)
		return 0, nil, err
	}

	req.Header.Add("Accept", "application/json")
	if conn.Username != "" || conn.Password != "" {
		req.SetBasicAuth(conn.Username, conn.Password)
	}

	resp, err := conn.http.Do(req)
	if err != nil {
		conn.connected = false
		return 0, nil, err
	}
	defer closing(resp.Body)

	status := resp.StatusCode
	if status >= 300 {
		conn.connected = false
		return status, nil, fmt.Errorf("%v", resp.Status)
	}

	obj, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		conn.connected = false
		return status, nil, err
	}
	return status, obj, nil
}
Beispiel #27
0
func getConnection(private protos.ProtocolData) *connection {
	if private == nil {
		return nil
	}

	priv, ok := private.(*connection)
	if !ok {
		logp.Warn("cassandra connection type error")
		return nil
	}
	if priv == nil {
		logp.Warn("Unexpected: cassandra connection data not set")
		return nil
	}
	return priv
}
Beispiel #28
0
func (mc *Memcache) pushAllTCPTrans(conn *connection) {
	if conn == nil {
		return
	}

	// first let's try to send finished transactions
	// (unlikely we have some, though)
	mc.correlateTCP(conn)

	// only requests in map:
	debug("publish incomplete transactions")
	for !conn.requests.empty() {
		msg := conn.requests.pop()
		if !msg.isQuiet && !msg.noreply {
			msg.AddNotes(NoteTransUnfinished)
			unfinishedTransactions.Add(1)
		}
		debug("push incomplete request=%p", msg)
		err := mc.onTCPTrans(msg, nil)
		if err != nil {
			logp.Warn("failed to publish unfinished transaction with %v", err)
		}
		// continue processing more transactions (reporting error only)
	}
}
Beispiel #29
0
func (h *Harvester) close() {

	// Mark harvester as finished
	h.state.Finished = true

	logp.Debug("harvester", "Stopping harvester for file: %s", h.state.Source)

	// Make sure file is closed as soon as harvester exits
	// If file was never opened, it can't be closed
	if h.file != nil {

		h.file.Close()

		logp.Debug("harvester", "Closing file: %s", h.state.Source)
		harvesterOpenFiles.Add(-1)

		// On completion, push offset so we can continue where we left off if we relaunch on the same file
		// Only send offset if file object was created successfully
		h.sendStateUpdate()
	} else {
		logp.Warn("Stopping harvester, NOT closing file as file info not available: %s", h.state.Source)
	}

	harvesterClosed.Add(1)
}
Beispiel #30
0
func ensureMemcacheConnection(private protos.ProtocolData) *tcpConnectionData {
	if private == nil {
		return &tcpConnectionData{}
	}

	priv, ok := private.(*tcpConnectionData)
	if !ok {
		logp.Warn("memcache connection data type error, create new one")
		return &tcpConnectionData{}
	}
	if priv == nil {
		logp.Warn("Unexpected: memcache TCP connection data not set, create new one")
		return &tcpConnectionData{}
	}
	return priv
}