예제 #1
0
func (self *FileOutput) startRotateNotifier() {
	now := time.Now()
	interval := time.Duration(self.config.RotationInterval) * time.Hour
	last := now.Truncate(interval)
	next := last.Add(interval)
	until := next.Sub(now)
	after := time.After(until)

	self.path = gostrftime.Strftime(self.config.Path, now)

	go func() {
		ok := true
		for ok {
			select {
			case _, ok = <-self.closing:
				break
			case <-after:
				last = next
				next = next.Add(interval)
				until = next.Sub(time.Now())
				after = time.After(until)
				self.rotateChan <- last
			}
		}
	}()
}
예제 #2
0
파일: file_output.go 프로젝트: Nitro/heka
// Runs in a separate goroutine, waits for buffered data on the committer
// channel, writes it out to the filesystem, and puts the now empty buffer on
// the return channel for reuse.
func (o *FileOutput) committer(or OutputRunner, errChan chan error) {
	initBatch := newOutBatch()
	o.backChan <- initBatch
	var out *outBatch
	var err error

	ok := true
	hupChan := make(chan interface{})
	notify.Start(RELOAD, hupChan)

	for ok {
		select {
		case out, ok = <-o.batchChan:
			if !ok {
				// Channel is closed => we're shutting down, exit cleanly.
				o.file.Close()
				close(o.closing)
				break
			}
			n, err := o.file.Write(out.data)
			if err != nil {
				or.LogError(fmt.Errorf("Can't write to %s: %s", o.path, err))
			} else if n != len(out.data) {
				or.LogError(fmt.Errorf("data loss - truncated output for %s", o.path))
				or.UpdateCursor(out.cursor)
			} else {
				o.file.Sync()
				or.UpdateCursor(out.cursor)
			}
			out.data = out.data[:0]
			o.backChan <- out
		case <-hupChan:
			o.file.Close()
			if err = o.openFile(); err != nil {
				close(o.closing)
				err = fmt.Errorf("unable to reopen file '%s': %s", o.path, err)
				errChan <- err
				ok = false
				break
			}
		case rotateTime := <-o.rotateChan:
			o.file.Close()
			o.path = gostrftime.Strftime(o.FileOutputConfig.Path, rotateTime)
			if err = o.openFile(); err != nil {
				close(o.closing)
				err = fmt.Errorf("unable to open rotated file '%s': %s", o.path, err)
				errChan <- err
				ok = false
				break
			}
		}
	}
}
예제 #3
0
파일: coordinates.go 프로젝트: Nitro/heka
// Replaces a date pattern (ex: %{2012.09.19} in the index name
func interpolateFlag(e *ElasticSearchCoordinates, m *message.Message, name string) (
	interpolatedValue string, err error) {

	iSlice := strings.Split(name, "%{")

	for i, element := range iSlice {
		elEnd := strings.Index(element, "}")

		if elEnd > -1 {
			elVal := element[:elEnd]
			switch elVal {
			case "Type":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetType(), -1)
			case "Hostname":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetHostname(), -1)
			case "Pid":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1],
					strconv.Itoa(int(m.GetPid())), -1)
			case "UUID":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetUuidString(), -1)
			case "Logger":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetLogger(), -1)
			case "EnvVersion":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], m.GetEnvVersion(), -1)
			case "Severity":
				iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1],
					strconv.Itoa(int(m.GetSeverity())), -1)
			default:
				if fname, ok := m.GetFieldValue(elVal); ok {
					iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], fname.(string), -1)
				} else {
					var t time.Time
					if e.ESIndexFromTimestamp && m.Timestamp != nil {
						t = time.Unix(0, *m.Timestamp).UTC()
					} else {
						t = time.Now().UTC()
					}
					iSlice[i] = strings.Replace(iSlice[i], element[:elEnd+1], gostrftime.Strftime(elVal, t), -1)
				}
			}
			if iSlice[i] == elVal {
				err = fmt.Errorf("Could not interpolate field from config: %s", name)
			}
		}
	}
	interpolatedValue = strings.Join(iSlice, "")
	return
}
예제 #4
0
// Runs in a separate goroutine, waits for buffered data on the committer
// channel, writes it out to the filesystem, and puts the now empty buffer on
// the return channel for reuse.
func (self *FileOutput) committer(or plugins.OutputRunner, errChan chan error) {
	initBatch := newOutBatch()
	self.backChan <- initBatch
	var out *outBatch
	var err error

	ok := true

	for ok {
		select {
		case out, ok = <-self.batchChan:
			if !ok {
				// Channel is closed => we're shutting down, exit cleanly.
				self.file.Close()
				close(self.closing)
				break
			}

			n, err := self.file.Write(out.data)
			if err != nil {
				log.Println(fmt.Errorf("Can't write to %s: %s", self.path, err))
			} else if n != len(out.data) {
				log.Println(fmt.Errorf("data loss - truncated output for %s", self.path))
			} else {
				self.file.Sync()
			}

			out.data = out.data[:0]
			self.backChan <- out
		case rotateTime := <-self.rotateChan:
			self.file.Close()
			self.path = gostrftime.Strftime(self.config.Path, rotateTime)
			if err = self.openFile(); err != nil {
				close(self.closing)
				err = fmt.Errorf("unable to open rotated file '%s': %s", self.path, err)
				errChan <- err
				ok = false
				break
			}
		}
	}
}
예제 #5
0
func (pe *PayloadEncoder) Encode(pack *pipeline.PipelinePack) (output []byte, err error) {
	payload := pack.Message.GetPayload()

	if !pe.config.AppendNewlines && !pe.config.PrefixTs {
		// Just the payload, ma'am.
		output = []byte(payload)
		return
	}

	if !pe.config.PrefixTs {
		// Payload + newline.
		output = make([]byte, 0, len(payload)+1)
		output = append(output, []byte(payload)...)
		output = append(output, '\n')
		return
	}

	// We're using a timestamp.
	var tm time.Time
	if pe.config.TsFromMessage {
		tm = time.Unix(0, pack.Message.GetTimestamp())
	} else {
		tm = time.Now()
	}
	ts := gostrftime.Strftime(pe.config.TsFormat, tm)

	// Timestamp + payload [+ optional newline].
	l := len(ts) + len(payload)
	output = make([]byte, 0, l+1)
	output = append(output, []byte(ts)...)
	output = append(output, []byte(payload)...)
	if pe.config.AppendNewlines {
		output = append(output, '\n')
	}
	return
}
예제 #6
0
파일: encoders.go 프로젝트: Nitro/heka
func (e *ESLogstashV0Encoder) Encode(pack *PipelinePack) (output []byte, err error) {
	m := pack.Message
	buf := bytes.Buffer{}
	e.coord.PopulateBuffer(pack.Message, &buf)
	buf.WriteByte(NEWLINE)
	buf.WriteString(`{`)

	first := true
	for _, f := range e.fields {
		switch strings.ToLower(f) {
		case "uuid":
			writeStringField(first, &buf, `@uuid`, m.GetUuidString())
		case "timestamp":
			t := time.Unix(0, m.GetTimestamp()).UTC()
			writeStringField(first, &buf, `@timestamp`, gostrftime.Strftime(e.timestampFormat, t))
		case "type":
			if e.useMessageType || len(e.coord.Type) < 1 {
				writeStringField(first, &buf, `@type`, m.GetType())
			} else {
				var interpType string
				interpType, err = interpolateFlag(e.coord, m, e.coord.Type)
				if len(interpType) > 0 && err == nil {
					writeStringField(first, &buf, `@type`, interpType)
				} else {
					// fall back on writing the uninterpolated string
					writeStringField(first, &buf, `@type`, e.coord.Type)
				}
			}
		case "logger":
			writeStringField(first, &buf, `@logger`, m.GetLogger())
		case "severity":
			writeIntField(first, &buf, `@severity`, m.GetSeverity())
		case "payload":
			writeStringField(first, &buf, `@message`, m.GetPayload())
		case "envversion":
			writeStringField(first, &buf, `@envversion`, m.GetEnvVersion())
		case "pid":
			writeIntField(first, &buf, `@pid`, m.GetPid())
		case "hostname":
			writeStringField(first, &buf, `@source_host`, m.GetHostname())
		case "dynamicfields":
			if !first {
				buf.WriteString(`,`)
			}
			buf.WriteString(`"@fields":{`)
			firstfield := true

			listsDynamicFields := len(e.dynamicFields) > 0
			for _, field := range m.Fields {
				dynamicFieldMatch := false
				if listsDynamicFields {
					for _, fieldName := range e.dynamicFields {
						if *field.Name == fieldName {
							dynamicFieldMatch = true
						}
					}
				} else {
					dynamicFieldMatch = true
				}

				if dynamicFieldMatch {
					raw := false
					if len(e.rawBytesFields) > 0 {
						for _, raw_field_name := range e.rawBytesFields {
							if *field.Name == raw_field_name {
								raw = true
							}
						}
					}
					writeField(firstfield, &buf, field, raw)
					firstfield = false
				}
			}
			buf.WriteString(`}`) // end of fields
		default:
			err = fmt.Errorf("Unable to find field: %s", f)
			return
		}
		first = false
	}
	buf.WriteString(`}`)
	buf.WriteByte(NEWLINE)
	return buf.Bytes(), err
}
예제 #7
0
파일: encoders.go 프로젝트: Nitro/heka
func (e *ESJsonEncoder) Encode(pack *PipelinePack) (output []byte, err error) {
	m := pack.Message
	buf := bytes.Buffer{}
	e.coord.PopulateBuffer(pack.Message, &buf)
	buf.WriteByte(NEWLINE)
	buf.WriteString(`{`)
	first := true

	for _, f := range e.fields {
		switch strings.ToLower(f) {
		case "uuid":
			writeStringField(first, &buf, e.fieldMappings.Uuid, m.GetUuidString())
		case "timestamp":
			t := time.Unix(0, m.GetTimestamp()).UTC()
			writeStringField(first, &buf, e.fieldMappings.Timestamp, gostrftime.Strftime(e.timestampFormat, t))
		case "type":
			writeStringField(first, &buf, e.fieldMappings.Type, m.GetType())
		case "logger":
			writeStringField(first, &buf, e.fieldMappings.Logger, m.GetLogger())
		case "severity":
			writeIntField(first, &buf, e.fieldMappings.Severity, m.GetSeverity())
		case "payload":
			writeStringField(first, &buf, e.fieldMappings.Payload, m.GetPayload())
		case "envversion":
			writeStringField(first, &buf, e.fieldMappings.EnvVersion, m.GetEnvVersion())
		case "pid":
			writeIntField(first, &buf, e.fieldMappings.Pid, m.GetPid())
		case "hostname":
			writeStringField(first, &buf, e.fieldMappings.Hostname, m.GetHostname())
		case "dynamicfields":
			listsDynamicFields := len(e.dynamicFields) > 0

			for _, field := range m.Fields {
				dynamicFieldMatch := false
				if listsDynamicFields {
					for _, fieldName := range e.dynamicFields {
						if *field.Name == fieldName {
							dynamicFieldMatch = true
						}
					}
				} else {
					dynamicFieldMatch = true
				}

				if dynamicFieldMatch {
					raw := false
					if len(e.rawBytesFields) > 0 {
						for _, raw_field_name := range e.rawBytesFields {
							if *field.Name == raw_field_name {
								raw = true
							}
						}
					}
					writeField(first, &buf, field, raw)
					first = false
				}
			}
		default:
			err = fmt.Errorf("Unable to find field: %s", f)
			return
		}
		first = false
	}

	buf.WriteString(`}`)
	buf.WriteByte(NEWLINE)
	return buf.Bytes(), err
}