// Configure initializes this producer with values from a plugin config.
func (prod *Socket) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB
prod.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
prod.ackTimeout = time.Duration(conf.GetInt("AckTimeoutMs", 2000)) * time.Millisecond
prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))
if prod.protocol != "unix" {
if prod.acknowledge != "" {
prod.protocol = "tcp"
} else {
prod.protocol = "udp"
}
}
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.assembly = core.NewWriterAssembly(nil, prod.Drop, prod.GetFormatter())
prod.assembly.SetValidator(prod.validate)
prod.assembly.SetErrorHandler(prod.onWriteError)
prod.SetCheckFuseCallback(prod.tryConnect)
return nil
}
// Flush writes the content of the buffer to a given resource and resets the
// internal state, i.e. the buffer is empty after a call to Flush.
// Writing will be done in a separate go routine to be non-blocking.
//
// The validate callback will be called after messages have been successfully
// written to the io.Writer.
// If validate returns false the buffer will not be resetted (automatic retry).
// If validate is nil a return value of true is assumed (buffer reset).
//
// The onError callback will be called if the io.Writer returned an error.
// If onError returns false the buffer will not be resetted (automatic retry).
// If onError is nil a return value of true is assumed (buffer reset).
func (batch *MessageBatch) Flush(assemble AssemblyFunc) {
if batch.IsEmpty() {
return // ### return, nothing to do ###
}
// Only one flush at a time
batch.flushing.IncWhenDone()
// Switch the buffers so writers can go on writing
flushSet := atomic.SwapUint32(&batch.activeSet, (batch.activeSet&messageBatchIndexMask)^messageBatchIndexMask)
flushIdx := flushSet >> messageBatchIndexShift
writerCount := flushSet & messageBatchCountMask
flushQueue := &batch.queue[flushIdx]
spin := shared.NewSpinner(shared.SpinPriorityHigh)
// Wait for remaining writers to finish
for writerCount != atomic.LoadUint32(&flushQueue.doneCount) {
spin.Yield()
}
// Write data and reset buffer asynchronously
go shared.DontPanic(func() {
defer batch.flushing.Done()
messageCount := shared.MinI(int(writerCount), len(flushQueue.messages))
assemble(flushQueue.messages[:messageCount])
atomic.StoreUint32(&flushQueue.doneCount, 0)
batch.Touch()
})
}
func (spool *spoolFile) getFileNumbering() (min int, max int) {
min, max = maxSpoolFileNumber+1, 0
files, _ := ioutil.ReadDir(spool.basePath)
for _, file := range files {
base := filepath.Base(file.Name())
number, _ := shared.Btoi([]byte(base)) // Because we need leading zero support
min = shared.MinI(min, int(number))
max = shared.MaxI(max, int(number))
}
return min, max
}
// Configure initializes this producer with values from a plugin config.
func (prod *Scribe) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
host := conf.GetString("Address", "localhost:1463")
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.windowSize = prod.batchMaxCount
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB
prod.category = conf.GetStreamMap("Category", "")
// Initialize scribe connection
prod.socket, err = thrift.NewTSocket(host)
if err != nil {
Log.Error.Print("Scribe socket error:", err)
return err
}
prod.transport = thrift.NewTFramedTransport(prod.socket)
binProtocol := thrift.NewTBinaryProtocol(prod.transport, false, false)
prod.scribe = scribe.NewScribeClientProtocol(prod.transport, binProtocol, binProtocol)
prod.lastMetricUpdate = time.Now()
prod.counters = make(map[string]*int64)
shared.Metric.New(scribeMetricWindowSize)
shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)
for _, category := range prod.category {
shared.Metric.New(scribeMetricMessages + category)
shared.Metric.New(scribeMetricMessagesSec + category)
prod.counters[category] = new(int64)
}
prod.SetCheckFuseCallback(prod.tryOpenConnection)
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *InfluxDB) Configure(conf core.PluginConfig) error {
if err := prod.ProducerBase.Configure(conf); err != nil {
return err
}
prod.SetStopCallback(prod.close)
version := conf.GetInt("Version", 100)
if conf.GetBool("UseVersion08", false) {
version = 80
}
switch {
case version < 90:
Log.Debug.Print("Using InfluxDB 0.8.x format")
prod.writer = new(influxDBWriter08)
case version == 90:
Log.Debug.Print("Using InfluxDB 0.9.0 format")
prod.writer = new(influxDBWriter09)
default:
Log.Debug.Print("Using InfluxDB 0.9.1+ format")
prod.writer = new(influxDBWriter10)
}
if err := prod.writer.configure(conf, prod); err != nil {
return err
}
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.batch = core.NewMessageBatch(prod.batchMaxCount)
prod.assembly = core.NewWriterAssembly(prod.writer, prod.Drop, prod.GetFormatter())
return nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *File) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetRollCallback(prod.rotateLog)
prod.SetStopCallback(prod.close)
prod.filesByStream = make(map[core.MessageStreamID]*fileState)
prod.files = make(map[string]*fileState)
prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
prod.overwriteFile = conf.GetBool("FileOverwrite", false)
flags, err := strconv.ParseInt(conf.GetString("Permissions", "0664"), 8, 32)
prod.permissions = os.FileMode(flags)
if err != nil {
return err
}
logFile := conf.GetString("File", "/var/log/gollum.log")
prod.wildcardPath = strings.IndexByte(logFile, '*') != -1
prod.fileDir = filepath.Dir(logFile)
prod.fileExt = filepath.Ext(logFile)
prod.fileName = filepath.Base(logFile)
prod.fileName = prod.fileName[:len(prod.fileName)-len(prod.fileExt)]
prod.timestamp = conf.GetString("RotateTimestamp", "2006-01-02_15")
prod.flushTimeout = time.Duration(conf.GetInt("FlushTimeoutSec", 5)) * time.Second
prod.rotate.enabled = conf.GetBool("Rotate", false)
prod.rotate.timeout = time.Duration(conf.GetInt("RotateTimeoutMin", 1440)) * time.Minute
prod.rotate.sizeByte = int64(conf.GetInt("RotateSizeMB", 1024)) << 20
prod.rotate.atHour = -1
prod.rotate.atMinute = -1
prod.rotate.compress = conf.GetBool("Compress", false)
prod.pruneCount = conf.GetInt("RotatePruneCount", 0)
prod.pruneSize = int64(conf.GetInt("RotatePruneTotalSizeMB", 0)) << 20
if prod.pruneSize > 0 && prod.rotate.sizeByte > 0 {
prod.pruneSize -= prod.rotate.sizeByte >> 20
if prod.pruneSize <= 0 {
prod.pruneCount = 1
prod.pruneSize = 0
}
}
rotateAt := conf.GetString("RotateAt", "")
if rotateAt != "" {
parts := strings.Split(rotateAt, ":")
rotateAtHour, _ := strconv.ParseInt(parts[0], 10, 8)
rotateAtMin, _ := strconv.ParseInt(parts[1], 10, 8)
prod.rotate.atHour = int(rotateAtHour)
prod.rotate.atMinute = int(rotateAtMin)
}
return nil
}
func (prod *Scribe) transformMessages(messages []core.Message) {
logBuffer := make([]*scribe.LogEntry, len(messages))
for idx, msg := range messages {
msg.Data, msg.StreamID = prod.Format(msg)
category, exists := prod.category[msg.StreamID]
if !exists {
if category, exists = prod.category[core.WildcardStreamID]; !exists {
category = core.StreamRegistry.GetStreamName(msg.StreamID)
}
shared.Metric.New(scribeMetricMessages + category)
shared.Metric.New(scribeMetricMessagesSec + category)
prod.counters[category] = new(int64)
prod.category[msg.StreamID] = category
}
logBuffer[idx] = &scribe.LogEntry{
Category: category,
Message: string(msg.Data),
}
atomic.AddInt64(prod.counters[category], 1)
}
// Try to send the whole batch.
// If this fails, reduce the number of items send until sending succeeds.
idxStart := 0
for retryCount := 0; retryCount < scribeMaxRetries; retryCount++ {
idxEnd := shared.MinI(len(logBuffer), idxStart+prod.windowSize)
resultCode, err := prod.scribe.Log(logBuffer[idxStart:idxEnd])
if resultCode == scribe.ResultCode_OK {
idxStart = idxEnd
if idxStart < len(logBuffer) {
retryCount = -1 // incremented to 0 after continue
continue // ### continue, data left to send ###
}
// Grow the window on success so we don't get stuck at 1
if prod.windowSize < len(logBuffer) {
prod.windowSize += (len(logBuffer) - prod.windowSize) / 2
}
return // ### return, success ###
}
if err != nil || resultCode != scribe.ResultCode_TRY_LATER {
Log.Error.Printf("Scribe log error %d: %s", resultCode, err.Error())
prod.transport.Close() // reconnect
prod.dropMessages(messages[idxStart:])
return // ### return, failure ###
}
prod.windowSize = shared.MaxI(1, prod.windowSize/2)
shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)
time.Sleep(time.Duration(scribeMaxSleepTimeMs/scribeMaxRetries) * time.Millisecond)
}
Log.Error.Printf("Scribe server seems to be busy")
prod.dropMessages(messages[idxStart:])
}
