Beispiel #1
0
// Configure initializes this producer with values from a plugin config.
func (prod *Socket) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)

	prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
	prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
	prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
	prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
	prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB

	prod.acknowledge = shared.Unescape(conf.GetString("Acknowledge", ""))
	prod.ackTimeout = time.Duration(conf.GetInt("AckTimeoutMs", 2000)) * time.Millisecond
	prod.address, prod.protocol = shared.ParseAddress(conf.GetString("Address", ":5880"))

	if prod.protocol != "unix" {
		if prod.acknowledge != "" {
			prod.protocol = "tcp"
		} else {
			prod.protocol = "udp"
		}
	}

	prod.batch = core.NewMessageBatch(prod.batchMaxCount)
	prod.assembly = core.NewWriterAssembly(nil, prod.Drop, prod.GetFormatter())
	prod.assembly.SetValidator(prod.validate)
	prod.assembly.SetErrorHandler(prod.onWriteError)

	prod.SetCheckFuseCallback(prod.tryConnect)
	return nil
}
Beispiel #2
0
// Flush writes the content of the buffer to a given resource and resets the
// internal state, i.e. the buffer is empty after a call to Flush.
// Writing will be done in a separate go routine to be non-blocking.
//
// The validate callback will be called after messages have been successfully
// written to the io.Writer.
// If validate returns false the buffer will not be resetted (automatic retry).
// If validate is nil a return value of true is assumed (buffer reset).
//
// The onError callback will be called if the io.Writer returned an error.
// If onError returns false the buffer will not be resetted (automatic retry).
// If onError is nil a return value of true is assumed (buffer reset).
func (batch *MessageBatch) Flush(assemble AssemblyFunc) {
	if batch.IsEmpty() {
		return // ### return, nothing to do ###
	}

	// Only one flush at a time
	batch.flushing.IncWhenDone()

	// Switch the buffers so writers can go on writing
	flushSet := atomic.SwapUint32(&batch.activeSet, (batch.activeSet&messageBatchIndexMask)^messageBatchIndexMask)

	flushIdx := flushSet >> messageBatchIndexShift
	writerCount := flushSet & messageBatchCountMask
	flushQueue := &batch.queue[flushIdx]
	spin := shared.NewSpinner(shared.SpinPriorityHigh)

	// Wait for remaining writers to finish
	for writerCount != atomic.LoadUint32(&flushQueue.doneCount) {
		spin.Yield()
	}

	// Write data and reset buffer asynchronously
	go shared.DontPanic(func() {
		defer batch.flushing.Done()

		messageCount := shared.MinI(int(writerCount), len(flushQueue.messages))
		assemble(flushQueue.messages[:messageCount])
		atomic.StoreUint32(&flushQueue.doneCount, 0)
		batch.Touch()
	})
}
Beispiel #3
0
func (spool *spoolFile) getFileNumbering() (min int, max int) {
	min, max = maxSpoolFileNumber+1, 0
	files, _ := ioutil.ReadDir(spool.basePath)
	for _, file := range files {
		base := filepath.Base(file.Name())
		number, _ := shared.Btoi([]byte(base)) // Because we need leading zero support
		min = shared.MinI(min, int(number))
		max = shared.MaxI(max, int(number))
	}
	return min, max
}
Beispiel #4
0
// Configure initializes this producer with values from a plugin config.
func (prod *Scribe) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)
	host := conf.GetString("Address", "localhost:1463")

	prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
	prod.windowSize = prod.batchMaxCount
	prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
	prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
	prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
	prod.batch = core.NewMessageBatch(prod.batchMaxCount)

	prod.bufferSizeByte = conf.GetInt("ConnectionBufferSizeKB", 1<<10) << 10 // 1 MB
	prod.category = conf.GetStreamMap("Category", "")

	// Initialize scribe connection

	prod.socket, err = thrift.NewTSocket(host)
	if err != nil {
		Log.Error.Print("Scribe socket error:", err)
		return err
	}

	prod.transport = thrift.NewTFramedTransport(prod.socket)
	binProtocol := thrift.NewTBinaryProtocol(prod.transport, false, false)
	prod.scribe = scribe.NewScribeClientProtocol(prod.transport, binProtocol, binProtocol)
	prod.lastMetricUpdate = time.Now()
	prod.counters = make(map[string]*int64)

	shared.Metric.New(scribeMetricWindowSize)
	shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)

	for _, category := range prod.category {
		shared.Metric.New(scribeMetricMessages + category)
		shared.Metric.New(scribeMetricMessagesSec + category)
		prod.counters[category] = new(int64)
	}

	prod.SetCheckFuseCallback(prod.tryOpenConnection)
	return nil
}
Beispiel #5
0
// Configure initializes this producer with values from a plugin config.
func (prod *InfluxDB) Configure(conf core.PluginConfig) error {
	if err := prod.ProducerBase.Configure(conf); err != nil {
		return err
	}
	prod.SetStopCallback(prod.close)

	version := conf.GetInt("Version", 100)
	if conf.GetBool("UseVersion08", false) {
		version = 80
	}

	switch {
	case version < 90:
		Log.Debug.Print("Using InfluxDB 0.8.x format")
		prod.writer = new(influxDBWriter08)
	case version == 90:
		Log.Debug.Print("Using InfluxDB 0.9.0 format")
		prod.writer = new(influxDBWriter09)
	default:
		Log.Debug.Print("Using InfluxDB 0.9.1+ format")
		prod.writer = new(influxDBWriter10)
	}

	if err := prod.writer.configure(conf, prod); err != nil {
		return err
	}

	prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
	prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
	prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
	prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second

	prod.batch = core.NewMessageBatch(prod.batchMaxCount)
	prod.assembly = core.NewWriterAssembly(prod.writer, prod.Drop, prod.GetFormatter())
	return nil
}
Beispiel #6
0
// Configure initializes this producer with values from a plugin config.
func (prod *File) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}

	prod.SetRollCallback(prod.rotateLog)
	prod.SetStopCallback(prod.close)

	prod.filesByStream = make(map[core.MessageStreamID]*fileState)
	prod.files = make(map[string]*fileState)
	prod.batchMaxCount = conf.GetInt("BatchMaxCount", 8192)
	prod.batchFlushCount = conf.GetInt("BatchFlushCount", prod.batchMaxCount/2)
	prod.batchFlushCount = shared.MinI(prod.batchFlushCount, prod.batchMaxCount)
	prod.batchTimeout = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
	prod.overwriteFile = conf.GetBool("FileOverwrite", false)

	flags, err := strconv.ParseInt(conf.GetString("Permissions", "0664"), 8, 32)
	prod.permissions = os.FileMode(flags)
	if err != nil {
		return err
	}

	logFile := conf.GetString("File", "/var/log/gollum.log")
	prod.wildcardPath = strings.IndexByte(logFile, '*') != -1

	prod.fileDir = filepath.Dir(logFile)
	prod.fileExt = filepath.Ext(logFile)
	prod.fileName = filepath.Base(logFile)
	prod.fileName = prod.fileName[:len(prod.fileName)-len(prod.fileExt)]
	prod.timestamp = conf.GetString("RotateTimestamp", "2006-01-02_15")
	prod.flushTimeout = time.Duration(conf.GetInt("FlushTimeoutSec", 5)) * time.Second

	prod.rotate.enabled = conf.GetBool("Rotate", false)
	prod.rotate.timeout = time.Duration(conf.GetInt("RotateTimeoutMin", 1440)) * time.Minute
	prod.rotate.sizeByte = int64(conf.GetInt("RotateSizeMB", 1024)) << 20
	prod.rotate.atHour = -1
	prod.rotate.atMinute = -1
	prod.rotate.compress = conf.GetBool("Compress", false)

	prod.pruneCount = conf.GetInt("RotatePruneCount", 0)
	prod.pruneSize = int64(conf.GetInt("RotatePruneTotalSizeMB", 0)) << 20

	if prod.pruneSize > 0 && prod.rotate.sizeByte > 0 {
		prod.pruneSize -= prod.rotate.sizeByte >> 20
		if prod.pruneSize <= 0 {
			prod.pruneCount = 1
			prod.pruneSize = 0
		}
	}

	rotateAt := conf.GetString("RotateAt", "")
	if rotateAt != "" {
		parts := strings.Split(rotateAt, ":")
		rotateAtHour, _ := strconv.ParseInt(parts[0], 10, 8)
		rotateAtMin, _ := strconv.ParseInt(parts[1], 10, 8)

		prod.rotate.atHour = int(rotateAtHour)
		prod.rotate.atMinute = int(rotateAtMin)
	}

	return nil
}
Beispiel #7
0
func (prod *Scribe) transformMessages(messages []core.Message) {
	logBuffer := make([]*scribe.LogEntry, len(messages))

	for idx, msg := range messages {
		msg.Data, msg.StreamID = prod.Format(msg)

		category, exists := prod.category[msg.StreamID]
		if !exists {
			if category, exists = prod.category[core.WildcardStreamID]; !exists {
				category = core.StreamRegistry.GetStreamName(msg.StreamID)
			}
			shared.Metric.New(scribeMetricMessages + category)
			shared.Metric.New(scribeMetricMessagesSec + category)
			prod.counters[category] = new(int64)
			prod.category[msg.StreamID] = category
		}

		logBuffer[idx] = &scribe.LogEntry{
			Category: category,
			Message:  string(msg.Data),
		}

		atomic.AddInt64(prod.counters[category], 1)
	}

	// Try to send the whole batch.
	// If this fails, reduce the number of items send until sending succeeds.

	idxStart := 0
	for retryCount := 0; retryCount < scribeMaxRetries; retryCount++ {
		idxEnd := shared.MinI(len(logBuffer), idxStart+prod.windowSize)
		resultCode, err := prod.scribe.Log(logBuffer[idxStart:idxEnd])

		if resultCode == scribe.ResultCode_OK {
			idxStart = idxEnd
			if idxStart < len(logBuffer) {
				retryCount = -1 // incremented to 0 after continue
				continue        // ### continue, data left to send ###
			}
			// Grow the window on success so we don't get stuck at 1
			if prod.windowSize < len(logBuffer) {
				prod.windowSize += (len(logBuffer) - prod.windowSize) / 2
			}
			return // ### return, success ###
		}

		if err != nil || resultCode != scribe.ResultCode_TRY_LATER {
			Log.Error.Printf("Scribe log error %d: %s", resultCode, err.Error())
			prod.transport.Close() // reconnect
			prod.dropMessages(messages[idxStart:])
			return // ### return, failure ###
		}

		prod.windowSize = shared.MaxI(1, prod.windowSize/2)
		shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)

		time.Sleep(time.Duration(scribeMaxSleepTimeMs/scribeMaxRetries) * time.Millisecond)
	}

	Log.Error.Printf("Scribe server seems to be busy")
	prod.dropMessages(messages[idxStart:])
}