Example #1
0
func (spool *spoolFile) getFileNumbering() (min int, max int) {
	min, max = maxSpoolFileNumber+1, 0
	files, _ := ioutil.ReadDir(spool.basePath)
	for _, file := range files {
		base := filepath.Base(file.Name())
		number, _ := shared.Btoi([]byte(base)) // Because we need leading zero support
		min = shared.MinI(min, int(number))
		max = shared.MaxI(max, int(number))
	}
	return min, max
}
Example #2
0
// ReachedSizeThreshold returns true if the bytes stored in the buffer are
// above or equal to the size given.
// If there is no data this function returns false.
func (batch MessageBatch) ReachedSizeThreshold(size int) bool {
	activeIdx := atomic.LoadUint32(&batch.activeSet) >> messageBatchIndexShift
	threshold := uint32(shared.MaxI(size, len(batch.queue[activeIdx].messages)))
	return batch.queue[activeIdx].doneCount >= threshold
}
Example #3
0
func (prod *Scribe) transformMessages(messages []core.Message) {
	logBuffer := make([]*scribe.LogEntry, len(messages))

	for idx, msg := range messages {
		msg.Data, msg.StreamID = prod.Format(msg)

		category, exists := prod.category[msg.StreamID]
		if !exists {
			if category, exists = prod.category[core.WildcardStreamID]; !exists {
				category = core.StreamRegistry.GetStreamName(msg.StreamID)
			}
			shared.Metric.New(scribeMetricMessages + category)
			shared.Metric.New(scribeMetricMessagesSec + category)
			prod.counters[category] = new(int64)
			prod.category[msg.StreamID] = category
		}

		logBuffer[idx] = &scribe.LogEntry{
			Category: category,
			Message:  string(msg.Data),
		}

		atomic.AddInt64(prod.counters[category], 1)
	}

	// Try to send the whole batch.
	// If this fails, reduce the number of items send until sending succeeds.

	idxStart := 0
	for retryCount := 0; retryCount < scribeMaxRetries; retryCount++ {
		idxEnd := shared.MinI(len(logBuffer), idxStart+prod.windowSize)
		resultCode, err := prod.scribe.Log(logBuffer[idxStart:idxEnd])

		if resultCode == scribe.ResultCode_OK {
			idxStart = idxEnd
			if idxStart < len(logBuffer) {
				retryCount = -1 // incremented to 0 after continue
				continue        // ### continue, data left to send ###
			}
			// Grow the window on success so we don't get stuck at 1
			if prod.windowSize < len(logBuffer) {
				prod.windowSize += (len(logBuffer) - prod.windowSize) / 2
			}
			return // ### return, success ###
		}

		if err != nil || resultCode != scribe.ResultCode_TRY_LATER {
			Log.Error.Printf("Scribe log error %d: %s", resultCode, err.Error())
			prod.transport.Close() // reconnect
			prod.dropMessages(messages[idxStart:])
			return // ### return, failure ###
		}

		prod.windowSize = shared.MaxI(1, prod.windowSize/2)
		shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)

		time.Sleep(time.Duration(scribeMaxSleepTimeMs/scribeMaxRetries) * time.Millisecond)
	}

	Log.Error.Printf("Scribe server seems to be busy")
	prod.dropMessages(messages[idxStart:])
}