func (spool *spoolFile) getFileNumbering() (min int, max int) {
min, max = maxSpoolFileNumber+1, 0
files, _ := ioutil.ReadDir(spool.basePath)
for _, file := range files {
base := filepath.Base(file.Name())
number, _ := shared.Btoi([]byte(base)) // Because we need leading zero support
min = shared.MinI(min, int(number))
max = shared.MaxI(max, int(number))
}
return min, max
}
// ReachedSizeThreshold returns true if the bytes stored in the buffer are
// above or equal to the size given.
// If there is no data this function returns false.
func (batch MessageBatch) ReachedSizeThreshold(size int) bool {
activeIdx := atomic.LoadUint32(&batch.activeSet) >> messageBatchIndexShift
threshold := uint32(shared.MaxI(size, len(batch.queue[activeIdx].messages)))
return batch.queue[activeIdx].doneCount >= threshold
}
func (prod *Scribe) transformMessages(messages []core.Message) {
logBuffer := make([]*scribe.LogEntry, len(messages))
for idx, msg := range messages {
msg.Data, msg.StreamID = prod.Format(msg)
category, exists := prod.category[msg.StreamID]
if !exists {
if category, exists = prod.category[core.WildcardStreamID]; !exists {
category = core.StreamRegistry.GetStreamName(msg.StreamID)
}
shared.Metric.New(scribeMetricMessages + category)
shared.Metric.New(scribeMetricMessagesSec + category)
prod.counters[category] = new(int64)
prod.category[msg.StreamID] = category
}
logBuffer[idx] = &scribe.LogEntry{
Category: category,
Message: string(msg.Data),
}
atomic.AddInt64(prod.counters[category], 1)
}
// Try to send the whole batch.
// If this fails, reduce the number of items send until sending succeeds.
idxStart := 0
for retryCount := 0; retryCount < scribeMaxRetries; retryCount++ {
idxEnd := shared.MinI(len(logBuffer), idxStart+prod.windowSize)
resultCode, err := prod.scribe.Log(logBuffer[idxStart:idxEnd])
if resultCode == scribe.ResultCode_OK {
idxStart = idxEnd
if idxStart < len(logBuffer) {
retryCount = -1 // incremented to 0 after continue
continue // ### continue, data left to send ###
}
// Grow the window on success so we don't get stuck at 1
if prod.windowSize < len(logBuffer) {
prod.windowSize += (len(logBuffer) - prod.windowSize) / 2
}
return // ### return, success ###
}
if err != nil || resultCode != scribe.ResultCode_TRY_LATER {
Log.Error.Printf("Scribe log error %d: %s", resultCode, err.Error())
prod.transport.Close() // reconnect
prod.dropMessages(messages[idxStart:])
return // ### return, failure ###
}
prod.windowSize = shared.MaxI(1, prod.windowSize/2)
shared.Metric.SetI(scribeMetricWindowSize, prod.windowSize)
time.Sleep(time.Duration(scribeMaxSleepTimeMs/scribeMaxRetries) * time.Millisecond)
}
Log.Error.Printf("Scribe server seems to be busy")
prod.dropMessages(messages[idxStart:])
}