func (ngram *NGramIndex) splitInput(str string) ([]uint32, error) {
if len(str) == 0 {
return nil, errors.New("empty string")
}
pad := ngram.pad
n := ngram.n
input := pad + str + pad
prevIndexes := make([]int, maxN)
counter := 0
results := make([]uint32, 0)
for index := range input {
counter++
if counter > n {
top := prevIndexes[(counter-n)%len(prevIndexes)]
substr := input[top:index]
hash := mmh3.Hash32([]byte(substr))
results = append(results, hash)
}
prevIndexes[counter%len(prevIndexes)] = index
}
for i := n - 1; i > 1; i-- {
if len(input) >= i {
top := prevIndexes[(len(input)-i)%len(prevIndexes)]
substr := input[top:]
hash := mmh3.Hash32([]byte(substr))
results = append(results, hash)
}
}
return results, nil
}
func processCounters(ch chan Datapoint) {
currentSlots := make([]int64, len(gobatsd.Config.Retentions))
maxSlots := make([]int64, len(gobatsd.Config.Retentions))
for i := range gobatsd.Config.Retentions {
currentSlots[i] = 0
maxSlots[i] = gobatsd.Config.Retentions[i].Interval / heartbeatInterval
}
counters := make([][]map[string]float64, len(gobatsd.Config.Retentions))
for i := range counters {
counters[i] = make([]map[string]float64, maxSlots[i])
for j := range counters[i] {
counters[i][j] = make(map[string]float64)
}
}
for {
select {
case d := <-ch:
//fmt.Printf("Processing counter %v with value %v and timestamp %v \n", d.Name, d.Value, d.Timestamp)
for i := range gobatsd.Config.Retentions {
hashSlot := int64(mmh3.Hash32([]byte(d.Name))) % maxSlots[i]
counters[i][hashSlot][d.Name] += d.Value
}
case <-counterHeartbeat:
for i := range currentSlots {
timestamp := time.Now().Unix() - (time.Now().Unix() % gobatsd.Config.Retentions[i].Interval)
for key, value := range counters[i][currentSlots[i]] {
if value > 0 {
if i == 0 { //Store to redis
observation := AggregateObservation{"counters:" + key, fmt.Sprintf("%d<X>%v", timestamp, value), timestamp, "counters:" + key}
redisAppendChannel <- observation
} else {
observation := AggregateObservation{"counters:" + key + ":" + strconv.FormatInt(gobatsd.Config.Retentions[i].Interval, 10), fmt.Sprintf("%d %v\n", timestamp, value), timestamp, "counters:" + key}
diskAppendChannel <- observation
}
delete(counters[i][currentSlots[i]], key)
}
}
currentSlots[i] += 1
if currentSlots[i] == maxSlots[i] {
currentSlots[i] = 0
}
}
}
}
}
func processTimers(ch chan Datapoint) {
currentSlots := make([]int64, len(gobatsd.Config.Retentions))
maxSlots := make([]int64, len(gobatsd.Config.Retentions))
for i := range gobatsd.Config.Retentions {
currentSlots[i] = 0
maxSlots[i] = gobatsd.Config.Retentions[i].Interval / heartbeatInterval
}
timers := make([][]map[string][]float64, len(gobatsd.Config.Retentions))
for i := range timers {
timers[i] = make([]map[string][]float64, maxSlots[i])
for j := range timers[i] {
timers[i][j] = make(map[string][]float64)
}
}
for {
select {
case d := <-ch:
//fmt.Printf("Processing timer %v with value %v and timestamp %v \n", d.Name, d.Value, d.Timestamp)
for i := range gobatsd.Config.Retentions {
hashSlot := int64(mmh3.Hash32([]byte(d.Name))) % maxSlots[i]
timers[i][hashSlot][d.Name] = append(timers[i][hashSlot][d.Name], d.Value)
}
case <-timerHeartbeat:
for i := range currentSlots {
//fmt.Printf("%v %v %v\n", i, currentSlots[i], timers[i][currentSlots[i]])
timestamp := time.Now().Unix() - (time.Now().Unix() % gobatsd.Config.Retentions[i].Interval)
for key, value := range timers[i][currentSlots[i]] {
if len(value) > 0 {
count := len(value)
min := gobatsd.Min(value)
max := gobatsd.Max(value)
median := gobatsd.Median(value)
mean := gobatsd.Mean(value)
stddev := gobatsd.Stddev(value)
percentile_90 := gobatsd.Percentile(value, 0.9)
percentile_95 := gobatsd.Percentile(value, 0.95)
percentile_99 := gobatsd.Percentile(value, 0.99)
aggregates := fmt.Sprintf("%v/%v/%v/%v/%v/%v/%v/%v/%v", count, min, max, median, mean, stddev, percentile_90, percentile_95, percentile_99)
if i == 0 { //Store to redis
observation := AggregateObservation{"timers:" + key, fmt.Sprintf("%d<X>%v", timestamp, aggregates), timestamp, "timers:" + key}
redisAppendChannel <- observation
} else { // Store to disk
observation := AggregateObservation{"timers:" + key + ":" + strconv.FormatInt(gobatsd.Config.Retentions[i].Interval, 10) + ":2", fmt.Sprintf("%d %v\n", timestamp, aggregates), timestamp, "timers:" + key}
diskAppendChannel <- observation
}
delete(timers[i][currentSlots[i]], key)
}
}
currentSlots[i] += 1
if currentSlots[i] == maxSlots[i] {
currentSlots[i] = 0
}
}
}
}
}
/*
32bit murmur algorithm hash
*/
func murmurHash(str string) uint32 {
return mmh3.Hash32([]byte(str))
}