// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToHTTPGetRequest) Run() {
var respPath, msgPath string
var respTree, msgTree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
respPath, err = util.ParseString(ruleI, "RespPath")
respTree, err = util.BuildTokenTree(respPath)
if err != nil {
b.Error(err)
break
}
msgPath, err = util.ParseString(ruleI, "MsgPath")
msgTree, err = util.BuildTokenTree(msgPath)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
return
case msg := <-b.in:
if respTree == nil {
continue
}
if msgTree == nil {
continue
}
cI, err := jee.Eval(respTree, msg)
if err != nil {
b.Error(err)
break
}
c, ok := cI.(blocks.MsgChan)
if !ok {
b.Error(errors.New("response path must point to a channel"))
continue
}
m, err := jee.Eval(msgTree, msg)
if err != nil {
b.Error(err)
break
}
c <- m
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"RespPath": respPath,
"MsgPath": msgPath,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Set) Run() {
var path string
set := make(map[interface{}]bool)
var tree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
// quit the block
return
case msg := <-b.add:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
if _, ok := v.(string); !ok {
b.Error(errors.New("can only build sets of strings"))
continue
}
set[v] = true
// deal with inbound data
case msg := <-b.isMember:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
_, ok := set[v]
b.out <- map[string]interface{}{
"isMember": ok,
}
case c := <-b.cardinality:
c <- map[string]interface{}{
"cardinality": len(set),
}
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *DeDupe) Run() {
var path string
set := make(map[interface{}]bool)
var tree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
// quit the block
return
// deal with inbound data
case msg := <-b.in:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
if _, ok := v.(string); !ok {
b.Error(errors.New("can only dedupe sets of strings"))
continue
}
_, ok := set[v]
// emit the incoming message if it isn't found in the set
if !ok {
b.out <- msg
set[v] = true // and add it to the set
}
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *MovingAverage) Run() {
var tree *jee.TokenTree
var path, windowString string
var err error
window := time.Duration(0)
waitTimer := time.NewTimer(100 * time.Millisecond)
pq := &PriorityQueue{}
heap.Init(pq)
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
if err != nil {
b.Error(err)
}
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
windowString, err = util.ParseString(ruleI, "Window")
if err != nil {
b.Error(err)
}
window, err = time.ParseDuration(windowString)
if err != nil {
b.Error(err)
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
// deal with inbound data
if tree == nil {
break
}
val, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
// TODO make this a type swtich and convert anything we can to a
// float
val, ok := val.(float64)
if !ok {
b.Error(errors.New("trying to put a non-float into the moving average"))
continue
}
queueMessage := &PQMessage{
val: val,
t: time.Now(),
}
heap.Push(pq, queueMessage)
case <-b.inpoll:
// deal with a poll request
outMsg := map[string]interface{}{
"Average": pqAverage(pq),
}
b.out <- outMsg
case c := <-b.queryavg:
outMsg := map[string]interface{}{
"Average": pqAverage(pq),
}
c <- outMsg
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
"Window": windowString,
}
case <-waitTimer.C:
}
for {
pqMsg, diff := pq.PeekAndShift(time.Now(), window)
if pqMsg == nil {
// either the queue is empty, or it's not time to emit
if diff == 0 {
// then the queue is empty. Pause for 5 seconds before checking again
diff = time.Duration(500) * time.Millisecond
}
waitTimer.Reset(diff)
break
}
}
}
}
func (b *KullbackLeibler) Run() {
var qtree, ptree *jee.TokenTree
var qpath, ppath string
var err error
for {
select {
case ruleI := <-b.inrule:
qpath, err = util.ParseString(ruleI, "QPath")
if err != nil {
b.Error(err)
}
qtree, err = util.BuildTokenTree(qpath)
ppath, err = util.ParseString(ruleI, "PPath")
if err != nil {
b.Error(err)
}
ptree, err = util.BuildTokenTree(ppath)
case c := <-b.queryrule:
c <- map[string]interface{}{
"QPath": qpath,
"PPath": ppath,
}
case <-b.quit:
return
case msg := <-b.in:
if ptree == nil {
continue
}
if qtree == nil {
continue
}
pI, err := jee.Eval(ptree, msg)
if err != nil {
b.Error(err)
break
}
qI, err := jee.Eval(qtree, msg)
if err != nil {
b.Error(err)
break
}
p, ok := newHistogram(pI)
if !ok {
b.Error(errors.New("p is not a Histogram"))
continue
}
q, ok := newHistogram(qI)
if !ok {
b.Error(errors.New("q is not a Histogram"))
continue
}
q.normalise(p)
p.normalise(q)
kl := 0.0
for k, pi := range p {
kl += math.Log(pi/q[k]) * pi
}
b.out <- map[string]interface{}{
"KL": kl,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Cache) Run() {
var keyPath, valuePath, ttlString string
var ttl time.Duration
values := make(map[string]item)
ttlQueue := &PriorityQueue{}
var keyTree, valueTree *jee.TokenTree
var err error
emitTick := time.NewTimer(500 * time.Millisecond)
for {
select {
case <-emitTick.C:
case ruleI := <-b.inrule:
keyPath, err = util.ParseString(ruleI, "KeyPath")
keyTree, err = util.BuildTokenTree(keyPath)
if err != nil {
b.Error(err)
break
}
valuePath, err = util.ParseString(ruleI, "ValuePath")
valueTree, err = util.BuildTokenTree(valuePath)
if err != nil {
b.Error(err)
break
}
ttlString, err = util.ParseString(ruleI, "TimeToLive")
if err != nil {
b.Error(err)
break
}
ttl, err = time.ParseDuration(ttlString)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
return
case msg := <-b.lookup:
if keyTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
break
}
k, ok := kI.(string)
if !ok {
b.Error(errors.New("key must be a string"))
continue
}
i, ok := values[k]
var v interface{}
if ok {
v = i.value
now := time.Now()
i.lastSeen = now
queueMessage := &PQMessage{
val: k,
t: now,
}
heap.Push(ttlQueue, queueMessage)
}
b.out <- map[string]interface{}{
"key": k,
"value": v,
}
case msg := <-b.in:
if keyTree == nil {
continue
}
if valueTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
break
}
k, ok := kI.(string)
if !ok {
b.Error(errors.New("key must be a string"))
continue
}
v, err := jee.Eval(valueTree, msg)
if err != nil {
b.Error(err)
break
}
now := time.Now()
values[k] = item{
value: v,
lastSeen: now,
}
queueMessage := &PQMessage{
val: k,
t: now,
}
heap.Push(ttlQueue, queueMessage)
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"KeyPath": keyPath,
"ValuePath": valuePath,
"TimeToLive": ttlString,
}
}
now := time.Now()
for {
itemI, diff := ttlQueue.PeekAndShift(now, ttl)
if itemI == nil {
// then the queue is empty. don't check again for 5s
if diff == 0 {
diff = time.Duration(500) * time.Millisecond
}
emitTick.Reset(diff)
break
}
i := itemI.(*PQMessage)
k := i.val.(string)
if i.t.Equal(values[k].lastSeen) {
delete(values, k)
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Timeseries) Run() {
var err error
//var path, lagStr string
var path string
var tree *jee.TokenTree
//var lag time.Duration
var data tsData
var numSamples float64
// defaults
numSamples = 1
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
rule, ok := ruleI.(map[string]interface{})
if !ok {
b.Error(errors.New("could not assert rule to map"))
}
path, err = util.ParseString(rule, "Path")
if err != nil {
b.Error(err)
continue
}
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
continue
}
/*
lagStr, err = util.ParseString(rule, "Lag")
if err != nil {
b.Error(err)
continue
}
lag, err = time.ParseDuration(lagStr)
if err != nil {
b.Error(err)
continue
}
*/
numSamples, err = util.ParseFloat(rule, "NumSamples")
if err != nil {
b.Error(err)
continue
}
data = tsData{
Values: make([]tsDataPoint, int(numSamples)),
}
case <-b.quit:
// quit * time.Second the block
return
case msg := <-b.in:
if tree == nil {
continue
}
if data.Values == nil {
continue
}
// deal with inbound data
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
continue
}
var val float64
switch v := v.(type) {
case float32:
val = float64(v)
case int:
val = float64(v)
case float64:
val = v
}
//t := float64(time.Now().Add(-lag).Unix())
t := float64(time.Now().Unix())
d := tsDataPoint{
Timestamp: t,
Value: val,
}
data.Values = append(data.Values[1:], d)
case MsgChan := <-b.queryrule:
// deal with a query request
MsgChan <- map[string]interface{}{
//"Window": lagStr,
"Path": path,
"NumSamples": numSamples,
}
case MsgChan := <-b.querystate:
out := map[string]interface{}{
"timeseries": data,
}
MsgChan <- out
case <-b.inpoll:
outArray := make([]interface{}, len(data.Values))
for i, d := range data.Values {
di := map[string]interface{}{
"timestamp": d.Timestamp,
"value": d.Value,
}
outArray[i] = di
}
out := map[string]interface{}{
"timeseries": outArray,
}
b.out <- out
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Cache) Run() {
var keyPath, valuePath, ttlString string
var ttl time.Duration
cache := make(map[string]item)
ttlQueue := &PriorityQueue{}
var keyTree, valueTree *jee.TokenTree
var err error
emitTick := time.NewTimer(500 * time.Millisecond)
for {
select {
case <-emitTick.C:
case ruleI := <-b.inrule:
keyPath, err = util.ParseString(ruleI, "KeyPath")
keyTree, err = util.BuildTokenTree(keyPath)
if err != nil {
b.Error(err)
break
}
valuePath, err = util.ParseString(ruleI, "ValuePath")
valueTree, err = util.BuildTokenTree(valuePath)
if err != nil {
b.Error(err)
break
}
ttlString, err = util.ParseString(ruleI, "TimeToLive")
if err != nil {
b.Error(err)
break
}
ttl, err = time.ParseDuration(ttlString)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
return
case msg := <-b.lookup:
if keyTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
continue
}
k, ok := kI.(string)
if !ok {
b.Error(err)
continue
}
out, err := extractAndUpdate(k, cache, ttlQueue)
if err != nil {
b.Error(err)
continue
}
b.out <- out
case q := <-b.querylookup:
k, ok := q.Params["key"]
if !ok {
b.Error(errors.New("Must specify a key to lookup"))
}
for _, ki := range k {
out, err := extractAndUpdate(ki, cache, ttlQueue)
if err != nil {
b.Error(err)
continue
}
q.RespChan <- out
}
case responseChan := <-b.keys:
keys := make([]string, len(cache))
i := 0
for key := range cache {
keys[i] = key
i++
}
responseChan <- map[string]interface{}{
"keys": keys,
}
case responseChan := <-b.values:
values := make([]interface{}, len(cache))
i := 0
for _, item := range cache {
values[i] = item
i++
}
responseChan <- map[string]interface{}{
"values": values,
}
case responseChan := <-b.dump:
responseChan <- map[string]interface{}{
"dump": cache,
}
case msg := <-b.in:
if keyTree == nil {
continue
}
if valueTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
break
}
k, ok := kI.(string)
if !ok {
b.Error(errors.New("key must be a string"))
continue
}
v, err := jee.Eval(valueTree, msg)
if err != nil {
b.Error(err)
break
}
now := time.Now()
cache[k] = item{
value: v,
lastSeen: now,
}
queueMessage := &PQMessage{
val: k,
t: now,
}
heap.Push(ttlQueue, queueMessage)
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"KeyPath": keyPath,
"ValuePath": valuePath,
"TimeToLive": ttlString,
}
}
now := time.Now()
for {
itemI, diff := ttlQueue.PeekAndShift(now, ttl)
if itemI == nil {
// then the queue is empty. don't check again for 5s
if diff == 0 {
diff = time.Duration(500) * time.Millisecond
}
emitTick.Reset(diff)
break
}
i := itemI.(*PQMessage)
k := i.val.(string)
if i.t.Equal(cache[k].lastSeen) {
delete(cache, k)
}
}
}
}
func (b *FFT) Run() {
var err error
var path string
var tree *jee.TokenTree
for {
select {
case ruleI := <-b.inrule:
rule, ok := ruleI.(map[string]interface{})
if !ok {
b.Error(errors.New("could not assert rule to map"))
}
path, err = util.ParseString(rule, "Path")
if err != nil {
b.Error(err)
continue
}
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
return
case msg := <-b.in:
if tree == nil {
continue
}
vI, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
continue
}
v, ok := vI.([]interface{})
if !ok {
b.Error(errors.New("could not assert timeseries to an array"))
continue
}
values := make([]tsDataPoint, len(v))
for i, vi := range v {
value, ok := vi.(map[string]interface{})
if !ok {
b.Error(errors.New("could not assert value to map"))
continue
}
tI, ok := value["timestamp"]
if !ok {
b.Error(errors.New("could not find timestamp in value"))
continue
}
t, ok := tI.(float64)
if !ok {
b.Error(errors.New("could not assert timestamp to float"))
continue
}
yI, ok := value["value"]
if !ok {
b.Error(errors.New("could not assert timeseries value to float"))
continue
}
y, ok := yI.(float64)
values[i] = tsDataPoint{
Timestamp: t,
Value: y,
}
}
data := tsData{
Values: values,
}
out := map[string]interface{}{
"fft": buildFFT(data),
}
b.out <- out
case respChan := <-b.queryrule:
respChan <- map[string]interface{}{
"Path": path,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Histogram) Run() {
var tree *jee.TokenTree
var path string
waitTimer := time.NewTimer(100 * time.Millisecond)
window := time.Duration(0)
histogram := map[string]*PriorityQueue{}
emptyByte := make([]byte, 0)
MainLoop:
for {
select {
case ruleI := <-b.inrule:
// window
windowString, err := util.ParseString(ruleI, "Window")
if err != nil {
b.Error(err)
}
window, err = time.ParseDuration(windowString)
if err != nil {
b.Error(err)
}
path, err = util.ParseString(ruleI, "Path")
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
var valueString string
switch v := v.(type) {
default:
b.Error(errors.New("unexpected value type"))
continue MainLoop
case string:
valueString = v
case int:
valueString = strconv.Itoa(v)
case bool:
valueString = strconv.FormatBool(v)
case float64:
valueString = strconv.FormatFloat(v, 'g', -1, 64)
}
if pq, ok := histogram[valueString]; ok {
queueMessage := &PQMessage{
val: &emptyByte,
t: time.Now(),
}
heap.Push(pq, queueMessage)
} else {
pq := &PriorityQueue{}
heap.Init(pq)
histogram[valueString] = pq
queueMessage := &PQMessage{
val: &emptyByte,
t: time.Now(),
}
heap.Push(pq, queueMessage)
}
case <-waitTimer.C:
case <-b.inpoll:
// deal with a poll request
data := buildHistogram(histogram)
b.out <- data
case MsgChan := <-b.queryrule:
// deal with a query request
out := map[string]interface{}{
"Window": window.String(),
"Path": path,
}
MsgChan <- out
case MsgChan := <-b.historule:
data := buildHistogram(histogram)
MsgChan <- data
}
for _, pq := range histogram {
for {
pqMsg, diff := pq.PeekAndShift(time.Now(), window)
if pqMsg == nil {
// either the queue is empty, or it's not time to emit
if diff == 0 {
// then the queue is empty. Pause for 5 seconds before checking again
diff = time.Duration(500) * time.Millisecond
}
waitTimer.Reset(diff)
break
}
}
}
}
}