// connects to an NSQ topic and emits each message into streamtools.
func (b *ToNSQ) Run() {
var writer *nsq.Writer
for {
select {
case ruleI := <-b.inrule:
//rule := ruleI.(map[string]interface{})
topic, err := util.ParseString(ruleI, "Topic")
if err != nil {
b.Error(err)
break
}
nsqdTCPAddrs, err := util.ParseString(ruleI, "NsqdTCPAddrs")
if err != nil {
b.Error(err)
break
}
if writer != nil {
writer.Stop()
}
writer = nsq.NewWriter(nsqdTCPAddrs)
b.topic = topic
b.nsqdTCPAddrs = nsqdTCPAddrs
case msg := <-b.in:
if writer == nil {
continue
}
msgBytes, err := json.Marshal(msg)
if err != nil {
b.Error(err)
}
if len(msgBytes) == 0 {
continue
}
_, _, err = writer.Publish(b.topic, msgBytes)
if err != nil {
b.Error(err)
}
case <-b.quit:
if writer != nil {
writer.Stop()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Topic": b.topic,
"NsqdTCPAddrs": b.nsqdTCPAddrs,
}
}
}
}
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToElasticsearch) Run() {
var err error
var esIndex string
var esType string
conn := elastigo.NewConn()
host := "localhost"
port := "9200"
for {
select {
case ruleI := <-b.inrule:
host, err = util.ParseString(ruleI, "Host")
if err != nil {
b.Error(err)
break
}
port, err = util.ParseString(ruleI, "Port")
if err != nil {
b.Error(err)
break
}
esIndex, err = util.ParseString(ruleI, "Index")
if err != nil {
b.Error(err)
break
}
esType, err = util.ParseString(ruleI, "Type")
if err != nil {
b.Error(err)
break
}
conn.Domain = host
conn.Port = port
case msg := <-b.in:
_, err := conn.Index(esIndex, esType, "", nil, msg)
if err != nil {
b.Error(err)
}
case <-b.quit:
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Index": esIndex,
"Type": esType,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
// This block posts a message to a specified Elasticsearch index with the given type.
func (b *ToElasticsearch) Run() {
var err error
var index string
var indextype string
host := "localhost"
port := "9200"
for {
select {
case msgI := <-b.inrule:
host, err = util.ParseString(msgI, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(msgI, "Port")
if err != nil {
b.Error(err)
continue
}
index, err = util.ParseString(msgI, "Index")
if err != nil {
b.Error(err)
continue
}
indextype, err = util.ParseString(msgI, "IndexType")
if err != nil {
b.Error(err)
continue
}
// Set the Elasticsearch Host/Port to Connect to
api.Domain = host
api.Port = port
case MsgChan := <-b.queryrule:
// deal with a query request
MsgChan <- map[string]interface{}{
"Host": host,
"Port": port,
"Index": index,
"IndexType": indextype,
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
var args map[string]interface{}
_, err := core.Index(index, indextype, "", args, msg)
if err != nil {
b.Error(err)
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToHTTPGetRequest) Run() {
var respPath, msgPath string
var respTree, msgTree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
respPath, err = util.ParseString(ruleI, "RespPath")
respTree, err = util.BuildTokenTree(respPath)
if err != nil {
b.Error(err)
break
}
msgPath, err = util.ParseString(ruleI, "MsgPath")
msgTree, err = util.BuildTokenTree(msgPath)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
return
case msg := <-b.in:
if respTree == nil {
continue
}
if msgTree == nil {
continue
}
cI, err := jee.Eval(respTree, msg)
if err != nil {
b.Error(err)
break
}
c, ok := cI.(blocks.MsgChan)
if !ok {
b.Error(errors.New("response path must point to a channel"))
continue
}
m, err := jee.Eval(msgTree, msg)
if err != nil {
b.Error(err)
break
}
c <- m
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"RespPath": respPath,
"MsgPath": msgPath,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromSQS) Run() {
var err error
for {
select {
case msgI := <-b.inrule:
for k, _ := range b.auth {
b.auth[k], err = util.ParseString(msgI, k)
if err != nil {
b.Error(err)
break
}
}
b.stopListening()
go b.listener()
case <-b.quit:
b.stopListening()
return
case msg := <-b.fromListener:
var outMsg interface{}
err := json.Unmarshal(msg, &outMsg)
if err != nil {
b.Error(err)
continue
}
b.out <- outMsg
case MsgChan := <-b.queryrule:
// deal with a query request
MsgChan <- b.auth
}
}
}
func (b *Count) Run() {
waitTimer := time.NewTimer(100 * time.Millisecond)
pq := &PriorityQueue{}
heap.Init(pq)
window := time.Duration(0)
for {
select {
case <-waitTimer.C:
case rule := <-b.inrule:
tmpDurStr, err := util.ParseString(rule, "Window")
if err != nil {
b.Error(err)
continue
}
tmpWindow, err := time.ParseDuration(tmpDurStr)
if err != nil {
b.Error(err)
continue
}
window = tmpWindow
case <-b.quit:
return
case <-b.in:
empty := make([]byte, 0)
queueMessage := &PQMessage{
val: &empty,
t: time.Now(),
}
heap.Push(pq, queueMessage)
case <-b.inpoll:
b.out <- map[string]interface{}{
"Count": len(*pq),
}
case c := <-b.queryrule:
c <- map[string]interface{}{
"Window": window.String(),
}
case c := <-b.querycount:
c <- map[string]interface{}{
"Count": len(*pq),
}
}
for {
pqMsg, diff := pq.PeekAndShift(time.Now(), window)
if pqMsg == nil {
// either the queue is empty, or it"s not time to emit
if diff == 0 {
// then the queue is empty. Pause for 5 seconds before checking again
diff = time.Duration(500) * time.Millisecond
}
waitTimer.Reset(diff)
break
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Set) Run() {
var path string
set := make(map[interface{}]bool)
var tree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
// quit the block
return
case msg := <-b.add:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
if _, ok := v.(string); !ok {
b.Error(errors.New("can only build sets of strings"))
continue
}
set[v] = true
// deal with inbound data
case msg := <-b.isMember:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
_, ok := set[v]
b.out <- map[string]interface{}{
"isMember": ok,
}
case c := <-b.cardinality:
c <- map[string]interface{}{
"cardinality": len(set),
}
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Unpack) Run() {
var path string
var err error
var tree *jee.TokenTree
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
rule, ok := ruleI.(map[string]interface{})
if !ok {
b.Error(errors.New("cannot assert rule to map"))
}
path, err = util.ParseString(rule, "Path")
if err != nil {
b.Error(err)
continue
}
token, err := jee.Lexer(path)
if err != nil {
b.Error(err)
continue
}
tree, err = jee.Parser(token)
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
if tree == nil {
continue
}
arrInterface, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
continue
}
arr, ok := arrInterface.([]interface{})
if !ok {
b.Error(errors.New("cannot assert " + path + " to array"))
continue
}
for _, out := range arr {
b.out <- out
}
case c := <-b.queryrule:
// deal with a query request
out := map[string]interface{}{
"Path": path,
}
c <- out
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromWebsocket) Run() {
var ws *websocket.Conn
var url string
to, _ := time.ParseDuration("10s")
var handshakeDialer = &websocket.Dialer{
Subprotocols: []string{"p1", "p2"},
HandshakeTimeout: to,
}
listenWS := make(blocks.MsgChan)
wsHeader := http.Header{"Origin": {"http://localhost/"}}
toOut := make(blocks.MsgChan)
toError := make(chan error)
for {
select {
case msg := <-toOut:
b.out <- msg
case ruleI := <-b.inrule:
var err error
// set a parameter of the block
url, err = util.ParseString(ruleI, "url")
if err != nil {
b.Error(err)
continue
}
if ws != nil {
ws.Close()
}
ws, _, err = handshakeDialer.Dial(url, wsHeader)
if err != nil {
b.Error("could not connect to url")
break
}
ws.SetReadDeadline(time.Time{})
h := recvHandler{toOut, toError}
go h.recv(ws, listenWS)
case err := <-toError:
b.Error(err)
case <-b.quit:
// quit the block
return
case o := <-b.queryrule:
o <- map[string]interface{}{
"url": url,
}
case in := <-listenWS:
b.out <- in
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *PackByInterval) Run() {
var batch []interface{}
interval := time.Duration(1) * time.Second
ticker := time.NewTicker(interval)
for {
select {
case <-ticker.C:
b.out <- map[string]interface{}{
"Pack": batch,
}
batch = nil
case ruleI := <-b.inrule:
intervalS, err := util.ParseString(ruleI, "Interval")
if err != nil {
b.Error("error parsing batch size")
break
}
dur, err := time.ParseDuration(intervalS)
if err != nil {
b.Error(err)
break
}
if dur <= 0 {
b.Error("interval must be positive")
break
}
interval = dur
ticker.Stop()
ticker = time.NewTicker(interval)
batch = nil
case <-b.quit:
// quit the block
return
case m := <-b.in:
batch = append(batch, m)
case <-b.clear:
batch = nil
case <-b.flush:
b.out <- map[string]interface{}{
"Pack": batch,
}
batch = nil
case r := <-b.queryrule:
r <- map[string]interface{}{
"Interval": interval.String(),
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromFile) Run() {
var file *os.File
var filename string
for {
select {
case msgI := <-b.inrule:
// set a parameter of the block
filename, err := util.ParseString(msgI, "Filename")
if err != nil {
b.Error(err)
continue
}
file, err := os.Open(filename)
if err != nil {
b.Error(err)
continue
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
var outMsg interface{}
lineBytes := scanner.Bytes()
err := json.Unmarshal(lineBytes, &outMsg)
// if the json parsing fails, store data unparsed as "data"
if err != nil {
outMsg = map[string]interface{}{
"data": lineBytes,
}
}
b.out <- outMsg
}
if err := scanner.Err(); err != nil {
b.Error(err)
continue
}
case <-b.quit:
// quit the block
if file != nil {
file.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Filename": filename,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
// This block posts a message to a specified Elasticsearch index with the given type.
func (b *ToElasticsearch) Run() {
for {
select {
case msgI := <-b.inrule:
host, _ := util.ParseString(msgI, "Host")
port, _ := util.ParseString(msgI, "Port")
index, _ := util.ParseString(msgI, "Index")
indextype, _ := util.ParseString(msgI, "IndexType")
// Set the Elasticsearch Host/Port to Connect to
api.Domain = host
api.Port = port
b.host = host
b.port = port
b.index = index
b.indextype = indextype
case respChan := <-b.queryrule:
// deal with a query request
respChan <- map[string]interface{}{
"Host": b.host,
"Port": b.port,
"Index": b.index,
"IndexType": b.indextype,
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
var args map[string]interface{}
_, err := core.Index(b.index, b.indextype, "", args, msg)
if err != nil {
b.Error(err)
}
}
}
}
// buildEmail will attempt to pull the email's properties from the expected paths and
// put the email body together.
func (e *ToEmail) buildEmail(msg interface{}) (from, to string, email []byte, err error) {
from, err = util.ParseString(msg, e.fromPath)
if err != nil {
return
}
to, err = util.ParseString(msg, e.toPath)
if err != nil {
return
}
var subject string
subject, err = util.ParseString(msg, e.subjectPath)
if err != nil {
return
}
var body string
body, err = util.ParseString(msg, e.msgPath)
if err != nil {
return
}
email = []byte(fmt.Sprintf(emailTmpl, from, to, subject, body))
return
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *DeDupe) Run() {
var path string
set := make(map[interface{}]bool)
var tree *jee.TokenTree
var err error
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
// quit the block
return
// deal with inbound data
case msg := <-b.in:
if tree == nil {
continue
}
v, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
if _, ok := v.(string); !ok {
b.Error(errors.New("can only dedupe sets of strings"))
continue
}
_, ok := set[v]
// emit the incoming message if it isn't found in the set
if !ok {
b.out <- msg
set[v] = true // and add it to the set
}
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
}
}
}
}
// parseEmailInRules will expect a payload from the inrules channel and
// attempt to pull and set the block's to, from and subject paths from it.
func (e *ToEmail) parseEmailRules(msgI interface{}) error {
var err error
e.toPath, err = util.ParseRequiredString(msgI, "ToPath")
if err != nil {
return err
}
e.fromPath, err = util.ParseRequiredString(msgI, "FromPath")
if err != nil {
return err
}
e.subjectPath, err = util.ParseString(msgI, "SubjectPath")
if err != nil {
return err
}
e.msgPath, err = util.ParseString(msgI, "MessagePath")
if err != nil {
return err
}
return nil
}
// parseAuthInRules will expect a payload from the inrules channel and
// attempt to pull the SMTP auth credentials out it. If successful, this
// will also create and set the block's auth.
func (e *ToEmail) parseAuthRules(msgI interface{}) error {
var err error
e.host, err = util.ParseRequiredString(msgI, "Host")
if err != nil {
return err
}
e.port, err = util.ParseInt(msgI, "Port")
if err != nil {
return err
}
e.username, err = util.ParseString(msgI, "Username")
if err != nil {
return err
}
e.password, err = util.ParseString(msgI, "Password")
if err != nil {
return err
}
return nil
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *toWebsocket) Run() {
var addr string
var err error
go h.run()
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
rule, ok := ruleI.(map[string]interface{})
if !ok {
b.Error(errors.New("couldn't assert rule to map"))
}
addr, err = util.ParseString(rule, "port")
if err != nil {
b.Error(err)
}
http.HandleFunc("/ws", serveWs)
go http.ListenAndServe(":"+addr, nil)
if err != nil {
b.Error(err)
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
if addr == "" {
continue
}
out, err := json.Marshal(msg)
if err != nil {
b.Error(err)
}
h.broadcast <- out
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"port": addr,
}
}
}
log.Println("done")
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToFile) Run() {
var err error
var file *os.File
var filename string
for {
select {
case msgI := <-b.inrule:
filename, err = util.ParseString(msgI, "Filename")
if err != nil {
b.Error(err)
}
file, err = os.Create(filename)
if err != nil {
b.Error(err)
}
case <-b.quit:
// quit the block
if file != nil {
file.Close()
}
return
case msg := <-b.in:
// deal with inbound data
writer := bufio.NewWriter(file)
msgStr, err := json.Marshal(msg)
if err != nil {
b.Error(err)
continue
}
fmt.Fprintln(writer, string(msgStr))
writer.Flush()
case MsgChan := <-b.queryrule:
// deal with a query request
MsgChan <- map[string]interface{}{
"Filename": filename,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Ticker) Run() {
interval := time.Duration(1) * time.Second
ticker := time.NewTicker(interval)
for {
select {
case tick := <-ticker.C:
b.out <- map[string]interface{}{
"tick": tick.String(),
}
case ruleI := <-b.inrule:
// set a parameter of the block
intervalS, err := util.ParseString(ruleI, "Interval")
if err != nil {
b.Error("bad input")
break
}
dur, err := time.ParseDuration(intervalS)
if err != nil {
b.Error(err)
break
}
if dur <= 0 {
b.Error("interval must be positive")
break
}
interval = dur
ticker.Stop()
ticker = time.NewTicker(interval)
case <-b.quit:
return
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Interval": interval.String(),
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToFile) Run() {
for {
select {
case msgI := <-b.inrule:
// set a parameter of the block
filename, _ := util.ParseString(msgI, "Filename")
fo, err := os.Create(filename)
if err != nil {
b.Error(err)
}
// set the new file
b.file = fo
// record the filename
b.filename = filename
case <-b.quit:
// quit the block
if b.file != nil {
b.file.Close()
}
return
case msg := <-b.in:
// deal with inbound data
w := bufio.NewWriter(b.file)
msgStr, err := json.Marshal(msg)
if err != nil {
b.Error(err)
continue
}
fmt.Fprintln(w, string(msgStr))
w.Flush()
case respChan := <-b.queryrule:
// deal with a query request
respChan <- map[string]interface{}{
"Filename": b.filename,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ParseCSV) Run() {
var tree *jee.TokenTree
var path string
var err error
var headers []string
var csvReader *csv.Reader
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
if err != nil {
b.Error(err)
continue
}
token, err := jee.Lexer(path)
if err != nil {
b.Error(err)
continue
}
tree, err = jee.Parser(token)
if err != nil {
b.Error(err)
continue
}
headers, err = util.ParseArrayString(ruleI, "Headers")
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
// deal with inbound data
if tree == nil {
continue
}
var data string
dataI, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
continue
}
switch value := dataI.(type) {
case []byte:
data = string(value[:])
case string:
data = value
default:
b.Error("data should be a string or a []byte")
continue
}
csvReader = csv.NewReader(strings.NewReader(data))
csvReader.TrimLeadingSpace = true
// allow records to have variable numbers of fields
csvReader.FieldsPerRecord = -1
case <-b.inpoll:
if csvReader == nil {
b.Error("this block needs data to be pollable")
break
}
record, err := csvReader.Read()
if err != nil && err != io.EOF {
b.Error(err)
continue
}
row := make(map[string]interface{})
for fieldIndex, field := range record {
if fieldIndex >= len(headers) {
row[strconv.Itoa(fieldIndex)] = field
} else {
header := headers[fieldIndex]
row[header] = field
}
}
b.out <- row
case MsgChan := <-b.queryrule:
// deal with a query request
MsgChan <- map[string]interface{}{
"Path": path,
"Headers": headers,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToBeanstalkd) Run() {
var conn *lentil.Beanstalkd
var tube = "default"
var ttr = 0
var host = ""
var err error
for {
select {
case msgI := <-b.inrule:
// set hostname for beanstalkd server
host, err = util.ParseString(msgI, "Host")
if err != nil {
b.Error(err.Error())
continue
}
// set tube name
tube, err = util.ParseString(msgI, "Tube")
if err != nil {
b.Error(errors.New("Could not parse tube name, setting to 'default'"))
tube = "default"
}
// set time to reserve
ttr, err = util.ParseInt(msgI, "TTR")
if err != nil || ttr < 0 {
b.Error(errors.New("Error parsing TTR. Setting TTR to 0"))
ttr = 0
}
// create beanstalkd connection
conn, err = lentil.Dial(host)
if err != nil {
// swallowing a panic from lentil here - streamtools must not die
b.Error(errors.New("Could not initiate connection with beanstalkd server"))
continue
}
// use the specified tube
conn.Use(tube)
case <-b.quit:
// close connection to beanstalkd and quit
if conn != nil {
conn.Quit()
}
return
case msg := <-b.in:
// deal with inbound data
msgStr, err := json.Marshal(msg)
if err != nil {
b.Error(err)
continue
}
if conn != nil {
_, err := conn.Put(0, 0, ttr, msgStr)
if err != nil {
b.Error(err.Error())
}
} else {
b.Error(errors.New("Beanstalkd connection not initated or lost. Please check your beanstalkd server or block settings."))
}
case respChan := <-b.queryrule:
// deal with a query request
respChan <- map[string]interface{}{
"Host": host,
"Tube": tube,
"TTR": ttr,
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Sync) Run() {
var lagString, path string
var tree *jee.TokenTree
lag := time.Duration(0)
emitTick := time.NewTimer(500 * time.Millisecond)
pq := &PriorityQueue{}
heap.Init(pq)
for {
select {
case <-emitTick.C:
case ruleI := <-b.inrule:
// set a parameter of the block
lagString, err := util.ParseString(ruleI, "Lag")
if err != nil {
b.Error(err)
break
}
lag, err = time.ParseDuration(lagString)
if err != nil {
b.Error(err)
continue
}
path, err = util.ParseString(ruleI, "Path")
if err != nil {
b.Error(err)
break
}
// build the parser for the model
token, err := jee.Lexer(path)
if err != nil {
b.Error(err)
continue
}
tree, err = jee.Parser(token)
if err != nil {
b.Error(err)
continue
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
// deal with inbound data
if tree == nil {
break
}
tI, err := jee.Eval(tree, interface{}(msg))
if err != nil {
b.Error(err)
}
t, ok := tI.(float64)
if !ok {
b.Error(errors.New("couldn't convert time value to float64"))
continue
}
ms := time.Unix(0, int64(t*1000000))
queueMessage := &PQMessage{
val: msg,
t: ms,
}
heap.Push(pq, queueMessage)
case respChan := <-b.queryrule:
// deal with a query request
respChan <- map[string]interface{}{
"Lag": lagString,
"Path": path,
}
}
now := time.Now()
for {
item, diff := pq.PeekAndShift(now, lag)
if item == nil {
// then the queue is empty. Pause for 5 seconds before checking again
if diff == 0 {
diff = time.Duration(500) * time.Millisecond
}
emitTick.Reset(diff)
break
}
b.out <- item.(*PQMessage).val.(map[string]interface{})
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *MovingAverage) Run() {
var tree *jee.TokenTree
var path, windowString string
var err error
window := time.Duration(0)
waitTimer := time.NewTimer(100 * time.Millisecond)
pq := &PriorityQueue{}
heap.Init(pq)
for {
select {
case ruleI := <-b.inrule:
// set a parameter of the block
path, err = util.ParseString(ruleI, "Path")
if err != nil {
b.Error(err)
}
tree, err = util.BuildTokenTree(path)
if err != nil {
b.Error(err)
break
}
windowString, err = util.ParseString(ruleI, "Window")
if err != nil {
b.Error(err)
}
window, err = time.ParseDuration(windowString)
if err != nil {
b.Error(err)
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
// deal with inbound data
if tree == nil {
break
}
val, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
// TODO make this a type swtich and convert anything we can to a
// float
val, ok := val.(float64)
if !ok {
b.Error(errors.New("trying to put a non-float into the moving average"))
continue
}
queueMessage := &PQMessage{
val: val,
t: time.Now(),
}
heap.Push(pq, queueMessage)
case <-b.inpoll:
// deal with a poll request
outMsg := map[string]interface{}{
"Average": pqAverage(pq),
}
b.out <- outMsg
case c := <-b.queryavg:
outMsg := map[string]interface{}{
"Average": pqAverage(pq),
}
c <- outMsg
case c := <-b.queryrule:
// deal with a query request
c <- map[string]interface{}{
"Path": path,
"Window": windowString,
}
case <-waitTimer.C:
}
for {
pqMsg, diff := pq.PeekAndShift(time.Now(), window)
if pqMsg == nil {
// either the queue is empty, or it's not time to emit
if diff == 0 {
// then the queue is empty. Pause for 5 seconds before checking again
diff = time.Duration(500) * time.Millisecond
}
waitTimer.Reset(diff)
break
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Redis) Run() {
var server = "localhost:6379"
var password string
var command string
var arguments = make([]string, 0, 10)
var argumentTrees []*jee.TokenTree
var err error
var pool *redis.Pool
for {
select {
case ruleI := <-b.inrule:
server, err = util.ParseString(ruleI, "Server")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(ruleI, "Password")
if err != nil {
b.Error(err)
continue
}
command, err = util.ParseString(ruleI, "Command")
if err != nil {
b.Error(err)
continue
}
if util.KeyExists(ruleI, "Arguments") {
arguments, err = util.ParseArrayString(ruleI, "Arguments")
if err != nil {
b.Error(err)
continue
}
}
if len(arguments) > 0 {
argumentTrees = make([]*jee.TokenTree, len(arguments))
for i, path := range arguments {
token, err := jee.Lexer(path)
if err != nil {
b.Error(err)
continue
}
tree, err := jee.Parser(token)
if err != nil {
b.Error(err)
continue
}
argumentTrees[i] = tree
}
}
pool = newPool(server, password)
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"Server": server,
"Password": password,
"Command": command,
"Arguments": arguments,
}
case <-b.quit:
// quit the block
return
case msg := <-b.in:
if pool == nil {
b.Error("not connected to redis")
break
}
conn := pool.Get()
args := make([]interface{}, len(argumentTrees))
for i, tree := range argumentTrees {
argument, err := jee.Eval(tree, msg)
if err != nil {
b.Error(err)
break
}
args[i] = argument
}
// commands like 'KEYS *' or 'SET NUMBERS 1'
reply, err := conn.Do(command, args...)
if err != nil {
b.Error(err)
break
}
conn.Close()
nicerReply, err := formatReply(reply)
out := map[string]interface{}{
"response": nicerReply,
}
b.out <- out
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Javascript) Run() {
messageIn := "input"
messageOut := "output"
script := `output = input`
vm := otto.New()
program, _ := vm.Compile("javascript", script)
for {
select {
case ruleI := <-b.inrule:
tmpMin, err := util.ParseString(ruleI, "MessageIn")
if err != nil {
b.Error(err)
break
}
tmpMout, err := util.ParseString(ruleI, "MessageOut")
if err != nil {
b.Error(err)
break
}
tmpScript, err := util.ParseString(ruleI, "Script")
if err != nil {
b.Error(err)
break
}
tmpProgram, err := vm.Compile("javascript", tmpScript)
if err != nil {
b.Error(err)
break
}
messageIn = tmpMin
messageOut = tmpMout
script = tmpScript
program = tmpProgram
case <-b.quit:
// quit the block
return
case m := <-b.in:
if program == nil {
break
}
err := vm.Set(messageOut, map[string]interface{}{})
if err != nil {
b.Error(err)
break
}
err = vm.Set(messageIn, m)
if err != nil {
b.Error(err)
break
}
_, err = vm.Run(program)
if err != nil {
b.Error(err)
break
}
g, err := vm.Get(messageOut)
if err != nil {
b.Error(err)
break
}
o, err := g.Export()
if err != nil {
b.Error(err)
break
}
b.out <- o
case c := <-b.queryrule:
c <- map[string]interface{}{
"MessageIn": messageIn,
"MessageOut": messageOut,
"Script": script,
}
}
}
}
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
var err error
var conn *amqp.Connection
var amqp_chan *amqp.Channel
toOut := make(blocks.MsgChan)
toError := make(chan error)
host := "localhost"
port := "5672"
username := "******"
password := "******"
routingkey := "#"
exchange := "amq.topic"
exchange_type := "topic"
for {
select {
case msg := <-toOut:
b.out <- msg
case err := <-toError:
b.Error(err)
case ruleI := <-b.inrule:
rule := ruleI.(map[string]interface{})
routingkey, err = util.ParseString(rule, "RoutingKey")
if err != nil {
b.Error(err)
continue
}
exchange, err = util.ParseString(rule, "Exchange")
if err != nil {
b.Error(err)
continue
}
exchange_type, err = util.ParseString(rule, "ExchangeType")
if err != nil {
b.Error(err)
continue
}
host, err = util.ParseString(rule, "Host")
if err != nil {
b.Error(err)
continue
}
port, err = util.ParseString(rule, "Port")
if err != nil {
b.Error(err)
continue
}
username, err = util.ParseString(rule, "Username")
if err != nil {
b.Error(err)
continue
}
password, err = util.ParseString(rule, "Password")
if err != nil {
b.Error(err)
continue
}
conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
if err != nil {
b.Error(err)
continue
}
amqp_chan, err = conn.Channel()
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.ExchangeDeclare(
exchange, // name
exchange_type, // type
true, // durable
false, // auto-deleted
false, // internal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
queue, err := amqp_chan.QueueDeclare(
"", // name
false, // durable
true, // delete when unused
false, // exclusive
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
err = amqp_chan.QueueBind(
queue.Name, // queue name
routingkey, // routing key
exchange, // exchange
false,
nil,
)
if err != nil {
b.Error(err)
continue
}
deliveries, err := amqp_chan.Consume(
queue.Name, // name
"", // consumerTag
true, // noAck
false, // exclusive
false, // noLocal
false, // noWait
nil, // arguments
)
if err != nil {
b.Error(err)
continue
}
h := readWriteAMQPHandler{toOut, toError}
go h.handle(deliveries)
case <-b.quit:
if amqp_chan != nil {
amqp_chan.Close()
}
if conn != nil {
conn.Close()
}
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Host": host,
"Port": port,
"Username": username,
"Password": password,
"Exchange": exchange,
"ExchangeType": exchange_type,
"RoutingKey": routingkey,
}
}
}
}
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToNSQMulti) Run() {
var err error
var nsqdTCPAddrs string
var topic string
var writer *nsq.Producer
var batch [][]byte
interval := time.Duration(1 * time.Second)
maxBatch := 100
conf := nsq.NewConfig()
dump := time.NewTicker(interval)
for {
select {
case <-dump.C:
if writer == nil || len(batch) == 0 {
break
}
err = writer.MultiPublish(topic, batch)
if err != nil {
b.Error(err.Error())
}
batch = nil
case ruleI := <-b.inrule:
//rule := ruleI.(map[string]interface{})
topic, err = util.ParseString(ruleI, "Topic")
if err != nil {
b.Error(err)
break
}
nsqdTCPAddrs, err = util.ParseString(ruleI, "NsqdTCPAddrs")
if err != nil {
b.Error(err)
break
}
intervalS, err := util.ParseString(ruleI, "Interval")
if err != nil {
b.Error("bad input")
break
}
dur, err := time.ParseDuration(intervalS)
if err != nil {
b.Error(err)
break
}
if dur <= 0 {
b.Error("interval must be positive")
break
}
batchSize, err := util.ParseFloat(ruleI, "MaxBatch")
if err != nil {
b.Error("error parsing batch size")
break
}
if writer != nil {
writer.Stop()
}
maxBatch = int(batchSize)
interval = dur
dump.Stop()
dump = time.NewTicker(interval)
writer, err = nsq.NewProducer(nsqdTCPAddrs, conf)
if err != nil {
b.Error(err)
break
}
topic = topic
nsqdTCPAddrs = nsqdTCPAddrs
case msg := <-b.in:
if writer == nil {
break
}
msgByte, err := json.Marshal(msg)
if err != nil {
b.Error(err)
}
batch = append(batch, msgByte)
if len(batch) > maxBatch {
err := writer.MultiPublish(topic, batch)
if err != nil {
b.Error(err)
break
}
batch = nil
}
case <-b.quit:
if writer != nil {
writer.Stop()
}
dump.Stop()
return
case c := <-b.queryrule:
c <- map[string]interface{}{
"Topic": topic,
"NsqdTCPAddrs": nsqdTCPAddrs,
"MaxBatch": maxBatch,
"Interval": interval.String(),
}
}
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Cache) Run() {
var keyPath, valuePath, ttlString string
var ttl time.Duration
values := make(map[string]item)
ttlQueue := &PriorityQueue{}
var keyTree, valueTree *jee.TokenTree
var err error
emitTick := time.NewTimer(500 * time.Millisecond)
for {
select {
case <-emitTick.C:
case ruleI := <-b.inrule:
keyPath, err = util.ParseString(ruleI, "KeyPath")
keyTree, err = util.BuildTokenTree(keyPath)
if err != nil {
b.Error(err)
break
}
valuePath, err = util.ParseString(ruleI, "ValuePath")
valueTree, err = util.BuildTokenTree(valuePath)
if err != nil {
b.Error(err)
break
}
ttlString, err = util.ParseString(ruleI, "TimeToLive")
if err != nil {
b.Error(err)
break
}
ttl, err = time.ParseDuration(ttlString)
if err != nil {
b.Error(err)
break
}
case <-b.quit:
return
case msg := <-b.lookup:
if keyTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
break
}
k, ok := kI.(string)
if !ok {
b.Error(errors.New("key must be a string"))
continue
}
i, ok := values[k]
var v interface{}
if ok {
v = i.value
now := time.Now()
i.lastSeen = now
queueMessage := &PQMessage{
val: k,
t: now,
}
heap.Push(ttlQueue, queueMessage)
}
b.out <- map[string]interface{}{
"key": k,
"value": v,
}
case msg := <-b.in:
if keyTree == nil {
continue
}
if valueTree == nil {
continue
}
kI, err := jee.Eval(keyTree, msg)
if err != nil {
b.Error(err)
break
}
k, ok := kI.(string)
if !ok {
b.Error(errors.New("key must be a string"))
continue
}
v, err := jee.Eval(valueTree, msg)
if err != nil {
b.Error(err)
break
}
now := time.Now()
values[k] = item{
value: v,
lastSeen: now,
}
queueMessage := &PQMessage{
val: k,
t: now,
}
heap.Push(ttlQueue, queueMessage)
case responseChan := <-b.queryrule:
// deal with a query request
responseChan <- map[string]interface{}{
"KeyPath": keyPath,
"ValuePath": valuePath,
"TimeToLive": ttlString,
}
}
now := time.Now()
for {
itemI, diff := ttlQueue.PeekAndShift(now, ttl)
if itemI == nil {
// then the queue is empty. don't check again for 5s
if diff == 0 {
diff = time.Duration(500) * time.Millisecond
}
emitTick.Reset(diff)
break
}
i := itemI.(*PQMessage)
k := i.val.(string)
if i.t.Equal(values[k].lastSeen) {
delete(values, k)
}
}
}
}
func (b *KullbackLeibler) Run() {
var qtree, ptree *jee.TokenTree
var qpath, ppath string
var err error
for {
select {
case ruleI := <-b.inrule:
qpath, err = util.ParseString(ruleI, "QPath")
if err != nil {
b.Error(err)
}
qtree, err = util.BuildTokenTree(qpath)
ppath, err = util.ParseString(ruleI, "PPath")
if err != nil {
b.Error(err)
}
ptree, err = util.BuildTokenTree(ppath)
case c := <-b.queryrule:
c <- map[string]interface{}{
"QPath": qpath,
"PPath": ppath,
}
case <-b.quit:
return
case msg := <-b.in:
if ptree == nil {
continue
}
if qtree == nil {
continue
}
pI, err := jee.Eval(ptree, msg)
if err != nil {
b.Error(err)
break
}
qI, err := jee.Eval(qtree, msg)
if err != nil {
b.Error(err)
break
}
p, ok := newHistogram(pI)
if !ok {
b.Error(errors.New("p is not a Histogram"))
continue
}
q, ok := newHistogram(qI)
if !ok {
b.Error(errors.New("q is not a Histogram"))
continue
}
q.normalise(p)
p.normalise(q)
kl := 0.0
for k, pi := range p {
kl += math.Log(pi/q[k]) * pi
}
b.out <- map[string]interface{}{
"KL": kl,
}
}
}
}