// connects to an NSQ topic and emits each message into streamtools. func (b *ToNSQ) Run() { var writer *nsq.Writer for { select { case ruleI := <-b.inrule: //rule := ruleI.(map[string]interface{}) topic, err := util.ParseString(ruleI, "Topic") if err != nil { b.Error(err) break } nsqdTCPAddrs, err := util.ParseString(ruleI, "NsqdTCPAddrs") if err != nil { b.Error(err) break } if writer != nil { writer.Stop() } writer = nsq.NewWriter(nsqdTCPAddrs) b.topic = topic b.nsqdTCPAddrs = nsqdTCPAddrs case msg := <-b.in: if writer == nil { continue } msgBytes, err := json.Marshal(msg) if err != nil { b.Error(err) } if len(msgBytes) == 0 { continue } _, _, err = writer.Publish(b.topic, msgBytes) if err != nil { b.Error(err) } case <-b.quit: if writer != nil { writer.Stop() } return case c := <-b.queryrule: c <- map[string]interface{}{ "Topic": b.topic, "NsqdTCPAddrs": b.nsqdTCPAddrs, } } } }
// connects to an NSQ topic and emits each message into streamtools. func (b *ToElasticsearch) Run() { var err error var esIndex string var esType string conn := elastigo.NewConn() host := "localhost" port := "9200" for { select { case ruleI := <-b.inrule: host, err = util.ParseString(ruleI, "Host") if err != nil { b.Error(err) break } port, err = util.ParseString(ruleI, "Port") if err != nil { b.Error(err) break } esIndex, err = util.ParseString(ruleI, "Index") if err != nil { b.Error(err) break } esType, err = util.ParseString(ruleI, "Type") if err != nil { b.Error(err) break } conn.Domain = host conn.Port = port case msg := <-b.in: _, err := conn.Index(esIndex, esType, "", nil, msg) if err != nil { b.Error(err) } case <-b.quit: return case c := <-b.queryrule: c <- map[string]interface{}{ "Host": host, "Port": port, "Index": esIndex, "Type": esType, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. // This block posts a message to a specified Elasticsearch index with the given type. func (b *ToElasticsearch) Run() { var err error var index string var indextype string host := "localhost" port := "9200" for { select { case msgI := <-b.inrule: host, err = util.ParseString(msgI, "Host") if err != nil { b.Error(err) continue } port, err = util.ParseString(msgI, "Port") if err != nil { b.Error(err) continue } index, err = util.ParseString(msgI, "Index") if err != nil { b.Error(err) continue } indextype, err = util.ParseString(msgI, "IndexType") if err != nil { b.Error(err) continue } // Set the Elasticsearch Host/Port to Connect to api.Domain = host api.Port = port case MsgChan := <-b.queryrule: // deal with a query request MsgChan <- map[string]interface{}{ "Host": host, "Port": port, "Index": index, "IndexType": indextype, } case <-b.quit: // quit the block return case msg := <-b.in: var args map[string]interface{} _, err := core.Index(index, indextype, "", args, msg) if err != nil { b.Error(err) } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *ToHTTPGetRequest) Run() { var respPath, msgPath string var respTree, msgTree *jee.TokenTree var err error for { select { case ruleI := <-b.inrule: respPath, err = util.ParseString(ruleI, "RespPath") respTree, err = util.BuildTokenTree(respPath) if err != nil { b.Error(err) break } msgPath, err = util.ParseString(ruleI, "MsgPath") msgTree, err = util.BuildTokenTree(msgPath) if err != nil { b.Error(err) break } case <-b.quit: return case msg := <-b.in: if respTree == nil { continue } if msgTree == nil { continue } cI, err := jee.Eval(respTree, msg) if err != nil { b.Error(err) break } c, ok := cI.(blocks.MsgChan) if !ok { b.Error(errors.New("response path must point to a channel")) continue } m, err := jee.Eval(msgTree, msg) if err != nil { b.Error(err) break } c <- m case responseChan := <-b.queryrule: // deal with a query request responseChan <- map[string]interface{}{ "RespPath": respPath, "MsgPath": msgPath, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *FromSQS) Run() { var err error for { select { case msgI := <-b.inrule: for k, _ := range b.auth { b.auth[k], err = util.ParseString(msgI, k) if err != nil { b.Error(err) break } } b.stopListening() go b.listener() case <-b.quit: b.stopListening() return case msg := <-b.fromListener: var outMsg interface{} err := json.Unmarshal(msg, &outMsg) if err != nil { b.Error(err) continue } b.out <- outMsg case MsgChan := <-b.queryrule: // deal with a query request MsgChan <- b.auth } } }
func (b *Count) Run() { waitTimer := time.NewTimer(100 * time.Millisecond) pq := &PriorityQueue{} heap.Init(pq) window := time.Duration(0) for { select { case <-waitTimer.C: case rule := <-b.inrule: tmpDurStr, err := util.ParseString(rule, "Window") if err != nil { b.Error(err) continue } tmpWindow, err := time.ParseDuration(tmpDurStr) if err != nil { b.Error(err) continue } window = tmpWindow case <-b.quit: return case <-b.in: empty := make([]byte, 0) queueMessage := &PQMessage{ val: &empty, t: time.Now(), } heap.Push(pq, queueMessage) case <-b.inpoll: b.out <- map[string]interface{}{ "Count": len(*pq), } case c := <-b.queryrule: c <- map[string]interface{}{ "Window": window.String(), } case c := <-b.querycount: c <- map[string]interface{}{ "Count": len(*pq), } } for { pqMsg, diff := pq.PeekAndShift(time.Now(), window) if pqMsg == nil { // either the queue is empty, or it"s not time to emit if diff == 0 { // then the queue is empty. Pause for 5 seconds before checking again diff = time.Duration(500) * time.Millisecond } waitTimer.Reset(diff) break } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Set) Run() { var path string set := make(map[interface{}]bool) var tree *jee.TokenTree var err error for { select { case ruleI := <-b.inrule: // set a parameter of the block path, err = util.ParseString(ruleI, "Path") tree, err = util.BuildTokenTree(path) if err != nil { b.Error(err) break } case <-b.quit: // quit the block return case msg := <-b.add: if tree == nil { continue } v, err := jee.Eval(tree, msg) if err != nil { b.Error(err) break } if _, ok := v.(string); !ok { b.Error(errors.New("can only build sets of strings")) continue } set[v] = true // deal with inbound data case msg := <-b.isMember: if tree == nil { continue } v, err := jee.Eval(tree, msg) if err != nil { b.Error(err) break } _, ok := set[v] b.out <- map[string]interface{}{ "isMember": ok, } case c := <-b.cardinality: c <- map[string]interface{}{ "cardinality": len(set), } case c := <-b.queryrule: // deal with a query request c <- map[string]interface{}{ "Path": path, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Unpack) Run() { var path string var err error var tree *jee.TokenTree for { select { case ruleI := <-b.inrule: // set a parameter of the block rule, ok := ruleI.(map[string]interface{}) if !ok { b.Error(errors.New("cannot assert rule to map")) } path, err = util.ParseString(rule, "Path") if err != nil { b.Error(err) continue } token, err := jee.Lexer(path) if err != nil { b.Error(err) continue } tree, err = jee.Parser(token) if err != nil { b.Error(err) continue } case <-b.quit: // quit the block return case msg := <-b.in: if tree == nil { continue } arrInterface, err := jee.Eval(tree, msg) if err != nil { b.Error(err) continue } arr, ok := arrInterface.([]interface{}) if !ok { b.Error(errors.New("cannot assert " + path + " to array")) continue } for _, out := range arr { b.out <- out } case c := <-b.queryrule: // deal with a query request out := map[string]interface{}{ "Path": path, } c <- out } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *FromWebsocket) Run() { var ws *websocket.Conn var url string to, _ := time.ParseDuration("10s") var handshakeDialer = &websocket.Dialer{ Subprotocols: []string{"p1", "p2"}, HandshakeTimeout: to, } listenWS := make(blocks.MsgChan) wsHeader := http.Header{"Origin": {"http://localhost/"}} toOut := make(blocks.MsgChan) toError := make(chan error) for { select { case msg := <-toOut: b.out <- msg case ruleI := <-b.inrule: var err error // set a parameter of the block url, err = util.ParseString(ruleI, "url") if err != nil { b.Error(err) continue } if ws != nil { ws.Close() } ws, _, err = handshakeDialer.Dial(url, wsHeader) if err != nil { b.Error("could not connect to url") break } ws.SetReadDeadline(time.Time{}) h := recvHandler{toOut, toError} go h.recv(ws, listenWS) case err := <-toError: b.Error(err) case <-b.quit: // quit the block return case o := <-b.queryrule: o <- map[string]interface{}{ "url": url, } case in := <-listenWS: b.out <- in } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *PackByInterval) Run() { var batch []interface{} interval := time.Duration(1) * time.Second ticker := time.NewTicker(interval) for { select { case <-ticker.C: b.out <- map[string]interface{}{ "Pack": batch, } batch = nil case ruleI := <-b.inrule: intervalS, err := util.ParseString(ruleI, "Interval") if err != nil { b.Error("error parsing batch size") break } dur, err := time.ParseDuration(intervalS) if err != nil { b.Error(err) break } if dur <= 0 { b.Error("interval must be positive") break } interval = dur ticker.Stop() ticker = time.NewTicker(interval) batch = nil case <-b.quit: // quit the block return case m := <-b.in: batch = append(batch, m) case <-b.clear: batch = nil case <-b.flush: b.out <- map[string]interface{}{ "Pack": batch, } batch = nil case r := <-b.queryrule: r <- map[string]interface{}{ "Interval": interval.String(), } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *FromFile) Run() { var file *os.File var filename string for { select { case msgI := <-b.inrule: // set a parameter of the block filename, err := util.ParseString(msgI, "Filename") if err != nil { b.Error(err) continue } file, err := os.Open(filename) if err != nil { b.Error(err) continue } scanner := bufio.NewScanner(file) for scanner.Scan() { var outMsg interface{} lineBytes := scanner.Bytes() err := json.Unmarshal(lineBytes, &outMsg) // if the json parsing fails, store data unparsed as "data" if err != nil { outMsg = map[string]interface{}{ "data": lineBytes, } } b.out <- outMsg } if err := scanner.Err(); err != nil { b.Error(err) continue } case <-b.quit: // quit the block if file != nil { file.Close() } return case c := <-b.queryrule: c <- map[string]interface{}{ "Filename": filename, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. // This block posts a message to a specified Elasticsearch index with the given type. func (b *ToElasticsearch) Run() { for { select { case msgI := <-b.inrule: host, _ := util.ParseString(msgI, "Host") port, _ := util.ParseString(msgI, "Port") index, _ := util.ParseString(msgI, "Index") indextype, _ := util.ParseString(msgI, "IndexType") // Set the Elasticsearch Host/Port to Connect to api.Domain = host api.Port = port b.host = host b.port = port b.index = index b.indextype = indextype case respChan := <-b.queryrule: // deal with a query request respChan <- map[string]interface{}{ "Host": b.host, "Port": b.port, "Index": b.index, "IndexType": b.indextype, } case <-b.quit: // quit the block return case msg := <-b.in: var args map[string]interface{} _, err := core.Index(b.index, b.indextype, "", args, msg) if err != nil { b.Error(err) } } } }
// buildEmail will attempt to pull the email's properties from the expected paths and // put the email body together. func (e *ToEmail) buildEmail(msg interface{}) (from, to string, email []byte, err error) { from, err = util.ParseString(msg, e.fromPath) if err != nil { return } to, err = util.ParseString(msg, e.toPath) if err != nil { return } var subject string subject, err = util.ParseString(msg, e.subjectPath) if err != nil { return } var body string body, err = util.ParseString(msg, e.msgPath) if err != nil { return } email = []byte(fmt.Sprintf(emailTmpl, from, to, subject, body)) return }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *DeDupe) Run() { var path string set := make(map[interface{}]bool) var tree *jee.TokenTree var err error for { select { case ruleI := <-b.inrule: // set a parameter of the block path, err = util.ParseString(ruleI, "Path") tree, err = util.BuildTokenTree(path) if err != nil { b.Error(err) break } case <-b.quit: // quit the block return // deal with inbound data case msg := <-b.in: if tree == nil { continue } v, err := jee.Eval(tree, msg) if err != nil { b.Error(err) break } if _, ok := v.(string); !ok { b.Error(errors.New("can only dedupe sets of strings")) continue } _, ok := set[v] // emit the incoming message if it isn't found in the set if !ok { b.out <- msg set[v] = true // and add it to the set } case c := <-b.queryrule: // deal with a query request c <- map[string]interface{}{ "Path": path, } } } }
// parseEmailInRules will expect a payload from the inrules channel and // attempt to pull and set the block's to, from and subject paths from it. func (e *ToEmail) parseEmailRules(msgI interface{}) error { var err error e.toPath, err = util.ParseRequiredString(msgI, "ToPath") if err != nil { return err } e.fromPath, err = util.ParseRequiredString(msgI, "FromPath") if err != nil { return err } e.subjectPath, err = util.ParseString(msgI, "SubjectPath") if err != nil { return err } e.msgPath, err = util.ParseString(msgI, "MessagePath") if err != nil { return err } return nil }
// parseAuthInRules will expect a payload from the inrules channel and // attempt to pull the SMTP auth credentials out it. If successful, this // will also create and set the block's auth. func (e *ToEmail) parseAuthRules(msgI interface{}) error { var err error e.host, err = util.ParseRequiredString(msgI, "Host") if err != nil { return err } e.port, err = util.ParseInt(msgI, "Port") if err != nil { return err } e.username, err = util.ParseString(msgI, "Username") if err != nil { return err } e.password, err = util.ParseString(msgI, "Password") if err != nil { return err } return nil }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *toWebsocket) Run() { var addr string var err error go h.run() for { select { case ruleI := <-b.inrule: // set a parameter of the block rule, ok := ruleI.(map[string]interface{}) if !ok { b.Error(errors.New("couldn't assert rule to map")) } addr, err = util.ParseString(rule, "port") if err != nil { b.Error(err) } http.HandleFunc("/ws", serveWs) go http.ListenAndServe(":"+addr, nil) if err != nil { b.Error(err) } case <-b.quit: // quit the block return case msg := <-b.in: if addr == "" { continue } out, err := json.Marshal(msg) if err != nil { b.Error(err) } h.broadcast <- out case c := <-b.queryrule: // deal with a query request c <- map[string]interface{}{ "port": addr, } } } log.Println("done") }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *ToFile) Run() { var err error var file *os.File var filename string for { select { case msgI := <-b.inrule: filename, err = util.ParseString(msgI, "Filename") if err != nil { b.Error(err) } file, err = os.Create(filename) if err != nil { b.Error(err) } case <-b.quit: // quit the block if file != nil { file.Close() } return case msg := <-b.in: // deal with inbound data writer := bufio.NewWriter(file) msgStr, err := json.Marshal(msg) if err != nil { b.Error(err) continue } fmt.Fprintln(writer, string(msgStr)) writer.Flush() case MsgChan := <-b.queryrule: // deal with a query request MsgChan <- map[string]interface{}{ "Filename": filename, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Ticker) Run() { interval := time.Duration(1) * time.Second ticker := time.NewTicker(interval) for { select { case tick := <-ticker.C: b.out <- map[string]interface{}{ "tick": tick.String(), } case ruleI := <-b.inrule: // set a parameter of the block intervalS, err := util.ParseString(ruleI, "Interval") if err != nil { b.Error("bad input") break } dur, err := time.ParseDuration(intervalS) if err != nil { b.Error(err) break } if dur <= 0 { b.Error("interval must be positive") break } interval = dur ticker.Stop() ticker = time.NewTicker(interval) case <-b.quit: return case c := <-b.queryrule: // deal with a query request c <- map[string]interface{}{ "Interval": interval.String(), } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *ToFile) Run() { for { select { case msgI := <-b.inrule: // set a parameter of the block filename, _ := util.ParseString(msgI, "Filename") fo, err := os.Create(filename) if err != nil { b.Error(err) } // set the new file b.file = fo // record the filename b.filename = filename case <-b.quit: // quit the block if b.file != nil { b.file.Close() } return case msg := <-b.in: // deal with inbound data w := bufio.NewWriter(b.file) msgStr, err := json.Marshal(msg) if err != nil { b.Error(err) continue } fmt.Fprintln(w, string(msgStr)) w.Flush() case respChan := <-b.queryrule: // deal with a query request respChan <- map[string]interface{}{ "Filename": b.filename, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *ParseCSV) Run() { var tree *jee.TokenTree var path string var err error var headers []string var csvReader *csv.Reader for { select { case ruleI := <-b.inrule: // set a parameter of the block path, err = util.ParseString(ruleI, "Path") if err != nil { b.Error(err) continue } token, err := jee.Lexer(path) if err != nil { b.Error(err) continue } tree, err = jee.Parser(token) if err != nil { b.Error(err) continue } headers, err = util.ParseArrayString(ruleI, "Headers") if err != nil { b.Error(err) continue } case <-b.quit: // quit the block return case msg := <-b.in: // deal with inbound data if tree == nil { continue } var data string dataI, err := jee.Eval(tree, msg) if err != nil { b.Error(err) continue } switch value := dataI.(type) { case []byte: data = string(value[:]) case string: data = value default: b.Error("data should be a string or a []byte") continue } csvReader = csv.NewReader(strings.NewReader(data)) csvReader.TrimLeadingSpace = true // allow records to have variable numbers of fields csvReader.FieldsPerRecord = -1 case <-b.inpoll: if csvReader == nil { b.Error("this block needs data to be pollable") break } record, err := csvReader.Read() if err != nil && err != io.EOF { b.Error(err) continue } row := make(map[string]interface{}) for fieldIndex, field := range record { if fieldIndex >= len(headers) { row[strconv.Itoa(fieldIndex)] = field } else { header := headers[fieldIndex] row[header] = field } } b.out <- row case MsgChan := <-b.queryrule: // deal with a query request MsgChan <- map[string]interface{}{ "Path": path, "Headers": headers, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *ToBeanstalkd) Run() { var conn *lentil.Beanstalkd var tube = "default" var ttr = 0 var host = "" var err error for { select { case msgI := <-b.inrule: // set hostname for beanstalkd server host, err = util.ParseString(msgI, "Host") if err != nil { b.Error(err.Error()) continue } // set tube name tube, err = util.ParseString(msgI, "Tube") if err != nil { b.Error(errors.New("Could not parse tube name, setting to 'default'")) tube = "default" } // set time to reserve ttr, err = util.ParseInt(msgI, "TTR") if err != nil || ttr < 0 { b.Error(errors.New("Error parsing TTR. Setting TTR to 0")) ttr = 0 } // create beanstalkd connection conn, err = lentil.Dial(host) if err != nil { // swallowing a panic from lentil here - streamtools must not die b.Error(errors.New("Could not initiate connection with beanstalkd server")) continue } // use the specified tube conn.Use(tube) case <-b.quit: // close connection to beanstalkd and quit if conn != nil { conn.Quit() } return case msg := <-b.in: // deal with inbound data msgStr, err := json.Marshal(msg) if err != nil { b.Error(err) continue } if conn != nil { _, err := conn.Put(0, 0, ttr, msgStr) if err != nil { b.Error(err.Error()) } } else { b.Error(errors.New("Beanstalkd connection not initated or lost. Please check your beanstalkd server or block settings.")) } case respChan := <-b.queryrule: // deal with a query request respChan <- map[string]interface{}{ "Host": host, "Tube": tube, "TTR": ttr, } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Sync) Run() { var lagString, path string var tree *jee.TokenTree lag := time.Duration(0) emitTick := time.NewTimer(500 * time.Millisecond) pq := &PriorityQueue{} heap.Init(pq) for { select { case <-emitTick.C: case ruleI := <-b.inrule: // set a parameter of the block lagString, err := util.ParseString(ruleI, "Lag") if err != nil { b.Error(err) break } lag, err = time.ParseDuration(lagString) if err != nil { b.Error(err) continue } path, err = util.ParseString(ruleI, "Path") if err != nil { b.Error(err) break } // build the parser for the model token, err := jee.Lexer(path) if err != nil { b.Error(err) continue } tree, err = jee.Parser(token) if err != nil { b.Error(err) continue } case <-b.quit: // quit the block return case msg := <-b.in: // deal with inbound data if tree == nil { break } tI, err := jee.Eval(tree, interface{}(msg)) if err != nil { b.Error(err) } t, ok := tI.(float64) if !ok { b.Error(errors.New("couldn't convert time value to float64")) continue } ms := time.Unix(0, int64(t*1000000)) queueMessage := &PQMessage{ val: msg, t: ms, } heap.Push(pq, queueMessage) case respChan := <-b.queryrule: // deal with a query request respChan <- map[string]interface{}{ "Lag": lagString, "Path": path, } } now := time.Now() for { item, diff := pq.PeekAndShift(now, lag) if item == nil { // then the queue is empty. Pause for 5 seconds before checking again if diff == 0 { diff = time.Duration(500) * time.Millisecond } emitTick.Reset(diff) break } b.out <- item.(*PQMessage).val.(map[string]interface{}) } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *MovingAverage) Run() { var tree *jee.TokenTree var path, windowString string var err error window := time.Duration(0) waitTimer := time.NewTimer(100 * time.Millisecond) pq := &PriorityQueue{} heap.Init(pq) for { select { case ruleI := <-b.inrule: // set a parameter of the block path, err = util.ParseString(ruleI, "Path") if err != nil { b.Error(err) } tree, err = util.BuildTokenTree(path) if err != nil { b.Error(err) break } windowString, err = util.ParseString(ruleI, "Window") if err != nil { b.Error(err) } window, err = time.ParseDuration(windowString) if err != nil { b.Error(err) } case <-b.quit: // quit the block return case msg := <-b.in: // deal with inbound data if tree == nil { break } val, err := jee.Eval(tree, msg) if err != nil { b.Error(err) break } // TODO make this a type swtich and convert anything we can to a // float val, ok := val.(float64) if !ok { b.Error(errors.New("trying to put a non-float into the moving average")) continue } queueMessage := &PQMessage{ val: val, t: time.Now(), } heap.Push(pq, queueMessage) case <-b.inpoll: // deal with a poll request outMsg := map[string]interface{}{ "Average": pqAverage(pq), } b.out <- outMsg case c := <-b.queryavg: outMsg := map[string]interface{}{ "Average": pqAverage(pq), } c <- outMsg case c := <-b.queryrule: // deal with a query request c <- map[string]interface{}{ "Path": path, "Window": windowString, } case <-waitTimer.C: } for { pqMsg, diff := pq.PeekAndShift(time.Now(), window) if pqMsg == nil { // either the queue is empty, or it's not time to emit if diff == 0 { // then the queue is empty. Pause for 5 seconds before checking again diff = time.Duration(500) * time.Millisecond } waitTimer.Reset(diff) break } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Redis) Run() { var server = "localhost:6379" var password string var command string var arguments = make([]string, 0, 10) var argumentTrees []*jee.TokenTree var err error var pool *redis.Pool for { select { case ruleI := <-b.inrule: server, err = util.ParseString(ruleI, "Server") if err != nil { b.Error(err) continue } password, err = util.ParseString(ruleI, "Password") if err != nil { b.Error(err) continue } command, err = util.ParseString(ruleI, "Command") if err != nil { b.Error(err) continue } if util.KeyExists(ruleI, "Arguments") { arguments, err = util.ParseArrayString(ruleI, "Arguments") if err != nil { b.Error(err) continue } } if len(arguments) > 0 { argumentTrees = make([]*jee.TokenTree, len(arguments)) for i, path := range arguments { token, err := jee.Lexer(path) if err != nil { b.Error(err) continue } tree, err := jee.Parser(token) if err != nil { b.Error(err) continue } argumentTrees[i] = tree } } pool = newPool(server, password) case responseChan := <-b.queryrule: // deal with a query request responseChan <- map[string]interface{}{ "Server": server, "Password": password, "Command": command, "Arguments": arguments, } case <-b.quit: // quit the block return case msg := <-b.in: if pool == nil { b.Error("not connected to redis") break } conn := pool.Get() args := make([]interface{}, len(argumentTrees)) for i, tree := range argumentTrees { argument, err := jee.Eval(tree, msg) if err != nil { b.Error(err) break } args[i] = argument } // commands like 'KEYS *' or 'SET NUMBERS 1' reply, err := conn.Do(command, args...) if err != nil { b.Error(err) break } conn.Close() nicerReply, err := formatReply(reply) out := map[string]interface{}{ "response": nicerReply, } b.out <- out } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Javascript) Run() { messageIn := "input" messageOut := "output" script := `output = input` vm := otto.New() program, _ := vm.Compile("javascript", script) for { select { case ruleI := <-b.inrule: tmpMin, err := util.ParseString(ruleI, "MessageIn") if err != nil { b.Error(err) break } tmpMout, err := util.ParseString(ruleI, "MessageOut") if err != nil { b.Error(err) break } tmpScript, err := util.ParseString(ruleI, "Script") if err != nil { b.Error(err) break } tmpProgram, err := vm.Compile("javascript", tmpScript) if err != nil { b.Error(err) break } messageIn = tmpMin messageOut = tmpMout script = tmpScript program = tmpProgram case <-b.quit: // quit the block return case m := <-b.in: if program == nil { break } err := vm.Set(messageOut, map[string]interface{}{}) if err != nil { b.Error(err) break } err = vm.Set(messageIn, m) if err != nil { b.Error(err) break } _, err = vm.Run(program) if err != nil { b.Error(err) break } g, err := vm.Get(messageOut) if err != nil { b.Error(err) break } o, err := g.Export() if err != nil { b.Error(err) break } b.out <- o case c := <-b.queryrule: c <- map[string]interface{}{ "MessageIn": messageIn, "MessageOut": messageOut, "Script": script, } } } }
// connects to an AMQP topic and emits each message into streamtools. func (b *FromAMQP) Run() { var err error var conn *amqp.Connection var amqp_chan *amqp.Channel toOut := make(blocks.MsgChan) toError := make(chan error) host := "localhost" port := "5672" username := "******" password := "******" routingkey := "#" exchange := "amq.topic" exchange_type := "topic" for { select { case msg := <-toOut: b.out <- msg case err := <-toError: b.Error(err) case ruleI := <-b.inrule: rule := ruleI.(map[string]interface{}) routingkey, err = util.ParseString(rule, "RoutingKey") if err != nil { b.Error(err) continue } exchange, err = util.ParseString(rule, "Exchange") if err != nil { b.Error(err) continue } exchange_type, err = util.ParseString(rule, "ExchangeType") if err != nil { b.Error(err) continue } host, err = util.ParseString(rule, "Host") if err != nil { b.Error(err) continue } port, err = util.ParseString(rule, "Port") if err != nil { b.Error(err) continue } username, err = util.ParseString(rule, "Username") if err != nil { b.Error(err) continue } password, err = util.ParseString(rule, "Password") if err != nil { b.Error(err) continue } conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/") if err != nil { b.Error(err) continue } amqp_chan, err = conn.Channel() if err != nil { b.Error(err) continue } err = amqp_chan.ExchangeDeclare( exchange, // name exchange_type, // type true, // durable false, // auto-deleted false, // internal false, // noWait nil, // arguments ) if err != nil { b.Error(err) continue } queue, err := amqp_chan.QueueDeclare( "", // name false, // durable true, // delete when unused false, // exclusive false, // noWait nil, // arguments ) if err != nil { b.Error(err) continue } err = amqp_chan.QueueBind( queue.Name, // queue name routingkey, // routing key exchange, // exchange false, nil, ) if err != nil { b.Error(err) continue } deliveries, err := amqp_chan.Consume( queue.Name, // name "", // consumerTag true, // noAck false, // exclusive false, // noLocal false, // noWait nil, // arguments ) if err != nil { b.Error(err) continue } h := readWriteAMQPHandler{toOut, toError} go h.handle(deliveries) case <-b.quit: if amqp_chan != nil { amqp_chan.Close() } if conn != nil { conn.Close() } return case c := <-b.queryrule: c <- map[string]interface{}{ "Host": host, "Port": port, "Username": username, "Password": password, "Exchange": exchange, "ExchangeType": exchange_type, "RoutingKey": routingkey, } } } }
// connects to an NSQ topic and emits each message into streamtools. func (b *ToNSQMulti) Run() { var err error var nsqdTCPAddrs string var topic string var writer *nsq.Producer var batch [][]byte interval := time.Duration(1 * time.Second) maxBatch := 100 conf := nsq.NewConfig() dump := time.NewTicker(interval) for { select { case <-dump.C: if writer == nil || len(batch) == 0 { break } err = writer.MultiPublish(topic, batch) if err != nil { b.Error(err.Error()) } batch = nil case ruleI := <-b.inrule: //rule := ruleI.(map[string]interface{}) topic, err = util.ParseString(ruleI, "Topic") if err != nil { b.Error(err) break } nsqdTCPAddrs, err = util.ParseString(ruleI, "NsqdTCPAddrs") if err != nil { b.Error(err) break } intervalS, err := util.ParseString(ruleI, "Interval") if err != nil { b.Error("bad input") break } dur, err := time.ParseDuration(intervalS) if err != nil { b.Error(err) break } if dur <= 0 { b.Error("interval must be positive") break } batchSize, err := util.ParseFloat(ruleI, "MaxBatch") if err != nil { b.Error("error parsing batch size") break } if writer != nil { writer.Stop() } maxBatch = int(batchSize) interval = dur dump.Stop() dump = time.NewTicker(interval) writer, err = nsq.NewProducer(nsqdTCPAddrs, conf) if err != nil { b.Error(err) break } topic = topic nsqdTCPAddrs = nsqdTCPAddrs case msg := <-b.in: if writer == nil { break } msgByte, err := json.Marshal(msg) if err != nil { b.Error(err) } batch = append(batch, msgByte) if len(batch) > maxBatch { err := writer.MultiPublish(topic, batch) if err != nil { b.Error(err) break } batch = nil } case <-b.quit: if writer != nil { writer.Stop() } dump.Stop() return case c := <-b.queryrule: c <- map[string]interface{}{ "Topic": topic, "NsqdTCPAddrs": nsqdTCPAddrs, "MaxBatch": maxBatch, "Interval": interval.String(), } } } }
// Run is the block's main loop. Here we listen on the different channels we set up. func (b *Cache) Run() { var keyPath, valuePath, ttlString string var ttl time.Duration values := make(map[string]item) ttlQueue := &PriorityQueue{} var keyTree, valueTree *jee.TokenTree var err error emitTick := time.NewTimer(500 * time.Millisecond) for { select { case <-emitTick.C: case ruleI := <-b.inrule: keyPath, err = util.ParseString(ruleI, "KeyPath") keyTree, err = util.BuildTokenTree(keyPath) if err != nil { b.Error(err) break } valuePath, err = util.ParseString(ruleI, "ValuePath") valueTree, err = util.BuildTokenTree(valuePath) if err != nil { b.Error(err) break } ttlString, err = util.ParseString(ruleI, "TimeToLive") if err != nil { b.Error(err) break } ttl, err = time.ParseDuration(ttlString) if err != nil { b.Error(err) break } case <-b.quit: return case msg := <-b.lookup: if keyTree == nil { continue } kI, err := jee.Eval(keyTree, msg) if err != nil { b.Error(err) break } k, ok := kI.(string) if !ok { b.Error(errors.New("key must be a string")) continue } i, ok := values[k] var v interface{} if ok { v = i.value now := time.Now() i.lastSeen = now queueMessage := &PQMessage{ val: k, t: now, } heap.Push(ttlQueue, queueMessage) } b.out <- map[string]interface{}{ "key": k, "value": v, } case msg := <-b.in: if keyTree == nil { continue } if valueTree == nil { continue } kI, err := jee.Eval(keyTree, msg) if err != nil { b.Error(err) break } k, ok := kI.(string) if !ok { b.Error(errors.New("key must be a string")) continue } v, err := jee.Eval(valueTree, msg) if err != nil { b.Error(err) break } now := time.Now() values[k] = item{ value: v, lastSeen: now, } queueMessage := &PQMessage{ val: k, t: now, } heap.Push(ttlQueue, queueMessage) case responseChan := <-b.queryrule: // deal with a query request responseChan <- map[string]interface{}{ "KeyPath": keyPath, "ValuePath": valuePath, "TimeToLive": ttlString, } } now := time.Now() for { itemI, diff := ttlQueue.PeekAndShift(now, ttl) if itemI == nil { // then the queue is empty. don't check again for 5s if diff == 0 { diff = time.Duration(500) * time.Millisecond } emitTick.Reset(diff) break } i := itemI.(*PQMessage) k := i.val.(string) if i.t.Equal(values[k].lastSeen) { delete(values, k) } } } }
func (b *KullbackLeibler) Run() { var qtree, ptree *jee.TokenTree var qpath, ppath string var err error for { select { case ruleI := <-b.inrule: qpath, err = util.ParseString(ruleI, "QPath") if err != nil { b.Error(err) } qtree, err = util.BuildTokenTree(qpath) ppath, err = util.ParseString(ruleI, "PPath") if err != nil { b.Error(err) } ptree, err = util.BuildTokenTree(ppath) case c := <-b.queryrule: c <- map[string]interface{}{ "QPath": qpath, "PPath": ppath, } case <-b.quit: return case msg := <-b.in: if ptree == nil { continue } if qtree == nil { continue } pI, err := jee.Eval(ptree, msg) if err != nil { b.Error(err) break } qI, err := jee.Eval(qtree, msg) if err != nil { b.Error(err) break } p, ok := newHistogram(pI) if !ok { b.Error(errors.New("p is not a Histogram")) continue } q, ok := newHistogram(qI) if !ok { b.Error(errors.New("q is not a Histogram")) continue } q.normalise(p) p.normalise(q) kl := 0.0 for k, pi := range p { kl += math.Log(pi/q[k]) * pi } b.out <- map[string]interface{}{ "KL": kl, } } } }