Esempio n. 1
0
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToNSQ) Run() {
	var writer *nsq.Writer

	for {
		select {
		case ruleI := <-b.inrule:
			//rule := ruleI.(map[string]interface{})

			topic, err := util.ParseString(ruleI, "Topic")
			if err != nil {
				b.Error(err)
				break
			}

			nsqdTCPAddrs, err := util.ParseString(ruleI, "NsqdTCPAddrs")
			if err != nil {
				b.Error(err)
				break
			}

			if writer != nil {
				writer.Stop()
			}

			writer = nsq.NewWriter(nsqdTCPAddrs)

			b.topic = topic
			b.nsqdTCPAddrs = nsqdTCPAddrs

		case msg := <-b.in:
			if writer == nil {
				continue
			}
			msgBytes, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
			}
			if len(msgBytes) == 0 {
				continue
			}
			_, _, err = writer.Publish(b.topic, msgBytes)
			if err != nil {
				b.Error(err)
			}

		case <-b.quit:
			if writer != nil {
				writer.Stop()
			}
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Topic":        b.topic,
				"NsqdTCPAddrs": b.nsqdTCPAddrs,
			}
		}
	}
}
Esempio n. 2
0
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToElasticsearch) Run() {
	var err error
	var esIndex string
	var esType string

	conn := elastigo.NewConn()

	host := "localhost"
	port := "9200"

	for {
		select {
		case ruleI := <-b.inrule:
			host, err = util.ParseString(ruleI, "Host")
			if err != nil {
				b.Error(err)
				break
			}

			port, err = util.ParseString(ruleI, "Port")
			if err != nil {
				b.Error(err)
				break
			}

			esIndex, err = util.ParseString(ruleI, "Index")
			if err != nil {
				b.Error(err)
				break
			}

			esType, err = util.ParseString(ruleI, "Type")
			if err != nil {
				b.Error(err)
				break
			}

			conn.Domain = host
			conn.Port = port

		case msg := <-b.in:
			_, err := conn.Index(esIndex, esType, "", nil, msg)
			if err != nil {
				b.Error(err)
			}
		case <-b.quit:
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Host":  host,
				"Port":  port,
				"Index": esIndex,
				"Type":  esType,
			}
		}
	}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
// This block posts a message to a specified Elasticsearch index with the given type.
func (b *ToElasticsearch) Run() {
	var err error
	var index string
	var indextype string

	host := "localhost"
	port := "9200"

	for {
		select {
		case msgI := <-b.inrule:
			host, err = util.ParseString(msgI, "Host")
			if err != nil {
				b.Error(err)
				continue
			}
			port, err = util.ParseString(msgI, "Port")
			if err != nil {
				b.Error(err)
				continue
			}
			index, err = util.ParseString(msgI, "Index")
			if err != nil {
				b.Error(err)
				continue
			}
			indextype, err = util.ParseString(msgI, "IndexType")
			if err != nil {
				b.Error(err)
				continue
			}

			// Set the Elasticsearch Host/Port to Connect to
			api.Domain = host
			api.Port = port

		case MsgChan := <-b.queryrule:
			// deal with a query request
			MsgChan <- map[string]interface{}{
				"Host":      host,
				"Port":      port,
				"Index":     index,
				"IndexType": indextype,
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			var args map[string]interface{}
			_, err := core.Index(index, indextype, "", args, msg)
			if err != nil {
				b.Error(err)
			}
		}
	}
}
Esempio n. 4
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToHTTPGetRequest) Run() {
	var respPath, msgPath string

	var respTree, msgTree *jee.TokenTree
	var err error
	for {
		select {
		case ruleI := <-b.inrule:
			respPath, err = util.ParseString(ruleI, "RespPath")
			respTree, err = util.BuildTokenTree(respPath)
			if err != nil {
				b.Error(err)
				break
			}
			msgPath, err = util.ParseString(ruleI, "MsgPath")
			msgTree, err = util.BuildTokenTree(msgPath)
			if err != nil {
				b.Error(err)
				break
			}
		case <-b.quit:
			return
		case msg := <-b.in:
			if respTree == nil {
				continue
			}
			if msgTree == nil {
				continue
			}
			cI, err := jee.Eval(respTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			c, ok := cI.(blocks.MsgChan)
			if !ok {
				b.Error(errors.New("response path must point to a channel"))
				continue
			}
			m, err := jee.Eval(msgTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			c <- m
		case responseChan := <-b.queryrule:
			// deal with a query request
			responseChan <- map[string]interface{}{
				"RespPath": respPath,
				"MsgPath":  msgPath,
			}
		}
	}
}
Esempio n. 5
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromSQS) Run() {
	var err error

	for {
		select {
		case msgI := <-b.inrule:
			for k, _ := range b.auth {
				b.auth[k], err = util.ParseString(msgI, k)
				if err != nil {
					b.Error(err)
					break
				}
			}

			b.stopListening()
			go b.listener()
		case <-b.quit:
			b.stopListening()
			return
		case msg := <-b.fromListener:
			var outMsg interface{}
			err := json.Unmarshal(msg, &outMsg)
			if err != nil {
				b.Error(err)
				continue
			}
			b.out <- outMsg
		case MsgChan := <-b.queryrule:
			// deal with a query request
			MsgChan <- b.auth
		}
	}
}
Esempio n. 6
0
func (b *Count) Run() {
	waitTimer := time.NewTimer(100 * time.Millisecond)
	pq := &PriorityQueue{}
	heap.Init(pq)
	window := time.Duration(0)

	for {
		select {
		case <-waitTimer.C:
		case rule := <-b.inrule:

			tmpDurStr, err := util.ParseString(rule, "Window")
			if err != nil {
				b.Error(err)
				continue
			}

			tmpWindow, err := time.ParseDuration(tmpDurStr)
			if err != nil {
				b.Error(err)
				continue
			}

			window = tmpWindow
		case <-b.quit:
			return
		case <-b.in:
			empty := make([]byte, 0)
			queueMessage := &PQMessage{
				val: &empty,
				t:   time.Now(),
			}
			heap.Push(pq, queueMessage)
		case <-b.inpoll:
			b.out <- map[string]interface{}{
				"Count": len(*pq),
			}
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Window": window.String(),
			}
		case c := <-b.querycount:
			c <- map[string]interface{}{
				"Count": len(*pq),
			}
		}
		for {
			pqMsg, diff := pq.PeekAndShift(time.Now(), window)
			if pqMsg == nil {
				// either the queue is empty, or it"s not time to emit
				if diff == 0 {
					// then the queue is empty. Pause for 5 seconds before checking again
					diff = time.Duration(500) * time.Millisecond
				}
				waitTimer.Reset(diff)
				break
			}
		}
	}
}
Esempio n. 7
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Set) Run() {
	var path string
	set := make(map[interface{}]bool)
	var tree *jee.TokenTree
	var err error
	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			path, err = util.ParseString(ruleI, "Path")
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				break
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.add:
			if tree == nil {
				continue
			}
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			if _, ok := v.(string); !ok {
				b.Error(errors.New("can only build sets of strings"))
				continue
			}
			set[v] = true
			// deal with inbound data
		case msg := <-b.isMember:
			if tree == nil {
				continue
			}
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			_, ok := set[v]
			b.out <- map[string]interface{}{
				"isMember": ok,
			}
		case c := <-b.cardinality:
			c <- map[string]interface{}{
				"cardinality": len(set),
			}
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Path": path,
			}

		}
	}
}
Esempio n. 8
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Unpack) Run() {
	var path string
	var err error
	var tree *jee.TokenTree
	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("cannot assert rule to map"))
			}
			path, err = util.ParseString(rule, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}

		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			arrInterface, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}
			arr, ok := arrInterface.([]interface{})
			if !ok {
				b.Error(errors.New("cannot assert " + path + " to array"))
				continue
			}
			for _, out := range arr {
				b.out <- out
			}
		case c := <-b.queryrule:
			// deal with a query request
			out := map[string]interface{}{
				"Path": path,
			}
			c <- out
		}
	}
}
Esempio n. 9
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromWebsocket) Run() {
	var ws *websocket.Conn
	var url string
	to, _ := time.ParseDuration("10s")
	var handshakeDialer = &websocket.Dialer{
		Subprotocols:     []string{"p1", "p2"},
		HandshakeTimeout: to,
	}
	listenWS := make(blocks.MsgChan)
	wsHeader := http.Header{"Origin": {"http://localhost/"}}

	toOut := make(blocks.MsgChan)
	toError := make(chan error)

	for {
		select {

		case msg := <-toOut:
			b.out <- msg

		case ruleI := <-b.inrule:
			var err error
			// set a parameter of the block
			url, err = util.ParseString(ruleI, "url")
			if err != nil {
				b.Error(err)
				continue
			}
			if ws != nil {
				ws.Close()
			}

			ws, _, err = handshakeDialer.Dial(url, wsHeader)
			if err != nil {
				b.Error("could not connect to url")
				break
			}
			ws.SetReadDeadline(time.Time{})
			h := recvHandler{toOut, toError}
			go h.recv(ws, listenWS)

		case err := <-toError:
			b.Error(err)

		case <-b.quit:
			// quit the block
			return
		case o := <-b.queryrule:
			o <- map[string]interface{}{
				"url": url,
			}
		case in := <-listenWS:
			b.out <- in
		}
	}
}
Esempio n. 10
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *PackByInterval) Run() {
	var batch []interface{}

	interval := time.Duration(1) * time.Second
	ticker := time.NewTicker(interval)
	for {
		select {
		case <-ticker.C:
			b.out <- map[string]interface{}{
				"Pack": batch,
			}
			batch = nil

		case ruleI := <-b.inrule:
			intervalS, err := util.ParseString(ruleI, "Interval")
			if err != nil {
				b.Error("error parsing batch size")
				break
			}

			dur, err := time.ParseDuration(intervalS)
			if err != nil {
				b.Error(err)
				break
			}

			if dur <= 0 {
				b.Error("interval must be positive")
				break
			}

			interval = dur
			ticker.Stop()
			ticker = time.NewTicker(interval)
			batch = nil
		case <-b.quit:
			// quit the block
			return
		case m := <-b.in:
			batch = append(batch, m)
		case <-b.clear:
			batch = nil
		case <-b.flush:
			b.out <- map[string]interface{}{
				"Pack": batch,
			}
			batch = nil
		case r := <-b.queryrule:
			r <- map[string]interface{}{
				"Interval": interval.String(),
			}
		}
	}
}
Esempio n. 11
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FromFile) Run() {
	var file *os.File
	var filename string

	for {
		select {
		case msgI := <-b.inrule:
			// set a parameter of the block
			filename, err := util.ParseString(msgI, "Filename")
			if err != nil {
				b.Error(err)
				continue
			}

			file, err := os.Open(filename)
			if err != nil {
				b.Error(err)
				continue
			}

			scanner := bufio.NewScanner(file)
			for scanner.Scan() {
				var outMsg interface{}
				lineBytes := scanner.Bytes()
				err := json.Unmarshal(lineBytes, &outMsg)
				// if the json parsing fails, store data unparsed as "data"
				if err != nil {
					outMsg = map[string]interface{}{
						"data": lineBytes,
					}
				}
				b.out <- outMsg
			}

			if err := scanner.Err(); err != nil {
				b.Error(err)
				continue
			}
		case <-b.quit:
			// quit the block
			if file != nil {
				file.Close()
			}
			return

		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Filename": filename,
			}
		}
	}
}
Esempio n. 12
0
// Run is the block's main loop. Here we listen on the different channels we set up.
// This block posts a message to a specified Elasticsearch index with the given type.
func (b *ToElasticsearch) Run() {
	for {
		select {
		case msgI := <-b.inrule:
			host, _ := util.ParseString(msgI, "Host")
			port, _ := util.ParseString(msgI, "Port")
			index, _ := util.ParseString(msgI, "Index")
			indextype, _ := util.ParseString(msgI, "IndexType")

			// Set the Elasticsearch Host/Port to Connect to
			api.Domain = host
			api.Port = port

			b.host = host
			b.port = port
			b.index = index
			b.indextype = indextype

		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				"Host":      b.host,
				"Port":      b.port,
				"Index":     b.index,
				"IndexType": b.indextype,
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			var args map[string]interface{}
			_, err := core.Index(b.index, b.indextype, "", args, msg)
			if err != nil {
				b.Error(err)
			}
		}
	}
}
Esempio n. 13
0
// buildEmail will attempt to pull the email's properties from the expected paths and
// put the email body together.
func (e *ToEmail) buildEmail(msg interface{}) (from, to string, email []byte, err error) {
	from, err = util.ParseString(msg, e.fromPath)
	if err != nil {
		return
	}
	to, err = util.ParseString(msg, e.toPath)
	if err != nil {
		return
	}
	var subject string
	subject, err = util.ParseString(msg, e.subjectPath)
	if err != nil {
		return
	}
	var body string
	body, err = util.ParseString(msg, e.msgPath)
	if err != nil {
		return
	}

	email = []byte(fmt.Sprintf(emailTmpl, from, to, subject, body))
	return
}
Esempio n. 14
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *DeDupe) Run() {
	var path string
	set := make(map[interface{}]bool)
	var tree *jee.TokenTree
	var err error
	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			path, err = util.ParseString(ruleI, "Path")
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				break
			}
		case <-b.quit:
			// quit the block
			return
			// deal with inbound data
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}

			if _, ok := v.(string); !ok {
				b.Error(errors.New("can only dedupe sets of strings"))
				continue
			}

			_, ok := set[v]
			// emit the incoming message if it isn't found in the set
			if !ok {
				b.out <- msg
				set[v] = true // and add it to the set
			}
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Path": path,
			}

		}
	}
}
Esempio n. 15
0
// parseEmailInRules will expect a payload from the inrules channel and
// attempt to pull and set the block's to, from and subject paths from it.
func (e *ToEmail) parseEmailRules(msgI interface{}) error {
	var err error
	e.toPath, err = util.ParseRequiredString(msgI, "ToPath")
	if err != nil {
		return err
	}

	e.fromPath, err = util.ParseRequiredString(msgI, "FromPath")
	if err != nil {
		return err
	}

	e.subjectPath, err = util.ParseString(msgI, "SubjectPath")
	if err != nil {
		return err
	}

	e.msgPath, err = util.ParseString(msgI, "MessagePath")
	if err != nil {
		return err
	}

	return nil
}
Esempio n. 16
0
// parseAuthInRules will expect a payload from the inrules channel and
// attempt to pull the SMTP auth credentials out it. If successful, this
// will also create and set the block's auth.
func (e *ToEmail) parseAuthRules(msgI interface{}) error {
	var err error
	e.host, err = util.ParseRequiredString(msgI, "Host")
	if err != nil {
		return err
	}

	e.port, err = util.ParseInt(msgI, "Port")
	if err != nil {
		return err
	}

	e.username, err = util.ParseString(msgI, "Username")
	if err != nil {
		return err
	}

	e.password, err = util.ParseString(msgI, "Password")
	if err != nil {
		return err
	}

	return nil
}
Esempio n. 17
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *toWebsocket) Run() {

	var addr string
	var err error

	go h.run()

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("couldn't assert rule to map"))
			}
			addr, err = util.ParseString(rule, "port")
			if err != nil {
				b.Error(err)
			}
			http.HandleFunc("/ws", serveWs)
			go http.ListenAndServe(":"+addr, nil)
			if err != nil {
				b.Error(err)
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if addr == "" {
				continue
			}
			out, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
			}
			h.broadcast <- out

		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"port": addr,
			}
		}
	}
	log.Println("done")
}
Esempio n. 18
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToFile) Run() {
	var err error
	var file *os.File
	var filename string

	for {
		select {
		case msgI := <-b.inrule:
			filename, err = util.ParseString(msgI, "Filename")
			if err != nil {
				b.Error(err)
			}

			file, err = os.Create(filename)
			if err != nil {
				b.Error(err)
			}

		case <-b.quit:
			// quit the block
			if file != nil {
				file.Close()
			}
			return
		case msg := <-b.in:
			// deal with inbound data
			writer := bufio.NewWriter(file)
			msgStr, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
				continue
			}
			fmt.Fprintln(writer, string(msgStr))
			writer.Flush()

		case MsgChan := <-b.queryrule:
			// deal with a query request
			MsgChan <- map[string]interface{}{
				"Filename": filename,
			}
		}
	}
}
Esempio n. 19
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Ticker) Run() {
	interval := time.Duration(1) * time.Second
	ticker := time.NewTicker(interval)
	for {
		select {
		case tick := <-ticker.C:
			b.out <- map[string]interface{}{
				"tick": tick.String(),
			}
		case ruleI := <-b.inrule:
			// set a parameter of the block
			intervalS, err := util.ParseString(ruleI, "Interval")
			if err != nil {
				b.Error("bad input")
				break
			}

			dur, err := time.ParseDuration(intervalS)
			if err != nil {
				b.Error(err)
				break
			}

			if dur <= 0 {
				b.Error("interval must be positive")
				break
			}

			interval = dur
			ticker.Stop()
			ticker = time.NewTicker(interval)
		case <-b.quit:
			return
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Interval": interval.String(),
			}
		}
	}
}
Esempio n. 20
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToFile) Run() {
	for {
		select {
		case msgI := <-b.inrule:
			// set a parameter of the block
			filename, _ := util.ParseString(msgI, "Filename")

			fo, err := os.Create(filename)
			if err != nil {
				b.Error(err)
			}
			// set the new file
			b.file = fo
			// record the filename
			b.filename = filename
		case <-b.quit:
			// quit the block
			if b.file != nil {
				b.file.Close()
			}
			return
		case msg := <-b.in:
			// deal with inbound data
			w := bufio.NewWriter(b.file)
			msgStr, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
				continue
			}
			fmt.Fprintln(w, string(msgStr))
			w.Flush()
		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				"Filename": b.filename,
			}
		}
	}
}
Esempio n. 21
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ParseCSV) Run() {
	var tree *jee.TokenTree
	var path string
	var err error
	var headers []string
	var csvReader *csv.Reader

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			path, err = util.ParseString(ruleI, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}

			headers, err = util.ParseArrayString(ruleI, "Headers")
			if err != nil {
				b.Error(err)
				continue
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				continue
			}
			var data string

			dataI, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}

			switch value := dataI.(type) {
			case []byte:
				data = string(value[:])

			case string:
				data = value

			default:
				b.Error("data should be a string or a []byte")
				continue
			}

			csvReader = csv.NewReader(strings.NewReader(data))
			csvReader.TrimLeadingSpace = true
			// allow records to have variable numbers of fields
			csvReader.FieldsPerRecord = -1

		case <-b.inpoll:
			if csvReader == nil {
				b.Error("this block needs data to be pollable")
				break
			}
			record, err := csvReader.Read()
			if err != nil && err != io.EOF {
				b.Error(err)
				continue
			}
			row := make(map[string]interface{})
			for fieldIndex, field := range record {
				if fieldIndex >= len(headers) {
					row[strconv.Itoa(fieldIndex)] = field
				} else {
					header := headers[fieldIndex]
					row[header] = field
				}
			}

			b.out <- row

		case MsgChan := <-b.queryrule:
			// deal with a query request
			MsgChan <- map[string]interface{}{
				"Path":    path,
				"Headers": headers,
			}
		}
	}
}
Esempio n. 22
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToBeanstalkd) Run() {
	var conn *lentil.Beanstalkd
	var tube = "default"
	var ttr = 0
	var host = ""
	var err error
	for {
		select {
		case msgI := <-b.inrule:
			// set hostname for beanstalkd server
			host, err = util.ParseString(msgI, "Host")
			if err != nil {
				b.Error(err.Error())
				continue
			}
			// set tube name
			tube, err = util.ParseString(msgI, "Tube")
			if err != nil {
				b.Error(errors.New("Could not parse tube name, setting to 'default'"))
				tube = "default"
			}
			// set time to reserve
			ttr, err = util.ParseInt(msgI, "TTR")
			if err != nil || ttr < 0 {
				b.Error(errors.New("Error parsing TTR. Setting TTR to 0"))
				ttr = 0
			}
			// create beanstalkd connection
			conn, err = lentil.Dial(host)
			if err != nil {
				// swallowing a panic from lentil here - streamtools must not die
				b.Error(errors.New("Could not initiate connection with beanstalkd server"))
				continue
			}
			// use the specified tube
			conn.Use(tube)
		case <-b.quit:
			// close connection to beanstalkd and quit
			if conn != nil {
				conn.Quit()
			}
			return
		case msg := <-b.in:
			// deal with inbound data
			msgStr, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
				continue
			}
			if conn != nil {
				_, err := conn.Put(0, 0, ttr, msgStr)
				if err != nil {
					b.Error(err.Error())
				}
			} else {
				b.Error(errors.New("Beanstalkd connection not initated or lost. Please check your beanstalkd server or block settings."))
			}
		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				"Host": host,
				"Tube": tube,
				"TTR":  ttr,
			}
		}
	}
}
Esempio n. 23
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Sync) Run() {
	var lagString, path string
	var tree *jee.TokenTree
	lag := time.Duration(0)
	emitTick := time.NewTimer(500 * time.Millisecond)
	pq := &PriorityQueue{}
	heap.Init(pq)
	for {
		select {
		case <-emitTick.C:
		case ruleI := <-b.inrule:
			// set a parameter of the block
			lagString, err := util.ParseString(ruleI, "Lag")
			if err != nil {
				b.Error(err)
				break
			}
			lag, err = time.ParseDuration(lagString)
			if err != nil {
				b.Error(err)
				continue
			}
			path, err = util.ParseString(ruleI, "Path")
			if err != nil {
				b.Error(err)
				break
			}
			// build the parser for the model
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				break
			}
			tI, err := jee.Eval(tree, interface{}(msg))
			if err != nil {
				b.Error(err)
			}
			t, ok := tI.(float64)
			if !ok {
				b.Error(errors.New("couldn't convert time value to float64"))
				continue
			}
			ms := time.Unix(0, int64(t*1000000))
			queueMessage := &PQMessage{
				val: msg,
				t:   ms,
			}
			heap.Push(pq, queueMessage)

		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				"Lag":  lagString,
				"Path": path,
			}

		}
		now := time.Now()
		for {
			item, diff := pq.PeekAndShift(now, lag)
			if item == nil {
				// then the queue is empty. Pause for 5 seconds before checking again
				if diff == 0 {
					diff = time.Duration(500) * time.Millisecond
				}
				emitTick.Reset(diff)
				break
			}
			b.out <- item.(*PQMessage).val.(map[string]interface{})
		}

	}
}
Esempio n. 24
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *MovingAverage) Run() {
	var tree *jee.TokenTree
	var path, windowString string
	var err error
	window := time.Duration(0)
	waitTimer := time.NewTimer(100 * time.Millisecond)

	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			path, err = util.ParseString(ruleI, "Path")
			if err != nil {
				b.Error(err)
			}
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				break
			}
			windowString, err = util.ParseString(ruleI, "Window")
			if err != nil {
				b.Error(err)
			}
			window, err = time.ParseDuration(windowString)
			if err != nil {
				b.Error(err)
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				break
			}
			val, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			// TODO make this a type swtich and convert anything we can to a
			// float
			val, ok := val.(float64)
			if !ok {
				b.Error(errors.New("trying to put a non-float into the moving average"))
				continue
			}
			queueMessage := &PQMessage{
				val: val,
				t:   time.Now(),
			}
			heap.Push(pq, queueMessage)
		case <-b.inpoll:
			// deal with a poll request
			outMsg := map[string]interface{}{
				"Average": pqAverage(pq),
			}
			b.out <- outMsg
		case c := <-b.queryavg:
			outMsg := map[string]interface{}{
				"Average": pqAverage(pq),
			}
			c <- outMsg
		case c := <-b.queryrule:
			// deal with a query request
			c <- map[string]interface{}{
				"Path":   path,
				"Window": windowString,
			}
		case <-waitTimer.C:
		}
		for {
			pqMsg, diff := pq.PeekAndShift(time.Now(), window)
			if pqMsg == nil {
				// either the queue is empty, or it's not time to emit
				if diff == 0 {
					// then the queue is empty. Pause for 5 seconds before checking again
					diff = time.Duration(500) * time.Millisecond
				}
				waitTimer.Reset(diff)
				break
			}
		}
	}
}
Esempio n. 25
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Redis) Run() {
	var server = "localhost:6379"
	var password string
	var command string
	var arguments = make([]string, 0, 10)
	var argumentTrees []*jee.TokenTree
	var err error
	var pool *redis.Pool

	for {
		select {
		case ruleI := <-b.inrule:
			server, err = util.ParseString(ruleI, "Server")
			if err != nil {
				b.Error(err)
				continue
			}
			password, err = util.ParseString(ruleI, "Password")
			if err != nil {
				b.Error(err)
				continue
			}
			command, err = util.ParseString(ruleI, "Command")
			if err != nil {
				b.Error(err)
				continue
			}

			if util.KeyExists(ruleI, "Arguments") {
				arguments, err = util.ParseArrayString(ruleI, "Arguments")
				if err != nil {
					b.Error(err)
					continue
				}
			}

			if len(arguments) > 0 {
				argumentTrees = make([]*jee.TokenTree, len(arguments))
				for i, path := range arguments {
					token, err := jee.Lexer(path)
					if err != nil {
						b.Error(err)
						continue
					}
					tree, err := jee.Parser(token)
					if err != nil {
						b.Error(err)
						continue
					}
					argumentTrees[i] = tree
				}
			}
			pool = newPool(server, password)

		case responseChan := <-b.queryrule:
			// deal with a query request
			responseChan <- map[string]interface{}{
				"Server":    server,
				"Password":  password,
				"Command":   command,
				"Arguments": arguments,
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if pool == nil {
				b.Error("not connected to redis")
				break
			}

			conn := pool.Get()

			args := make([]interface{}, len(argumentTrees))
			for i, tree := range argumentTrees {
				argument, err := jee.Eval(tree, msg)
				if err != nil {
					b.Error(err)
					break
				}
				args[i] = argument
			}

			// commands like 'KEYS *' or 'SET NUMBERS 1'
			reply, err := conn.Do(command, args...)
			if err != nil {
				b.Error(err)
				break
			}

			conn.Close()

			nicerReply, err := formatReply(reply)
			out := map[string]interface{}{
				"response": nicerReply,
			}
			b.out <- out
		}
	}
}
Esempio n. 26
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Javascript) Run() {
	messageIn := "input"
	messageOut := "output"
	script := `output = input`

	vm := otto.New()
	program, _ := vm.Compile("javascript", script)

	for {
		select {
		case ruleI := <-b.inrule:
			tmpMin, err := util.ParseString(ruleI, "MessageIn")
			if err != nil {
				b.Error(err)
				break
			}

			tmpMout, err := util.ParseString(ruleI, "MessageOut")
			if err != nil {
				b.Error(err)
				break
			}

			tmpScript, err := util.ParseString(ruleI, "Script")
			if err != nil {
				b.Error(err)
				break
			}

			tmpProgram, err := vm.Compile("javascript", tmpScript)
			if err != nil {
				b.Error(err)
				break
			}

			messageIn = tmpMin
			messageOut = tmpMout
			script = tmpScript
			program = tmpProgram

		case <-b.quit:
			// quit the block
			return
		case m := <-b.in:
			if program == nil {
				break
			}

			err := vm.Set(messageOut, map[string]interface{}{})
			if err != nil {
				b.Error(err)
				break
			}

			err = vm.Set(messageIn, m)
			if err != nil {
				b.Error(err)
				break
			}

			_, err = vm.Run(program)
			if err != nil {
				b.Error(err)
				break
			}

			g, err := vm.Get(messageOut)
			if err != nil {
				b.Error(err)
				break
			}
			o, err := g.Export()
			if err != nil {
				b.Error(err)
				break
			}

			b.out <- o
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"MessageIn":  messageIn,
				"MessageOut": messageOut,
				"Script":     script,
			}
		}
	}
}
Esempio n. 27
0
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
	var err error
	var conn *amqp.Connection
	var amqp_chan *amqp.Channel
	toOut := make(blocks.MsgChan)
	toError := make(chan error)

	host := "localhost"
	port := "5672"
	username := "******"
	password := "******"
	routingkey := "#"
	exchange := "amq.topic"
	exchange_type := "topic"

	for {
		select {
		case msg := <-toOut:
			b.out <- msg
		case err := <-toError:
			b.Error(err)
		case ruleI := <-b.inrule:
			rule := ruleI.(map[string]interface{})

			routingkey, err = util.ParseString(rule, "RoutingKey")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange, err = util.ParseString(rule, "Exchange")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange_type, err = util.ParseString(rule, "ExchangeType")
			if err != nil {
				b.Error(err)
				continue
			}
			host, err = util.ParseString(rule, "Host")
			if err != nil {
				b.Error(err)
				continue
			}
			port, err = util.ParseString(rule, "Port")
			if err != nil {
				b.Error(err)
				continue
			}
			username, err = util.ParseString(rule, "Username")
			if err != nil {
				b.Error(err)
				continue
			}
			password, err = util.ParseString(rule, "Password")
			if err != nil {
				b.Error(err)
				continue
			}

			conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
			if err != nil {
				b.Error(err)
				continue
			}

			amqp_chan, err = conn.Channel()
			if err != nil {
				b.Error(err)
				continue
			}

			err = amqp_chan.ExchangeDeclare(
				exchange,      // name
				exchange_type, // type
				true,          // durable
				false,         // auto-deleted
				false,         // internal
				false,         // noWait
				nil,           // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			queue, err := amqp_chan.QueueDeclare(
				"",    // name
				false, // durable
				true,  // delete when unused
				false, // exclusive
				false, // noWait
				nil,   // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			err = amqp_chan.QueueBind(
				queue.Name, // queue name
				routingkey, // routing key
				exchange,   // exchange
				false,
				nil,
			)

			if err != nil {
				b.Error(err)
				continue
			}

			deliveries, err := amqp_chan.Consume(
				queue.Name, // name
				"",         // consumerTag
				true,       // noAck
				false,      // exclusive
				false,      // noLocal
				false,      // noWait
				nil,        // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			h := readWriteAMQPHandler{toOut, toError}
			go h.handle(deliveries)
		case <-b.quit:
			if amqp_chan != nil {
				amqp_chan.Close()
			}
			if conn != nil {
				conn.Close()
			}
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Host":         host,
				"Port":         port,
				"Username":     username,
				"Password":     password,
				"Exchange":     exchange,
				"ExchangeType": exchange_type,
				"RoutingKey":   routingkey,
			}
		}
	}
}
Esempio n. 28
0
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToNSQMulti) Run() {
	var err error
	var nsqdTCPAddrs string
	var topic string
	var writer *nsq.Producer
	var batch [][]byte
	interval := time.Duration(1 * time.Second)
	maxBatch := 100

	conf := nsq.NewConfig()

	dump := time.NewTicker(interval)
	for {
		select {
		case <-dump.C:
			if writer == nil || len(batch) == 0 {
				break
			}
			err = writer.MultiPublish(topic, batch)
			if err != nil {
				b.Error(err.Error())
			}

			batch = nil
		case ruleI := <-b.inrule:
			//rule := ruleI.(map[string]interface{})

			topic, err = util.ParseString(ruleI, "Topic")
			if err != nil {
				b.Error(err)
				break
			}

			nsqdTCPAddrs, err = util.ParseString(ruleI, "NsqdTCPAddrs")
			if err != nil {
				b.Error(err)
				break
			}

			intervalS, err := util.ParseString(ruleI, "Interval")
			if err != nil {
				b.Error("bad input")
				break
			}

			dur, err := time.ParseDuration(intervalS)
			if err != nil {
				b.Error(err)
				break
			}

			if dur <= 0 {
				b.Error("interval must be positive")
				break
			}

			batchSize, err := util.ParseFloat(ruleI, "MaxBatch")
			if err != nil {
				b.Error("error parsing batch size")
				break
			}

			if writer != nil {
				writer.Stop()
			}

			maxBatch = int(batchSize)
			interval = dur

			dump.Stop()
			dump = time.NewTicker(interval)
			writer, err = nsq.NewProducer(nsqdTCPAddrs, conf)
			if err != nil {
				b.Error(err)
				break
			}
			topic = topic
			nsqdTCPAddrs = nsqdTCPAddrs
		case msg := <-b.in:
			if writer == nil {
				break
			}

			msgByte, err := json.Marshal(msg)
			if err != nil {
				b.Error(err)
			}
			batch = append(batch, msgByte)

			if len(batch) > maxBatch {
				err := writer.MultiPublish(topic, batch)
				if err != nil {
					b.Error(err)
					break
				}
				batch = nil
			}
		case <-b.quit:
			if writer != nil {
				writer.Stop()
			}
			dump.Stop()
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Topic":        topic,
				"NsqdTCPAddrs": nsqdTCPAddrs,
				"MaxBatch":     maxBatch,
				"Interval":     interval.String(),
			}
		}
	}
}
Esempio n. 29
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Cache) Run() {
	var keyPath, valuePath, ttlString string
	var ttl time.Duration
	values := make(map[string]item)
	ttlQueue := &PriorityQueue{}

	var keyTree, valueTree *jee.TokenTree
	var err error
	emitTick := time.NewTimer(500 * time.Millisecond)
	for {
		select {
		case <-emitTick.C:

		case ruleI := <-b.inrule:
			keyPath, err = util.ParseString(ruleI, "KeyPath")
			keyTree, err = util.BuildTokenTree(keyPath)
			if err != nil {
				b.Error(err)
				break
			}
			valuePath, err = util.ParseString(ruleI, "ValuePath")
			valueTree, err = util.BuildTokenTree(valuePath)
			if err != nil {
				b.Error(err)
				break
			}
			ttlString, err = util.ParseString(ruleI, "TimeToLive")
			if err != nil {
				b.Error(err)
				break
			}
			ttl, err = time.ParseDuration(ttlString)
			if err != nil {
				b.Error(err)
				break
			}
		case <-b.quit:
			return
		case msg := <-b.lookup:
			if keyTree == nil {
				continue
			}
			kI, err := jee.Eval(keyTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			k, ok := kI.(string)
			if !ok {
				b.Error(errors.New("key must be a string"))
				continue
			}
			i, ok := values[k]
			var v interface{}
			if ok {
				v = i.value
				now := time.Now()
				i.lastSeen = now
				queueMessage := &PQMessage{
					val: k,
					t:   now,
				}
				heap.Push(ttlQueue, queueMessage)
			}
			b.out <- map[string]interface{}{
				"key":   k,
				"value": v,
			}
		case msg := <-b.in:
			if keyTree == nil {
				continue
			}
			if valueTree == nil {
				continue
			}
			kI, err := jee.Eval(keyTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			k, ok := kI.(string)
			if !ok {
				b.Error(errors.New("key must be a string"))
				continue
			}
			v, err := jee.Eval(valueTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			now := time.Now()
			values[k] = item{
				value:    v,
				lastSeen: now,
			}
			queueMessage := &PQMessage{
				val: k,
				t:   now,
			}
			heap.Push(ttlQueue, queueMessage)
		case responseChan := <-b.queryrule:
			// deal with a query request
			responseChan <- map[string]interface{}{
				"KeyPath":    keyPath,
				"ValuePath":  valuePath,
				"TimeToLive": ttlString,
			}
		}
		now := time.Now()
		for {
			itemI, diff := ttlQueue.PeekAndShift(now, ttl)
			if itemI == nil {
				// then the queue is empty. don't check again for 5s
				if diff == 0 {
					diff = time.Duration(500) * time.Millisecond
				}
				emitTick.Reset(diff)
				break
			}
			i := itemI.(*PQMessage)
			k := i.val.(string)
			if i.t.Equal(values[k].lastSeen) {
				delete(values, k)
			}
		}
	}
}
Esempio n. 30
0
func (b *KullbackLeibler) Run() {
	var qtree, ptree *jee.TokenTree
	var qpath, ppath string
	var err error
	for {
		select {
		case ruleI := <-b.inrule:
			qpath, err = util.ParseString(ruleI, "QPath")
			if err != nil {
				b.Error(err)
			}
			qtree, err = util.BuildTokenTree(qpath)
			ppath, err = util.ParseString(ruleI, "PPath")
			if err != nil {
				b.Error(err)
			}
			ptree, err = util.BuildTokenTree(ppath)
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"QPath": qpath,
				"PPath": ppath,
			}
		case <-b.quit:
			return
		case msg := <-b.in:
			if ptree == nil {
				continue
			}
			if qtree == nil {
				continue
			}
			pI, err := jee.Eval(ptree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			qI, err := jee.Eval(qtree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			p, ok := newHistogram(pI)
			if !ok {
				b.Error(errors.New("p is not a Histogram"))
				continue
			}
			q, ok := newHistogram(qI)
			if !ok {
				b.Error(errors.New("q is not a Histogram"))
				continue
			}
			q.normalise(p)
			p.normalise(q)
			kl := 0.0
			for k, pi := range p {
				kl += math.Log(pi/q[k]) * pi
			}
			b.out <- map[string]interface{}{
				"KL": kl,
			}
		}
	}
}