Exemplo n.º 1
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *LogisticModel) Run() {

	var β []float64
	var featurePaths []string
	var featureTrees []*jee.TokenTree
	var err error

	for {
	Loop:
		select {
		case rule := <-b.inrule:
			β, err = util.ParseArrayFloat(rule, "Weights")
			if err != nil {
				b.Error(err)
				continue
			}
			featurePaths, err = util.ParseArrayString(rule, "FeaturePaths")
			if err != nil {
				b.Error(err)
				continue
			}
			featureTrees = make([]*jee.TokenTree, len(featurePaths))
			for i, path := range featurePaths {
				token, err := jee.Lexer(path)
				if err != nil {
					b.Error(err)
					break
				}
				tree, err := jee.Parser(token)
				if err != nil {
					b.Error(err)
					break
				}
				featureTrees[i] = tree
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if featureTrees == nil {
				continue
			}
			x := make([]float64, len(featureTrees))
			for i, tree := range featureTrees {
				feature, err := jee.Eval(tree, msg)
				if err != nil {
					b.Error(err)
					break Loop
				}
				fi, ok := feature.(float64)
				if !ok {
					b.Error(errors.New("features must be float64"))
					break Loop
				}
				x[i] = fi
			}
			μ := 0.0
			for i, βi := range β {
				μ += βi * x[i]
			}
			var y float64
			if rand.Float64() <= logit(μ) {
				y = 1
			} else {
				y = 0
			}
			b.out <- map[string]interface{}{
				"Response": y,
			}

		case MsgChan := <-b.queryrule:
			// deal with a query request
			out := map[string]interface{}{
				"Weights":      β,
				"FeaturePaths": featurePaths,
			}
			MsgChan <- out
		}
	}
}
Exemplo n.º 2
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Redis) Run() {
	var server = "localhost:6379"
	var password string
	var command string
	var arguments = make([]string, 0, 10)
	var argumentTrees []*jee.TokenTree
	var err error
	var pool *redis.Pool

	for {
		select {
		case ruleI := <-b.inrule:
			server, err = util.ParseString(ruleI, "Server")
			if err != nil {
				b.Error(err)
				continue
			}
			password, err = util.ParseString(ruleI, "Password")
			if err != nil {
				b.Error(err)
				continue
			}
			command, err = util.ParseString(ruleI, "Command")
			if err != nil {
				b.Error(err)
				continue
			}

			if util.KeyExists(ruleI, "Arguments") {
				arguments, err = util.ParseArrayString(ruleI, "Arguments")
				if err != nil {
					b.Error(err)
					continue
				}
			}

			if len(arguments) > 0 {
				argumentTrees = make([]*jee.TokenTree, len(arguments))
				for i, path := range arguments {
					token, err := jee.Lexer(path)
					if err != nil {
						b.Error(err)
						continue
					}
					tree, err := jee.Parser(token)
					if err != nil {
						b.Error(err)
						continue
					}
					argumentTrees[i] = tree
				}
			}
			pool = newPool(server, password)

		case responseChan := <-b.queryrule:
			// deal with a query request
			responseChan <- map[string]interface{}{
				"Server":    server,
				"Password":  password,
				"Command":   command,
				"Arguments": arguments,
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			if pool == nil {
				b.Error("not connected to redis")
				break
			}

			conn := pool.Get()

			args := make([]interface{}, len(argumentTrees))
			for i, tree := range argumentTrees {
				argument, err := jee.Eval(tree, msg)
				if err != nil {
					b.Error(err)
					break
				}
				args[i] = argument
			}

			// commands like 'KEYS *' or 'SET NUMBERS 1'
			reply, err := conn.Do(command, args...)
			if err != nil {
				b.Error(err)
				break
			}

			conn.Close()

			nicerReply, err := formatReply(reply)
			out := map[string]interface{}{
				"response": nicerReply,
			}
			b.out <- out
		}
	}
}
Exemplo n.º 3
0
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ParseCSV) Run() {
	var tree *jee.TokenTree
	var path string
	var err error
	var headers []string
	var csvReader *csv.Reader

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			path, err = util.ParseString(ruleI, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			token, err := jee.Lexer(path)
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				continue
			}

			headers, err = util.ParseArrayString(ruleI, "Headers")
			if err != nil {
				b.Error(err)
				continue
			}
		case <-b.quit:
			// quit the block
			return
		case msg := <-b.in:
			// deal with inbound data
			if tree == nil {
				continue
			}
			var data string

			dataI, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}

			switch value := dataI.(type) {
			case []byte:
				data = string(value[:])

			case string:
				data = value

			default:
				b.Error("data should be a string or a []byte")
				continue
			}

			csvReader = csv.NewReader(strings.NewReader(data))
			csvReader.TrimLeadingSpace = true
			// allow records to have variable numbers of fields
			csvReader.FieldsPerRecord = -1

		case <-b.inpoll:
			if csvReader == nil {
				b.Error("this block needs data to be pollable")
				break
			}
			record, err := csvReader.Read()
			if err != nil && err != io.EOF {
				b.Error(err)
				continue
			}
			row := make(map[string]interface{})
			for fieldIndex, field := range record {
				if fieldIndex >= len(headers) {
					row[strconv.Itoa(fieldIndex)] = field
				} else {
					header := headers[fieldIndex]
					row[header] = field
				}
			}

			b.out <- row

		case MsgChan := <-b.queryrule:
			// deal with a query request
			MsgChan <- map[string]interface{}{
				"Path":    path,
				"Headers": headers,
			}
		}
	}
}
Exemplo n.º 4
0
func (b *Learn) Run() {

	dataChan := make(chan sgd.Obs)
	paramChan := make(chan sgd.Params)
	stateChan := make(chan chan []float64)
	kernelQuitChan := make(chan bool)

	lossfuncs := map[string]sgd.LossFunc{
		"linear":   sgd.GradLinearLoss,
		"logistic": sgd.GradLogisticLoss,
	}
	stepfuncs := map[string]sgd.StepFunc{
		"inverse":  sgd.EtaInverse,
		"constant": sgd.EtaConstant,
		"bottou":   sgd.EtaBottou,
	}
	kernelStarted := false

	var responsePath, lossfuncString, stepfuncString string
	var featurePaths []string
	var θ_0 []float64
	var featureTrees []*jee.TokenTree
	var responseTree *jee.TokenTree
	var err error

	for {
	Loop:
		select {
		case rule := <-b.inrule:
			if kernelStarted {
				// if we already have a rule, then we've already started a
				// kernel, which we should now quit.
				kernelQuitChan <- true
			}

			featurePaths, err = util.ParseArrayString(rule, "FeaturePaths")
			if err != nil {
				b.Error(err)
				continue
			}
			featureTrees = make([]*jee.TokenTree, len(featurePaths))
			for i, path := range featurePaths {
				token, err := jee.Lexer(path)
				if err != nil {
					b.Error(err)
					break
				}
				tree, err := jee.Parser(token)
				if err != nil {
					b.Error(err)
					break
				}
				featureTrees[i] = tree
			}
			responsePath, err = util.ParseString(rule, "ResponsePath")
			if err != nil {
				b.Error(err)
				break
			}
			token, err := jee.Lexer(responsePath)
			if err != nil {
				b.Error(err)
				break
			}
			responseTree, err = jee.Parser(token)
			if err != nil {
				b.Error(err)
				break
			}
			lossfuncString, err = util.ParseString(rule, "Lossfunc")
			if err != nil {
				b.Error(err)
				break
			}
			stepfuncString, err = util.ParseString(rule, "Stepfunc")
			if err != nil {
				b.Error(err)
				break
			}
			grad, ok := lossfuncs[lossfuncString]
			if !ok {
				b.Error(errors.New("Unknown loss function: " + lossfuncString))
			}
			step, ok := stepfuncs[stepfuncString]
			if !ok {
				b.Error(errors.New("Unknown step function: " + stepfuncString))
			}
			θ_0, err = util.ParseArrayFloat(rule, "InitialState")
			if err != nil {
				b.Error(err)
				break
			}
			go sgd.SgdKernel(dataChan, paramChan, stateChan, kernelQuitChan, grad, step, θ_0)
			kernelStarted = true

		case <-b.quit:
			kernelQuitChan <- true
			return
		case msg := <-b.in:
			if featureTrees == nil {
				continue
			}
			if responseTree == nil {
				continue
			}
			x := make([]float64, len(featureTrees))
			for i, tree := range featureTrees {
				feature, err := jee.Eval(tree, msg)
				if err != nil {
					b.Error(err)
					break Loop
				}
				fi, ok := feature.(float64)
				if !ok {
					b.Error(errors.New("features must be float64"))
					break Loop
				}
				x[i] = fi
			}
			responseI, err := jee.Eval(responseTree, msg)
			if err != nil {
				b.Error(err)
				break
			}
			y, ok := responseI.(float64)
			if !ok {
				b.Error(errors.New("response must be float64"))
				break
			}
			d := sgd.Obs{
				X: x,
				Y: y,
			}
			dataChan <- d
		case <-b.inpoll:
			var params []interface{}
			var model []float64
			if kernelStarted {
				kernelMsgChan := make(chan []float64)
				stateChan <- kernelMsgChan
				model = <-kernelMsgChan
				params = make([]interface{}, len(model))
				for i, p := range model {
					params[i] = p
				}
			}
			b.out <- map[string]interface{}{
				"params": params,
			}
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Lossfunc":     lossfuncString,
				"Stepfunc":     stepfuncString,
				"FeaturePaths": featurePaths,
				"ResponsePath": responsePath,
				"InitialState": θ_0,
			}
		}
	}
}