예제 #1
0
파일: public.go 프로젝트: sensorbee/twitter
func getKeyParameters(params data.Map) (*apiKey, error) {
	// "key_file" parameter is preferred to other key parameters.
	if v, ok := params["key_file"]; ok {
		path, err := data.AsString(v)
		if err != nil {
			return nil, fmt.Errorf("key_file parameter must be a string: %v", v)
		}
		return loadKeyFile(path)
	}

	// look for other key parameters
	keys := &apiKey{}
	keyVars := []*string{&keys.ConsumerKey, &keys.ConsumerSecret,
		&keys.AccessToken, &keys.AccessTokenSecret}
	for i, p := range []string{"consumer_key", "consumer_secret",
		"access_token", "access_token_secret"} {
		v, ok := params[p]
		if !ok {
			return nil, fmt.Errorf("key_file or %v parameter is missing", p)
		}

		k, err := data.AsString(v)
		if err != nil {
			return nil, fmt.Errorf("%v parameter must be a string: %v", p, v)
		}
		*keyVars[i] = k
	}
	return keys, nil
}
예제 #2
0
func ExtractParamAsString(params data.Map, key string) (string, error) {
	v, ok := params[key]
	if !ok {
		return "", fmt.Errorf("%s parameter is missing", key)
	}
	s, err := data.AsString(v)
	if err != nil {
		return "", fmt.Errorf("%s parameter is not a string: %v", err)
	}
	return s, nil
}
예제 #3
0
func ExtractParamAsStringWithDefault(params data.Map, key, def string) (string, error) {
	v, ok := params[key]
	if !ok {
		return def, nil
	}

	s, err := data.AsString(v)
	if err != nil {
		return "", fmt.Errorf("%s parameter is not a string: %v", key, err)
	}
	return s, nil
}
예제 #4
0
파일: mnist.go 프로젝트: sensorbee/jubatus
func createSource(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Source, error) {
	v, ok := params["path"]
	if !ok {
		return nil, errors.New("path parameter is missing")
	}
	path, err := data.AsString(v)
	if err != nil {
		return nil, fmt.Errorf("path parameter is not a string: %v", err)
	}

	s := &source{path}
	return core.NewRewindableSource(perline.NewSource(s, s, 1)), nil
}
예제 #5
0
파일: source.go 프로젝트: sensorbee/fluentd
// NewSource create a new Source receiving data from fluentd's out_forward.
func NewSource(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Source, error) {
	s := &source{
		ioParams: ioParams,
		bind:     "127.0.0.1:24224",
		tagField: "tag",
	}

	if v, ok := params["bind"]; ok {
		b, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		s.bind = b
	}

	if v, ok := params["tag_field"]; ok {
		t, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		s.tagField = t
	}

	// TODO: optionally support logging
	logger, err := logging.GetLogger("fluentd-forwarder")
	if err != nil {
		return nil, err
	}
	logger.SetBackend(logging.AddModuleLevel(logging.NewLogBackend(&nullWriter{}, "", 0)))

	input, err := fluentd_forwarder.NewForwardInput(logger, s.bind, s)
	if err != nil {
		return nil, err
	}

	s.input = input
	return s, nil
}
예제 #6
0
파일: sink.go 프로젝트: sensorbee/fluentd
func getDuration(params data.Map, field string, d *time.Duration) error {
	v, ok := params[field]
	if !ok {
		return nil
	}

	// TODO: support other types
	str, err := data.AsString(v)
	if err != nil {
		return err
	}

	*d, err = time.ParseDuration(str)
	return err
}
예제 #7
0
// Processe implements udf.UDSF.Process. It tokenizes a field of tuples.
func (t *Tokenizer) Process(ctx *core.Context, tuple *core.Tuple, w core.Writer) error {
	var kwd []string
	if v, ok := tuple.Data[t.field]; !ok {
		return fmt.Errorf("the tuple doesn't have the required field: %v", t.field)
	} else if s, err := data.AsString(v); err != nil {
		return fmt.Errorf("'%v' field must be string: %v", t.field, err)
	} else {
		kwd = strings.Split(s, " ")
	}

	for _, k := range kwd {
		out := tuple.Copy()
		out.Data[t.field] = data.String(k)
		if err := w.Write(ctx, out); err != nil {
			return err
		}
	}
	return nil
}
예제 #8
0
// Write trains the machine learning model the state has with a given tuple.
func (a *AROWState) Write(ctx *core.Context, t *core.Tuple) error {
	vlabel, ok := t.Data[a.labelField]
	if !ok {
		return fmt.Errorf("%s field is missing", a.labelField)
	}
	label, err := data.AsString(vlabel)
	if err != nil {
		return fmt.Errorf("%s value is not a string: %v", a.labelField, err)
	}

	vfv, ok := t.Data[a.featureVectorField]
	if !ok {
		return fmt.Errorf("%s field is missing", a.featureVectorField)
	}
	fv, err := data.AsMap(vfv)
	if err != nil {
		return fmt.Errorf("%s value is not a map: %v", a.labelField, err)
	}

	err = a.train(fv, label)
	return err
}
예제 #9
0
파일: sink.go 프로젝트: sensorbee/fluentd
// NewSink creates a new Sink for SensorBee which sends tuples to fluentd.
func NewSink(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Sink, error) {
	conf := struct {
		// fluentd parameters
		forwardTo           string
		retryInterval       time.Duration
		connectionTimeout   time.Duration
		writeTimeout        time.Duration
		flushInterval       time.Duration
		journalGroupPath    string
		maxJournalChunkSize int64

		// plugin parameters
		tagField       string
		defaultTag     string
		removeTagField bool
	}{
		forwardTo:           "127.0.0.1:24224",
		retryInterval:       5 * time.Second,
		connectionTimeout:   10 * time.Second,
		writeTimeout:        10 * time.Second,
		flushInterval:       5 * time.Second,
		journalGroupPath:    "*", // TODO: check the meaning of this option carefully
		maxJournalChunkSize: 16777216,

		tagField:       "tag",
		defaultTag:     "sensorbee",
		removeTagField: true,
	}

	if v, ok := params["forward_to"]; ok {
		f, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		conf.forwardTo = f
	}

	if err := getDuration(params, "retry_interval", &conf.retryInterval); err != nil {
		return nil, err
	}
	if err := getDuration(params, "connection_timeout", &conf.connectionTimeout); err != nil {
		return nil, err
	}
	if err := getDuration(params, "write_timeout", &conf.writeTimeout); err != nil {
		return nil, err
	}
	if err := getDuration(params, "flush_interval", &conf.flushInterval); err != nil {
		return nil, err
	}

	if v, ok := params["journal_group_path"]; ok {
		p, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		conf.journalGroupPath = p
	}

	if v, ok := params["max_journal_chunk_size"]; ok {
		s, err := data.AsInt(v) // TODO: support something like '100MB'
		if err != nil {
			return nil, err
		}
		conf.maxJournalChunkSize = s
	}

	if v, ok := params["tag_field"]; ok {
		f, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		conf.tagField = f
	}

	if v, ok := params["default_tag"]; ok {
		t, err := data.AsString(v)
		if err != nil {
			return nil, err
		}
		conf.defaultTag = t
	}

	if v, ok := params["remove_tag_field"]; ok {
		r, err := data.ToBool(v)
		if err != nil {
			return nil, err
		}
		conf.removeTagField = r
	}

	// TODO: optionally support logging
	logger, err := logging.GetLogger("fluentd-forwarder")
	if err != nil {
		return nil, err
	}
	logger.SetBackend(logging.AddModuleLevel(logging.NewLogBackend(&nullWriter{}, "", 0)))

	out, err := fluentd_forwarder.NewForwardOutput(
		logger,
		conf.forwardTo,
		conf.retryInterval,
		conf.connectionTimeout,
		conf.writeTimeout,
		conf.flushInterval,
		conf.journalGroupPath,
		conf.maxJournalChunkSize,
		"")
	if err != nil {
		return nil, err
	}

	s := &sink{
		output:         out,
		tagField:       conf.tagField,
		defaultTag:     conf.defaultTag,
		removeTagField: conf.removeTagField,
	}
	out.Start()
	return s, nil
}