func getKeyParameters(params data.Map) (*apiKey, error) {
// "key_file" parameter is preferred to other key parameters.
if v, ok := params["key_file"]; ok {
path, err := data.AsString(v)
if err != nil {
return nil, fmt.Errorf("key_file parameter must be a string: %v", v)
}
return loadKeyFile(path)
}
// look for other key parameters
keys := &apiKey{}
keyVars := []*string{&keys.ConsumerKey, &keys.ConsumerSecret,
&keys.AccessToken, &keys.AccessTokenSecret}
for i, p := range []string{"consumer_key", "consumer_secret",
"access_token", "access_token_secret"} {
v, ok := params[p]
if !ok {
return nil, fmt.Errorf("key_file or %v parameter is missing", p)
}
k, err := data.AsString(v)
if err != nil {
return nil, fmt.Errorf("%v parameter must be a string: %v", p, v)
}
*keyVars[i] = k
}
return keys, nil
}
func ExtractParamAsString(params data.Map, key string) (string, error) {
v, ok := params[key]
if !ok {
return "", fmt.Errorf("%s parameter is missing", key)
}
s, err := data.AsString(v)
if err != nil {
return "", fmt.Errorf("%s parameter is not a string: %v", err)
}
return s, nil
}
func ExtractParamAsStringWithDefault(params data.Map, key, def string) (string, error) {
v, ok := params[key]
if !ok {
return def, nil
}
s, err := data.AsString(v)
if err != nil {
return "", fmt.Errorf("%s parameter is not a string: %v", key, err)
}
return s, nil
}
func createSource(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Source, error) {
v, ok := params["path"]
if !ok {
return nil, errors.New("path parameter is missing")
}
path, err := data.AsString(v)
if err != nil {
return nil, fmt.Errorf("path parameter is not a string: %v", err)
}
s := &source{path}
return core.NewRewindableSource(perline.NewSource(s, s, 1)), nil
}
// NewSource create a new Source receiving data from fluentd's out_forward.
func NewSource(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Source, error) {
s := &source{
ioParams: ioParams,
bind: "127.0.0.1:24224",
tagField: "tag",
}
if v, ok := params["bind"]; ok {
b, err := data.AsString(v)
if err != nil {
return nil, err
}
s.bind = b
}
if v, ok := params["tag_field"]; ok {
t, err := data.AsString(v)
if err != nil {
return nil, err
}
s.tagField = t
}
// TODO: optionally support logging
logger, err := logging.GetLogger("fluentd-forwarder")
if err != nil {
return nil, err
}
logger.SetBackend(logging.AddModuleLevel(logging.NewLogBackend(&nullWriter{}, "", 0)))
input, err := fluentd_forwarder.NewForwardInput(logger, s.bind, s)
if err != nil {
return nil, err
}
s.input = input
return s, nil
}
func getDuration(params data.Map, field string, d *time.Duration) error {
v, ok := params[field]
if !ok {
return nil
}
// TODO: support other types
str, err := data.AsString(v)
if err != nil {
return err
}
*d, err = time.ParseDuration(str)
return err
}
// Processe implements udf.UDSF.Process. It tokenizes a field of tuples.
func (t *Tokenizer) Process(ctx *core.Context, tuple *core.Tuple, w core.Writer) error {
var kwd []string
if v, ok := tuple.Data[t.field]; !ok {
return fmt.Errorf("the tuple doesn't have the required field: %v", t.field)
} else if s, err := data.AsString(v); err != nil {
return fmt.Errorf("'%v' field must be string: %v", t.field, err)
} else {
kwd = strings.Split(s, " ")
}
for _, k := range kwd {
out := tuple.Copy()
out.Data[t.field] = data.String(k)
if err := w.Write(ctx, out); err != nil {
return err
}
}
return nil
}
// Write trains the machine learning model the state has with a given tuple.
func (a *AROWState) Write(ctx *core.Context, t *core.Tuple) error {
vlabel, ok := t.Data[a.labelField]
if !ok {
return fmt.Errorf("%s field is missing", a.labelField)
}
label, err := data.AsString(vlabel)
if err != nil {
return fmt.Errorf("%s value is not a string: %v", a.labelField, err)
}
vfv, ok := t.Data[a.featureVectorField]
if !ok {
return fmt.Errorf("%s field is missing", a.featureVectorField)
}
fv, err := data.AsMap(vfv)
if err != nil {
return fmt.Errorf("%s value is not a map: %v", a.labelField, err)
}
err = a.train(fv, label)
return err
}
// NewSink creates a new Sink for SensorBee which sends tuples to fluentd.
func NewSink(ctx *core.Context, ioParams *bql.IOParams, params data.Map) (core.Sink, error) {
conf := struct {
// fluentd parameters
forwardTo string
retryInterval time.Duration
connectionTimeout time.Duration
writeTimeout time.Duration
flushInterval time.Duration
journalGroupPath string
maxJournalChunkSize int64
// plugin parameters
tagField string
defaultTag string
removeTagField bool
}{
forwardTo: "127.0.0.1:24224",
retryInterval: 5 * time.Second,
connectionTimeout: 10 * time.Second,
writeTimeout: 10 * time.Second,
flushInterval: 5 * time.Second,
journalGroupPath: "*", // TODO: check the meaning of this option carefully
maxJournalChunkSize: 16777216,
tagField: "tag",
defaultTag: "sensorbee",
removeTagField: true,
}
if v, ok := params["forward_to"]; ok {
f, err := data.AsString(v)
if err != nil {
return nil, err
}
conf.forwardTo = f
}
if err := getDuration(params, "retry_interval", &conf.retryInterval); err != nil {
return nil, err
}
if err := getDuration(params, "connection_timeout", &conf.connectionTimeout); err != nil {
return nil, err
}
if err := getDuration(params, "write_timeout", &conf.writeTimeout); err != nil {
return nil, err
}
if err := getDuration(params, "flush_interval", &conf.flushInterval); err != nil {
return nil, err
}
if v, ok := params["journal_group_path"]; ok {
p, err := data.AsString(v)
if err != nil {
return nil, err
}
conf.journalGroupPath = p
}
if v, ok := params["max_journal_chunk_size"]; ok {
s, err := data.AsInt(v) // TODO: support something like '100MB'
if err != nil {
return nil, err
}
conf.maxJournalChunkSize = s
}
if v, ok := params["tag_field"]; ok {
f, err := data.AsString(v)
if err != nil {
return nil, err
}
conf.tagField = f
}
if v, ok := params["default_tag"]; ok {
t, err := data.AsString(v)
if err != nil {
return nil, err
}
conf.defaultTag = t
}
if v, ok := params["remove_tag_field"]; ok {
r, err := data.ToBool(v)
if err != nil {
return nil, err
}
conf.removeTagField = r
}
// TODO: optionally support logging
logger, err := logging.GetLogger("fluentd-forwarder")
if err != nil {
return nil, err
}
logger.SetBackend(logging.AddModuleLevel(logging.NewLogBackend(&nullWriter{}, "", 0)))
out, err := fluentd_forwarder.NewForwardOutput(
logger,
conf.forwardTo,
conf.retryInterval,
conf.connectionTimeout,
conf.writeTimeout,
conf.flushInterval,
conf.journalGroupPath,
conf.maxJournalChunkSize,
"")
if err != nil {
return nil, err
}
s := &sink{
output: out,
tagField: conf.tagField,
defaultTag: conf.defaultTag,
removeTagField: conf.removeTagField,
}
out.Start()
return s, nil
}