func TimeExtractFunc(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
switch len(items) {
case 0:
// if we have no "items", return time associated with ctx
// This is an alias of now()
t := ctx.Ts()
if !t.IsZero() {
return value.NewStringValue(t.String()), true
}
return value.EmptyStringValue, false
case 1:
// if only 1 item, convert item to time
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
t, err := dateparse.ParseAny(dateStr)
if err != nil {
return value.EmptyStringValue, false
}
return value.NewStringValue(t.String()), true
case 2:
// if we have 2 items, the first is the time string
// and the second is the format string.
// Use leekchan/timeutil package
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
formatStr, ok := value.ToString(items[1].Rv())
if !ok {
return value.EmptyStringValue, false
}
t, err := dateparse.ParseAny(dateStr)
if err != nil {
return value.EmptyStringValue, false
}
formatted := timeutil.Strftime(&t, formatStr)
return value.NewStringValue(formatted), true
default:
return value.EmptyStringValue, false
}
}
func Cast(valType ValueType, val Value) (Value, error) {
switch valType {
case ByteSliceType:
return NewByteSliceValue([]byte(val.ToString())), nil
case TimeType:
switch valt := val.(type) {
case StringValue:
if t, err := dateparse.ParseAny(valt.Val()); err == nil {
return NewTimeValue(t), nil
} else {
return nil, err
}
case TimeValue:
return valt, nil
}
case StringType:
sv := val.ToString()
return NewStringValue(sv), nil
case IntType:
iv, ok := ToInt64(val.Rv())
if ok {
return NewIntValue(iv), nil
}
return nil, ErrConversion
}
return nil, ErrConvestionNotSupported
}
// Get year in integer from field, must be able to convert to date
//
// yy() => 15, true // assuming it is 2015
// yy("2014-03-01") => 14, true
//
func Yy(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
yy := 0
if len(items) == 0 {
if !ctx.Ts().IsZero() {
yy = ctx.Ts().Year()
} else {
// Do we want to use Now()?
}
} else if len(items) == 1 {
//u.Debugf("has 1 items? %#v", items[0].Rv())
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Debugf("v=%v %v", dateStr, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err != nil {
return value.NewIntValue(0), false
} else {
yy = t.Year()
}
} else {
return value.NewIntValue(0), false
}
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Debugf("yy = %v", yy)
return value.NewIntValue(int64(yy)), true
}
func TestStaticDataSource(t *testing.T) {
static := NewStaticDataSource("users", 0, nil, []string{"user_id", "name", "email", "created", "roles"})
created, _ := dateparse.ParseAny("2015/07/04")
static.Put(nil, &datasource.KeyInt{123}, []driver.Value{123, "aaron", "*****@*****.**", created.In(time.UTC), []string{"admin"}})
assert.Tf(t, static.Length() == 1, "has 1 rows after Put()")
row, _ := static.Get(123)
assert.Tf(t, row != nil, "Should find row with Get() part of Seeker interface")
di, ok := row.Body().(*datasource.SqlDriverMessageMap)
assert.Tf(t, ok, "Must be []driver.Value type: %T", row.Body())
vals := di.Values()
assert.Tf(t, len(vals) == 5, "want 5 cols in user but got %v", len(vals))
assert.Tf(t, vals[0].(int) == 123, "want user_id=123 but got %v", vals[0])
assert.Tf(t, vals[2].(string) == "*****@*****.**", "want [email protected] but got %v", vals[2])
static.Put(nil, &datasource.KeyInt{123}, []driver.Value{123, "aaron", "*****@*****.**", created.In(time.UTC), []string{"root", "admin"}})
assert.Tf(t, static.Length() == 1, "has 1 rows after Put()")
row, _ = static.Get(123)
assert.Tf(t, row != nil, "Should find row with Get() part of Seeker interface")
vals2 := row.Body().(*datasource.SqlDriverMessageMap).Values()
assert.Tf(t, vals2[2].(string) == "*****@*****.**", "want [email protected] but got %v", vals2[2])
assert.Equal(t, []string{"root", "admin"}, vals2[4], "Roles should match updated vals")
assert.Equal(t, created, vals2[3], "created date should match updated vals")
}
// todate: convert to Date
//
// todate(field) uses araddon\dateparse util to recognize formats
//
// todate("01/02/2006", field ) uses golang date parse rules
// first parameter is the layout/format
//
//
func ToDate(ctx expr.EvalContext, items ...value.Value) (value.TimeValue, bool) {
if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewTimeValue(t), true
}
} else if len(items) == 2 {
dateStr, ok := value.ToString(items[1].Rv())
if !ok {
return value.TimeZeroValue, false
}
formatStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("hello layout=%v time=%v", formatStr, dateStr)
if t, err := time.Parse(formatStr, dateStr); err == nil {
return value.NewTimeValue(t), true
}
}
return value.TimeZeroValue, false
}
// Get year in integer from date
func Yy(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
yy := 0
if len(items) == 0 {
if !ctx.Ts().IsZero() {
yy = ctx.Ts().Year()
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err != nil {
return value.NewIntValue(0), false
} else {
yy = t.Year()
}
} else {
return value.NewIntValue(0), false
}
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Infof("%v yy = %v", item, yy)
return value.NewIntValue(int64(yy)), true
}
func TestMemDb(t *testing.T) {
created, _ := dateparse.ParseAny("2015/07/04")
db, err := NewMemDb("users", []string{"user_id", "name", "email", "created", "roles"})
assert.Tf(t, err == nil, "wanted no error got %v", err)
c, err := db.Open("users")
assert.Tf(t, err == nil, "wanted no error got %v", err)
dc, ok := c.(schema.ConnAll)
assert.T(t, ok)
dc.Put(nil, &datasource.KeyInt{123}, []driver.Value{123, "aaron", "*****@*****.**", created.In(time.UTC), []string{"admin"}})
row, err := dc.Get(123)
assert.T(t, err == nil)
assert.Tf(t, row != nil, "Should find row with Get() part of Seeker interface")
di, ok := row.(*datasource.SqlDriverMessage)
assert.Tf(t, ok, "Must be []driver.Value type: %T", row)
vals := di.Vals
assert.Tf(t, len(vals) == 5, "want 5 cols in user but got %v", len(vals))
assert.Tf(t, vals[0].(int) == 123, "want user_id=123 but got %v", vals[0])
assert.Tf(t, vals[2].(string) == "*****@*****.**", "want [email protected] but got %v", vals[2])
dc.Put(nil, &datasource.KeyInt{123}, []driver.Value{123, "aaron", "*****@*****.**", created.In(time.UTC), []string{"root", "admin"}})
row, _ = dc.Get(123)
assert.Tf(t, row != nil, "Should find row with Get() part of Seeker interface")
vals2 := row.Body().([]driver.Value)
assert.Tf(t, vals2[2].(string) == "*****@*****.**", "want [email protected] but got %v", vals2[2])
assert.Equal(t, []string{"root", "admin"}, vals2[4], "Roles should match updated vals")
assert.Equal(t, created, vals2[3], "created date should match updated vals")
}
func guessValueType(val string) value.ValueType {
if _, err := strconv.ParseInt(val, 10, 64); err == nil {
return value.IntType
} else if _, err := strconv.ParseBool(val); err == nil {
return value.IntType
} else if _, err := strconv.ParseFloat(val, 64); err == nil {
return value.NumberType
} else if _, err := dateparse.ParseAny(val); err == nil {
return value.TimeType
}
return value.StringType
}
// totimestamp: convert to date, then to unix Seconds
//
func ToTimestamp(ctx expr.EvalContext, item value.Value) (value.IntValue, bool) {
dateStr, ok := value.ToString(item.Rv())
if !ok {
return value.NewIntValue(0), false
}
if t, err := dateparse.ParseAny(dateStr); err == nil {
//u.Infof("v=%v %v unix=%v", item, item.Rv(), t.Unix())
return value.NewIntValue(int64(t.Unix())), true
}
return value.NewIntValue(0), false
}
// todate
func ToDate(ctx expr.EvalContext, item value.Value) (value.TimeValue, bool) {
dateStr, ok := value.ToString(item.Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewTimeValue(t), true
}
return value.TimeZeroValue, false
}
func (m *TimeValue) Scan(src interface{}) error {
//u.Debugf("scan: '%v'", src)
var t time.Time
switch val := src.(type) {
case string:
//u.Infof("trying to scan string: '%v'", val)
t2, err := dateparse.ParseAny(val)
if err == nil {
*m = TimeValue(t2)
return nil
}
//u.Infof("%v %v", t2, err)
err = json.Unmarshal([]byte(val), &t)
if err == nil {
*m = TimeValue(t)
} else {
u.Warnf("error for %q err=%v", val, err)
return err
}
case []byte:
t2, err := dateparse.ParseAny(string(val))
if err == nil {
*m = TimeValue(t2)
return nil
}
err = json.Unmarshal(val, &t)
if err == nil {
*m = TimeValue(t)
} else {
return err
}
case nil:
return nil
default:
u.Warnf("unknown type: %T", m)
return errors.New("Incompatible type for TimeValue")
}
return nil
}
// hour of day [0-23]
func HourOfDay(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
if len(items) == 0 {
if !ctx.Ts().IsZero() {
return value.NewIntValue(int64(ctx.Ts().Hour())), true
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", v, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewIntValue(int64(t.Hour())), true
}
}
return value.NewIntValue(0), false
}
// Get yymm in 4 digits from argument if supplied, else uses message context ts
//
func YyMm(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
if len(items) == 0 {
if !ctx.Ts().IsZero() {
t := ctx.Ts()
return value.NewStringValue(t.Format(yymmTimeLayout)), true
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
//u.Infof("v=%v %v ", v, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewStringValue(t.Format(yymmTimeLayout)), true
}
}
return value.EmptyStringValue, false
}
func Yy(ctx expr.EvalContext, item value.Value) (value.IntValue, bool) {
//u.Info("yy: %T", item)
val, ok := value.ToString(item.Rv())
if !ok || val == "" {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", val, item.Rv())
if t, err := dateparse.ParseAny(val); err == nil {
yy := t.Year()
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Infof("Yy = %v yy = %v", item, yy)
return value.NewIntValue(int64(yy)), true
}
return value.NewIntValue(0), false
}
func TestStructWrapper(t *testing.T) {
t1, _ := dateparse.ParseAny("12/18/2015")
tr := true
user := &User{
Name: "Yoda",
Created: t1,
Updated: &t1,
Authenticated: true,
HasSession: &tr,
Roles: []string{"admin", "api"},
BankAmount: 55.5,
}
readers := []expr.ContextReader{
datasource.NewContextWrapper(user),
datasource.NewContextSimpleNative(map[string]interface{}{
"str1": "str1",
"int1": 1,
"t1": t1,
"Name": "notyoda",
}),
}
nc := datasource.NewNestedContextReader(readers, time.Now())
expected := value.NewMapValue(map[string]interface{}{
"str1": "str1",
"int1": 1,
"Name": "Yoda",
"Authenticated": true,
"bankamount": 55.5,
"FullName": "Yoda, Jedi",
"Roles": []string{"admin", "api"},
})
for k, v := range expected.Val() {
//u.Infof("k:%v v:%#v", k, v)
checkval(t, nc, k, v)
}
}
// This formatter reads go files and performs:
// 1. Squashes multiple lines into one (as needed), Tries to squash panics(go) into one line
// 2. Reads out the LineType/Level [DEBUG,INFO,METRIC] into a field
//
// This expects log files in this format
// 2013-05-25 13:25:32.475 authctx.go:169: [DEBUG] sink Building sink for kafka from factory method
func MakeFileFlattener(filename string, msgChan chan *LineEvent) func(string) {
// Builder used to build the colored string.
buf := new(bytes.Buffer)
startsDate := false
prevWasDate := false
pos := 0
posEnd := 0
var dataType []byte
var loglevel string
var dateStr, prevDateStr string
var prevLogTs time.Time
lineCt := 0
return func(line string) {
lineCt++
if len(line) < 8 {
buf.WriteString(line)
return
}
startsDate = false
spaceCt := 0
// [DATE] [SOURCE] [LEVEL] [MESSAGE]
// 2014/07/10 11:04:20.653185 filter_fluentd.go:16: [DEBUG] %s
for i := 0; i < len(line); i++ {
r := line[i]
if r == ' ' {
if spaceCt == 1 {
dateStr = string(line[:i])
if dts, err := dateparse.ParseAny(dateStr); err == nil {
startsDate = true
defer func() {
// defer will run after prevDateStr already used to send message
prevLogTs = dts
prevDateStr = dateStr
}()
}
break
}
spaceCt++
}
}
// Find first square bracket wrapper: [WARN]
// 2014/07/10 11:04:20.653185 filter_fluentd.go:16: [DEBUG] %s
// datestr pos, posEnd
pos = strings.IndexRune(line, '[')
posEnd = strings.IndexRune(line, ']')
if pos > 0 && posEnd > 0 && pos < posEnd && len(line) > pos && len(line) > posEnd {
loglevel = line[pos+1 : posEnd]
// If we don't find, it probably wasn't one of [INFO],[WARN] etc so accumulate
if _, ok := expectedLevels[loglevel]; !ok {
buf.WriteString(line)
return
}
}
//u.Debugf("pos=%d datatype=%s num?=%v", pos, dataType, startsDate)
//u.Infof("starts with date?=%v prev?%v pos=%d lvl=%s short[]%v len=%d buf.len=%d", startsDate, prevWasDate, pos, loglevel, (posEnd-pos) < 8, len(line), buf.Len())
if pos == -1 && !prevWasDate {
// accumulate in buffer, probably/possibly a panic?
buf.WriteString(line)
buf.WriteString(" \n")
} else if !startsDate {
// accumulate in buffer
buf.WriteString(line)
buf.WriteString(" \n")
} else if posEnd-8 > pos {
// position of [block] too long, so ignore
buf.WriteString(line)
buf.WriteString(" \n")
} else if pos > 80 {
// [WARN] should be at beginning of line
buf.WriteString(line)
buf.WriteString(" \n")
} else {
// Line had [LEVEL] AND startsDate at start so go ahead and log it
if buf.Len() == 0 {
// lets buffer it, ensuring we have the completion of this line
buf.WriteString(line)
return
}
// we already have previous line in buffer
data, err := ioutil.ReadAll(buf)
if err == nil {
pos = bytes.IndexRune(data, '[')
posEnd = bytes.IndexRune(data, ']')
preFix := ""
if posEnd-8 > pos {
//u.Warnf("level:%s \n\nline=%s", string(data[pos+1:posEnd]), string(data))
//buf.WriteString(line)
return
} else if pos > 0 && posEnd > 0 && pos < posEnd && len(data) > pos && len(data) > posEnd {
dataType = data[pos+1 : posEnd]
if len(data) > len(prevDateStr) {
preFix = string(data[len(prevDateStr)+1 : posEnd])
// [prefix |- posEnd
// 2016/09/14 02:33:01.465711 entity.go:179: [ERROR]
preFixParts := strings.Split(preFix, ": ")
if len(preFixParts) > 1 {
preFix = preFixParts[0]
}
data = data[posEnd+1:]
}
} else {
dataType = []byte("NA")
//u.Warnf("level:%s \n\nline=%s", string(data[pos+1:posEnd]), string(data))
}
// if !bytes.HasPrefix(data, datePrefix) {
// u.Warnf("ct=%d level:%s \n\nline=%s", lineCt, string(data[pos+1:posEnd]), string(data))
// }
le := LineEvent{Data: data, Prefix: preFix, Ts: prevLogTs, LogLevel: string(dataType), Source: filename, WriteErrs: 0}
//u.Debugf("lineevent: %+v", le)
msgChan <- &le
} else {
u.Error(err)
}
// now write this line for next analysis
buf.WriteString(line)
}
prevWasDate = startsDate
}
}
var (
VerboseTests *bool = flag.Bool("vv", false, "Verbose Logging?")
)
func init() {
flag.Parse()
if *VerboseTests {
u.SetupLogging("debug")
u.SetColorOutput()
}
builtins.LoadAllBuiltins()
}
var (
t1, _ = dateparse.ParseAny("12/18/2019")
// This is the message context which will be added to all tests below
// and be available to the VM runtime for evaluation by using
// key's such as "int5" or "user_id"
msgContext = datasource.NewContextSimpleData(map[string]value.Value{
"int5": value.NewIntValue(5),
"str5": value.NewStringValue("5"),
"created": value.NewTimeValue(t1),
"bvalt": value.NewBoolValue(true),
"bvalf": value.NewBoolValue(false),
"user_id": value.NewStringValue("abc"),
"urls": value.NewStringsValue([]string{"abc", "123"}),
"hits": value.NewMapIntValue(map[string]int64{"google.com": 5, "bing.com": 1}),
"email": value.NewStringValue("*****@*****.**"),
})
vmTestsx = []vmTest{
func TimeSeconds(ctx expr.EvalContext, val value.Value) (value.NumberValue, bool) {
switch vt := val.(type) {
case value.StringValue:
ts := vt.ToString()
// First, lets try to treat it as a time/date and
// then extract unix seconds
if tv, err := dateparse.ParseAny(ts); err == nil {
return value.NewNumberValue(float64(tv.In(time.UTC).Unix())), true
}
// Since that didn't work, lets look for a variety of seconds/minutes type
// pseudo standards
// M10:30
// 10:30
// 100:30
//
if strings.HasPrefix(ts, "M") {
ts = ts[1:]
}
if strings.Contains(ts, ":") {
parts := strings.Split(ts, ":")
switch len(parts) {
case 1:
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
return value.NewNumberValue(float64(iv)), true
}
if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
return value.NewNumberValue(fv), true
}
case 2:
min, sec := float64(0), float64(0)
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
min = float64(iv)
} else if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
min = fv
}
if iv, err := strconv.ParseInt(parts[1], 10, 64); err == nil {
sec = float64(iv)
} else if fv, err := strconv.ParseFloat(parts[1], 64); err == nil {
sec = fv
}
if min > 0 || sec > 0 {
return value.NewNumberValue(60*min + sec), true
}
case 3:
}
} else {
parts := strings.Split(ts, ":")
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
return value.NewNumberValue(float64(iv)), true
}
if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
return value.NewNumberValue(fv), true
}
}
case value.NumberValue:
return vt, true
case value.IntValue:
return vt.NumberValue(), true
}
return value.NewNumberValue(0), false
}
// This formatter reads go files and performs:
// 1. Squashes multiple lines into one (as needed), Tries to squash panics(go) into one line
// 2. Reads out the LineType/Level [DEBUG,INFO,METRIC] into a field
//
// This expects log files in this format
// 2013-05-25 13:25:32.475 authctx.go:169: [DEBUG] sink Building sink for kafka from factory method
func MakeFileFlattener(filename string, msgChan chan *LineEvent) func(string) {
// Builder used to build the colored string.
buf := new(bytes.Buffer)
startsDate := false
prevWasDate := false
pos := 0
posEnd := 0
var dataType []byte
var loglevel string
var dateStr string
lineCt := 0
return func(line string) {
lineCt++
if len(line) < 8 {
buf.WriteString(line)
return
}
startsDate = false
spaceCt := 0
// 2014/07/10 11:04:20.653185 filter_fluentd.go:16: [DEBUG] %s %s
for i := 0; i < len(line); i++ {
r := line[i]
if r == ' ' {
if spaceCt == 1 {
dateStr = string(line[:i])
if _, err := dateparse.ParseAny(dateStr); err == nil {
startsDate = true
}
break
}
spaceCt++
}
}
// Find first square bracket wrapper: [WARN]
pos = strings.IndexRune(line, '[')
posEnd = strings.IndexRune(line, ']')
if pos > 0 && posEnd > 0 && pos < posEnd && len(line) > pos && len(line) > posEnd {
loglevel = line[pos+1 : posEnd]
if _, ok := expectedLevels[loglevel]; !ok {
buf.WriteString(line)
return
}
}
//u.Debugf("pos=%d datatype=%s num?=%v", pos, dataType, startsDate)
//u.Infof("starts with date?=%v prev?%v pos=%d lvl=%s short[]%v len=%d buf.len=%d", startsDate, prevWasDate, pos, loglevel, (posEnd-pos) < 8, len(line), buf.Len())
if pos == -1 && !prevWasDate {
// accumulate in buffer, probably/possibly a panic?
buf.WriteString(line)
buf.WriteString(" \n")
} else if !startsDate {
// accumulate in buffer
buf.WriteString(line)
buf.WriteString(" \n")
} else if posEnd-8 > pos {
// position of [block] too long, so ignore
buf.WriteString(line)
buf.WriteString(" \n")
} else if pos > 80 {
// [WARN] should be at beginning of line
buf.WriteString(line)
buf.WriteString(" \n")
} else {
// Line had [STUFF] AND startsDate at start
if buf.Len() == 0 {
// lets buffer it, ensuring we have the completion of this line
buf.WriteString(line)
return
}
// we already have previous line in buffer
data, err := ioutil.ReadAll(buf)
if err == nil {
pos = bytes.IndexRune(data, '[')
posEnd = bytes.IndexRune(data, ']')
if posEnd-8 > pos {
//u.Warnf("level:%s \n\nline=%s", string(data[pos+1:posEnd]), string(data))
//buf.WriteString(line)
return
} else if pos > 0 && posEnd > 0 && pos < posEnd && len(data) > pos && len(data) > posEnd {
dataType = data[pos+1 : posEnd]
} else {
dataType = []byte("NA")
//u.Warnf("level:%s \n\nline=%s", string(data[pos+1:posEnd]), string(data))
}
// if !bytes.HasPrefix(data, datePrefix) {
// u.Warnf("ct=%d level:%s \n\nline=%s", lineCt, string(data[pos+1:posEnd]), string(data))
// }
//u.Debugf("dt='%s' data=%s", string(dataType), string(data[0:20]))
msgChan <- &LineEvent{Data: data, DataType: string(dataType), Source: filename, WriteErrs: 0}
} else {
u.Error(err)
}
// now write this line for next analysis
buf.WriteString(line)
}
prevWasDate = startsDate
}
}
// Binary operands: =, ==, !=, OR, AND, >, <, >=, <=, LIKE, contains
//
// x == y, x = y
// x != y
// x OR y
// x > y
// x < =
//
func walkBinary(ctx expr.EvalContext, node *expr.BinaryNode) (value.Value, bool) {
ar, aok := Eval(ctx, node.Args[0])
br, bok := Eval(ctx, node.Args[1])
//u.Debugf("walkBinary: aok?%v ar:%v %T node=%s", aok, ar, ar, node.Args[0])
//u.Debugf("walkBinary: bok?%v br:%v %T node=%s", bok, br, br, node.Args[1])
//u.Debugf("walkBinary: l:%v r:%v %T %T node=%s", ar, br, ar, br, node)
// If we could not evaluate either we can shortcut
if !aok && !bok {
switch node.Operator.T {
case lex.TokenLogicOr, lex.TokenOr:
return value.NewBoolValue(false), true
case lex.TokenEqualEqual, lex.TokenEqual:
// We don't alllow nil == nil here bc we have a NilValue type
// that we would use for that
return value.NewBoolValue(false), true
case lex.TokenNE:
return value.NewBoolValue(false), true
case lex.TokenGT, lex.TokenGE, lex.TokenLT, lex.TokenLE, lex.TokenLike:
return value.NewBoolValue(false), true
}
//u.Debugf("walkBinary not ok: op=%s %v l:%v r:%v %T %T", node.Operator, node, ar, br, ar, br)
return nil, false
}
// Else if we can only evaluate one, we can short circuit as well
if !aok || !bok {
switch node.Operator.T {
case lex.TokenAnd, lex.TokenLogicAnd:
return value.NewBoolValue(false), true
case lex.TokenEqualEqual, lex.TokenEqual:
return value.NewBoolValue(false), true
case lex.TokenNE:
// they are technically not equal?
return value.NewBoolValue(true), true
case lex.TokenGT, lex.TokenGE, lex.TokenLT, lex.TokenLE, lex.TokenLike:
return value.NewBoolValue(false), true
}
//u.Debugf("walkBinary not ok: op=%s %v l:%v r:%v %T %T", node.Operator, node, ar, br, ar, br)
// need to fall through to below
}
switch at := ar.(type) {
case value.IntValue:
switch bt := br.(type) {
case value.IntValue:
//u.Debugf("doing operate ints %v %v %v", at, node.Operator.V, bt)
n := operateInts(node.Operator, at, bt)
return n, true
case value.StringValue:
bi, err := strconv.ParseInt(bt.Val(), 10, 64)
if err == nil {
n, err := operateIntVals(node.Operator, at.Val(), bi)
if err != nil {
return nil, false
}
return n, true
}
case value.NumberValue:
//u.Debugf("doing operate ints/numbers %v %v %v", at, node.Operator.V, bt)
n := operateNumbers(node.Operator, at.NumberValue(), bt)
return n, true
case value.SliceValue:
switch node.Operator.T {
case lex.TokenIN:
for _, val := range bt.Val() {
switch valt := val.(type) {
case value.StringValue:
if at.Val() == valt.IntValue().Val() {
return value.BoolValueTrue, true
}
case value.IntValue:
if at.Val() == valt.Val() {
return value.BoolValueTrue, true
}
case value.NumberValue:
if at.Val() == valt.Int() {
return value.BoolValueTrue, true
}
default:
u.Debugf("Could not coerce to number: T:%T v:%v", val, val)
}
}
return value.NewBoolValue(false), true
default:
u.Debugf("unsupported op for SliceValue op:%v rhT:%T", node.Operator, br)
return nil, false
}
case nil, value.NilValue:
return nil, false
default:
u.Errorf("unknown type: %T %v", bt, bt)
}
case value.NumberValue:
switch bt := br.(type) {
case value.IntValue:
n := operateNumbers(node.Operator, at, bt.NumberValue())
return n, true
case value.NumberValue:
n := operateNumbers(node.Operator, at, bt)
return n, true
case value.SliceValue:
for _, val := range bt.Val() {
switch valt := val.(type) {
case value.StringValue:
if at.Val() == valt.NumberValue().Val() {
return value.BoolValueTrue, true
}
case value.IntValue:
if at.Val() == valt.NumberValue().Val() {
return value.BoolValueTrue, true
}
case value.NumberValue:
if at.Val() == valt.Val() {
return value.BoolValueTrue, true
}
default:
u.Debugf("Could not coerce to number: T:%T v:%v", val, val)
}
}
return value.BoolValueFalse, true
//case value.StringValue:
case nil, value.NilValue:
return nil, false
default:
u.Errorf("unknown type: %T %v", bt, bt)
}
case value.BoolValue:
switch bt := br.(type) {
case value.BoolValue:
atv, btv := at.Value().(bool), bt.Value().(bool)
switch node.Operator.T {
case lex.TokenLogicAnd, lex.TokenAnd:
return value.NewBoolValue(atv && btv), true
case lex.TokenLogicOr, lex.TokenOr:
return value.NewBoolValue(atv || btv), true
case lex.TokenEqualEqual, lex.TokenEqual:
return value.NewBoolValue(atv == btv), true
case lex.TokenNE:
return value.NewBoolValue(atv != btv), true
default:
u.Warnf("bool binary?: %#v %v %v", node, at, bt)
}
case nil, value.NilValue:
switch node.Operator.T {
case lex.TokenLogicAnd:
return value.NewBoolValue(false), true
case lex.TokenLogicOr, lex.TokenOr:
return at, true
case lex.TokenEqualEqual, lex.TokenEqual:
return value.NewBoolValue(false), true
case lex.TokenNE:
return value.NewBoolValue(true), true
// case lex.TokenGE, lex.TokenGT, lex.TokenLE, lex.TokenLT:
// return value.NewBoolValue(false), true
default:
u.Warnf("right side nil binary: %q", node)
return nil, false
}
default:
//u.Warnf("br: %#v", br)
//u.Errorf("at?%T %v coerce?%v bt? %T %v", at, at.Value(), at.CanCoerce(stringRv), bt, bt.Value())
return nil, false
}
case value.StringValue:
switch bt := br.(type) {
case value.StringValue:
// Nice, both strings
return operateStrings(node.Operator, at, bt), true
case nil, value.NilValue:
switch node.Operator.T {
case lex.TokenEqualEqual, lex.TokenEqual:
if at.Nil() {
return value.NewBoolValue(true), true
}
return value.NewBoolValue(false), true
case lex.TokenNE:
if at.Nil() {
return value.NewBoolValue(false), true
}
return value.NewBoolValue(true), true
default:
u.Debugf("unsupported op: %v", node.Operator)
return nil, false
}
case value.SliceValue:
switch node.Operator.T {
case lex.TokenIN:
for _, val := range bt.Val() {
if at.Val() == val.ToString() {
return value.NewBoolValue(true), true
}
}
return value.NewBoolValue(false), true
default:
u.Debugf("unsupported op for SliceValue op:%v rhT:%T", node.Operator, br)
return nil, false
}
case value.StringsValue:
switch node.Operator.T {
case lex.TokenIN:
for _, val := range bt.Val() {
if at.Val() == val {
return value.NewBoolValue(true), true
}
}
return value.NewBoolValue(false), true
default:
u.Debugf("unsupported op for Strings op:%v rhT:%T", node.Operator, br)
return nil, false
}
case value.MapIntValue:
switch node.Operator.T {
case lex.TokenIN:
for key, _ := range bt.Val() {
if at.Val() == key {
return value.NewBoolValue(true), true
}
}
return value.NewBoolValue(false), true
default:
u.Debugf("unsupported op for MapInt op:%v rhT:%T", node.Operator, br)
return nil, false
}
case value.BoolValue:
if value.IsBool(at.Val()) {
//u.Warnf("bool eval: %v %v %v :: %v", value.BoolStringVal(at.Val()), node.Operator.T.String(), bt.Val(), value.NewBoolValue(value.BoolStringVal(at.Val()) == bt.Val()))
switch node.Operator.T {
case lex.TokenEqualEqual, lex.TokenEqual:
return value.NewBoolValue(value.BoolStringVal(at.Val()) == bt.Val()), true
case lex.TokenNE:
return value.NewBoolValue(value.BoolStringVal(at.Val()) != bt.Val()), true
default:
u.Debugf("unsupported op: %v", node.Operator)
return nil, false
}
} else {
// Should we evaluate strings that are non-nil to be = true?
u.Debugf("not handled: boolean %v %T=%v expr: %s", node.Operator, at.Value(), at.Val(), node.String())
return nil, false
}
default:
// TODO: this doesn't make sense, we should be able to operate on other types
if at.CanCoerce(int64Rv) {
switch bt := br.(type) {
case value.StringValue:
n := operateNumbers(node.Operator, at.NumberValue(), bt.NumberValue())
return n, true
case value.IntValue:
n := operateNumbers(node.Operator, at.NumberValue(), bt.NumberValue())
return n, true
case value.NumberValue:
n := operateNumbers(node.Operator, at.NumberValue(), bt)
return n, true
default:
u.Errorf("at?%T %v coerce?%v bt? %T %v", at, at.Value(), at.CanCoerce(stringRv), bt, bt.Value())
}
} else {
u.Errorf("at?%T %v coerce?%v bt? %T %v", at, at.Value(), at.CanCoerce(stringRv), br, br)
}
}
case value.SliceValue:
switch node.Operator.T {
case lex.TokenContains:
switch bval := br.(type) {
case nil, value.NilValue:
return nil, false
case value.StringValue:
// [x,y,z] contains str
for _, val := range at.Val() {
if strings.Contains(val.ToString(), bval.Val()) {
return value.BoolValueTrue, true
}
}
return value.BoolValueFalse, true
case value.IntValue:
// [] contains int
for _, val := range at.Val() {
//u.Infof("int contains? %v %v", val.Value(), br.Value())
if eq, _ := value.Equal(val, br); eq {
return value.BoolValueTrue, true
}
}
return value.BoolValueFalse, true
}
case lex.TokenLike:
switch bv := br.(type) {
case value.StringValue:
// [x,y,z] LIKE str
for _, val := range at.Val() {
if boolVal, ok := LikeCompare(val.ToString(), bv.Val()); ok && boolVal.Val() == true {
return boolVal, true
}
}
return value.BoolValueFalse, true
}
case lex.TokenIntersects:
switch bt := br.(type) {
case nil, value.NilValue:
return nil, false
case value.SliceValue:
for _, aval := range at.Val() {
for _, bval := range bt.Val() {
if eq, _ := value.Equal(aval, bval); eq {
return value.BoolValueTrue, true
}
}
}
return value.BoolValueFalse, true
case value.StringsValue:
for _, aval := range at.Val() {
for _, bstr := range bt.Val() {
if aval.ToString() == bstr {
return value.BoolValueTrue, true
}
}
}
return value.BoolValueFalse, true
}
}
return nil, false
case value.StringsValue:
switch node.Operator.T {
case lex.TokenContains:
switch bv := br.(type) {
case value.StringValue:
// [x,y,z] contains str
for _, val := range at.Val() {
//u.Infof("str contains? %v %v", val, bv.Val())
if strings.Contains(val, bv.Val()) {
return value.BoolValueTrue, true
}
}
return value.BoolValueFalse, true
}
case lex.TokenLike:
switch bv := br.(type) {
case value.StringValue:
// [x,y,z] LIKE str
for _, val := range at.Val() {
boolVal, ok := LikeCompare(val, bv.Val())
//u.Debugf("%s like %s ?? ok?%v result=%v", val, bv.Val(), ok, boolVal)
if ok && boolVal.Val() == true {
return boolVal, true
}
}
return value.BoolValueFalse, true
}
case lex.TokenIntersects:
switch bt := br.(type) {
case nil, value.NilValue:
return nil, false
case value.SliceValue:
for _, astr := range at.Val() {
for _, bval := range bt.Val() {
if astr == bval.ToString() {
return value.BoolValueTrue, true
}
}
}
return value.BoolValueFalse, true
case value.StringsValue:
for _, astr := range at.Val() {
for _, bstr := range bt.Val() {
if astr == bstr {
return value.BoolValueTrue, true
}
}
}
return value.BoolValueFalse, true
}
}
return nil, false
case value.TimeValue:
rht := time.Time{}
lht := at.Val()
var err error
switch bv := br.(type) {
case value.TimeValue:
rht = bv.Val()
case value.StringValue:
te := bv.Val()
if len(te) > 3 && strings.ToLower(te[:3]) == "now" {
// Is date math
rht, err = datemath.Eval(te[3:])
} else {
rht, err = dateparse.ParseAny(te)
}
if err != nil {
u.Warnf("error? %s err=%v", te, err)
return value.BoolValueFalse, false
}
case value.IntValue:
// really? we are going to try ints?
rht, err = dateparse.ParseAny(bv.ToString())
if err != nil {
return value.BoolValueFalse, false
}
if rht.Year() < 1800 || rht.Year() > 2300 {
return value.BoolValueFalse, false
}
default:
//u.Warnf("un-handled? %#v", bv)
}
// if rht.IsZero() {
// return nil, false
// }
switch node.Operator.T {
case lex.TokenEqual, lex.TokenEqualEqual:
if lht.Unix() == rht.Unix() {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
case lex.TokenGT:
// lhexpr > rhexpr
if lht.Unix() > rht.Unix() {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
case lex.TokenGE:
// lhexpr >= rhexpr
if lht.Unix() >= rht.Unix() {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
case lex.TokenLT:
// lhexpr < rhexpr
if lht.Unix() < rht.Unix() {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
case lex.TokenLE:
// lhexpr <= rhexpr
if lht.Unix() <= rht.Unix() {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
default:
u.Warnf("unhandled date op %v", node.Operator)
}
return nil, false
case nil, value.NilValue:
switch node.Operator.T {
case lex.TokenLogicAnd:
return value.NewBoolValue(false), true
case lex.TokenLogicOr, lex.TokenOr:
switch bt := br.(type) {
case value.BoolValue:
return bt, true
default:
return value.NewBoolValue(false), true
}
case lex.TokenEqualEqual, lex.TokenEqual:
// does nil==nil = true ??
switch br.(type) {
case nil, value.NilValue:
return value.NewBoolValue(true), true
default:
return value.NewBoolValue(false), true
}
case lex.TokenNE:
return value.NewBoolValue(true), true
// case lex.TokenGE, lex.TokenGT, lex.TokenLE, lex.TokenLT:
// return value.NewBoolValue(false), true
case lex.TokenContains, lex.TokenLike, lex.TokenIN:
return value.NewBoolValue(false), true
default:
//u.Debugf("left side nil binary: %q", node)
return nil, false
}
default:
u.Debugf("Unknown op? %T %T %v", ar, at, ar)
return value.NewErrorValue(fmt.Sprintf("unsupported left side value: %T in %s", at, node)), false
}
return value.NewErrorValue(fmt.Sprintf("unsupported binary expression: %s", node)), false
}
func TestFilterQlVm(t *testing.T) {
t.Parallel()
t1, _ := dateparse.ParseAny("12/18/2015")
//u.Infof("t1 %v", t1)
nminus1 := time.Now().Add(time.Hour * -1)
tr := true
user := &User{
Name: "Yoda",
Created: t1,
Updated: &nminus1,
Authenticated: true,
HasSession: &tr,
Address: Address{"Detroit", 55},
Roles: []string{"admin", "api"},
BankAmount: 55.5,
}
readers := []expr.ContextReader{
datasource.NewContextWrapper(user),
datasource.NewContextSimpleNative(map[string]interface{}{
"city": "Peoria, IL",
"zip": 5,
}),
}
nc := datasource.NewNestedContextReader(readers, time.Now())
hits := []string{
`FILTER name == "Yoda"`, // upper case sensitive name
`FILTER name != "yoda"`, // we should be case-sensitive by default
`FILTER name = "Yoda"`, // is equivalent to ==
`FILTER "Yoda" == name`, // reverse order of identity/value
`FILTER name != "Anakin"`, // negation on missing fields == true
`FILTER first_name != "Anakin"`, // key doesn't exist
`FILTER tolower(name) == "yoda"`, // use functions in evaluation
`FILTER FullName == "Yoda, Jedi"`, // use functions on structs in evaluation
`FILTER Address.City == "Detroit"`, // traverse struct with path.field
`FILTER name LIKE "*da"`, // LIKE
`FILTER name NOT LIKE "*kin"`, // LIKE Negation
`FILTER name CONTAINS "od"`, // Contains
`FILTER name NOT CONTAINS "kin"`, // Contains
`FILTER roles INTERSECTS ("user", "api")`, // Intersects
`FILTER roles NOT INTERSECTS ("user", "guest")`, // Intersects
`FILTER Created < "now-1d"`, // Date Math
`FILTER Updated > "now-2h"`, // Date Math
`FILTER *`, // match all
`FILTER OR (
EXISTS name, -- inline comments
EXISTS not_a_key, -- more inline comments
)`,
// show that line-breaks serve as expression separators
`FILTER OR (
EXISTS name
EXISTS not_a_key -- even if they have inline comments
)`,
//`FILTER a == "Yoda" AND b == "Peoria, IL" AND c == 5`,
`FILTER AND (name == "Yoda", city == "Peoria, IL", zip == 5, BankAmount > 50)`,
`FILTER AND (zip == 5, "Yoda" == name, OR ( city IN ( "Portland, OR", "New York, NY", "Peoria, IL" ) ) )`,
`FILTER OR (
EXISTS q,
AND (
zip > 0,
OR ( zip > 10000, zip < 100 )
),
NOT ( name == "Yoda" ) )`,
}
for _, q := range hits {
fs, err := rel.ParseFilterQL(q)
assert.Equal(t, nil, err)
match, err := NewFilterVm(nil).Matches(nc, fs)
assert.Equalf(t, nil, err, "error matching on query %q: %v", q, err)
assert.T(t, match, q)
}
misses := []string{
`FILTER name == "yoda"`, // casing
"FILTER OR (false, false, AND (true, false))",
`FILTER AND (name == "Yoda", city == "xxx", zip == 5)`,
}
for _, q := range misses {
fs, err := rel.ParseFilterQL(q)
assert.Equal(t, nil, err)
match, err := NewFilterVm(nil).Matches(nc, fs)
assert.Equal(t, nil, err)
assert.T(t, !match)
}
// Filter Select Statements
filterSelects := []fsel{
fsel{`select name, zip FROM mycontext FILTER name == "Yoda"`, map[string]interface{}{"name": "Yoda", "zip": 5}},
}
for _, test := range filterSelects {
//u.Debugf("about to parse: %v", test.qlText)
sel, err := rel.ParseFilterSelect(test.query)
assert.T(t, err == nil, "expected no error but got ", err, " for ", test.query)
writeContext := datasource.NewContextSimple()
_, err = EvalFilerSelect(sel, nil, writeContext, nc)
assert.T(t, err == nil, "expected no error but got ", err, " for ", test.query)
for key, val := range test.expect {
v := value.NewValue(val)
v2, ok := writeContext.Get(key)
assert.Tf(t, ok, "Get(%q)=%v but got: %#v", key, val, writeContext.Row())
assert.Equalf(t, v2.Value(), v.Value(), "?? %s %v!=%v %T %T", key, v.Value(), v2.Value(), v.Value(), v2.Value())
}
}
}
u.SetColorOutput()
}
type testBuiltins struct {
expr string
val value.Value
}
var (
// This is used so we have a constant understood time for message context
// normally we would use time.Now()
// "Apr 7, 2014 4:58:55 PM"
regDate = "10/13/2014"
ts = time.Date(2014, 4, 7, 16, 58, 55, 00, time.UTC)
ts2 = time.Date(2014, 4, 7, 0, 0, 0, 00, time.UTC)
regTime, _ = dateparse.ParseAny(regDate)
readContext = datasource.NewContextUrlValuesTs(url.Values{
"event": {"hello"},
"reg_date": {"10/13/2014"},
"price": {"$55"},
"email": {"*****@*****.**"},
"url": {"http://www.site.com/membership/all.html"},
"score_amount": {"22"},
"tag_name": {"bob"},
}, ts)
float3pt1 = float64(3.1)
)
var builtinTestsx = []testBuiltins{
{`cast(reg_date as time)`, value.NewTimeValue(regTime)},
{`CHAR_LENGTH(CAST("abc" AS CHAR))`, value.NewIntValue(3)},
