// Convert to Number: Best attempt at converting to integer
//
// tonumber("5") => 5.0
// tonumber("5.75") => 5.75
// tonumber("5,555") => 5555
// tonumber("$5") => 5.00
// tonumber("5,555.00") => 5555
//
func ToNumber(ctx expr.EvalContext, item value.Value) (value.NumberValue, bool) {
fv, ok := value.ToFloat64(reflect.ValueOf(item.Value()))
if !ok {
return value.NewNumberValue(0), false
}
return value.NewNumberValue(fv), true
}
// ValueArray
// IN ("a","b","c")
// ["a","b","c"]
func ValueArray(pg TokenPager) (value.Value, error) {
//u.Debugf("valueArray cur:%v peek:%v", pg.Cur().V, pg.Peek().V)
vals := make([]value.Value, 0)
arrayLoop:
for {
tok := pg.Next() // consume token
//u.Infof("valueArray() consumed token?: %v", tok)
switch tok.T {
case lex.TokenComma:
// continue
case lex.TokenRightParenthesis:
//u.Warnf("found right paren %v cur: %v", tok, pg.Cur())
break arrayLoop
case lex.TokenEOF, lex.TokenEOS, lex.TokenFrom, lex.TokenAs:
//u.Debugf("return: %v", tok)
break arrayLoop
case lex.TokenValue:
vals = append(vals, value.NewStringValue(tok.V))
case lex.TokenInteger:
fv, err := strconv.ParseFloat(tok.V, 64)
if err == nil {
vals = append(vals, value.NewNumberValue(fv))
} else {
return value.NilValueVal, err
}
case lex.TokenFloat:
fv, err := strconv.ParseFloat(tok.V, 64)
if err == nil {
vals = append(vals, value.NewNumberValue(fv))
} else {
return value.NilValueVal, err
}
default:
return value.NilValueVal, fmt.Errorf("Could not recognize token: %v", tok)
}
tok = pg.Next()
switch tok.T {
case lex.TokenComma:
// fine, consume the comma
case lex.TokenRightBracket:
//u.Warnf("right bracket: %v", tok)
break arrayLoop
default:
u.Warnf("unrecognized token: %v", tok)
return value.NilValueVal, fmt.Errorf("unrecognized token %v", tok)
}
}
//u.Debugf("returning array: %v", vals)
return value.NewSliceValues(vals), nil
}
// Sqrt
//
// sqrt(4) => 2, true
// sqrt(9) => 3, true
// sqrt(not_number) => 0, false
//
func SqrtFunc(ctx expr.EvalContext, val value.Value) (value.NumberValue, bool) {
nv, ok := val.(value.NumericValue)
if !ok {
return value.NewNumberValue(math.NaN()), false
}
if val.Err() || val.Nil() {
return value.NewNumberValue(0), false
}
fv := nv.Float()
fv = math.Sqrt(fv)
//u.Infof("??? vals=[%v]", val.Value())
return value.NewNumberValue(fv), true
}
// avg: average doesn't avg bc it doesn't have a storage, but does return number
//
func AvgFunc(ctx expr.EvalContext, val value.Value) (value.NumberValue, bool) {
switch node := val.(type) {
case value.StringValue:
if fv, err := strconv.ParseFloat(node.Val(), 64); err == nil {
return value.NewNumberValue(fv), true
}
case value.NumberValue:
return node, true
case value.IntValue:
return value.NewNumberValue(node.Float()), true
case nil, value.NilValue:
return value.NewNumberValue(0), false
}
return value.NewNumberValue(0), false
}
func walkUnary(ctx expr.EvalContext, node *expr.UnaryNode) (value.Value, bool) {
a, ok := Eval(ctx, node.Arg)
if !ok {
u.Infof("whoops, %#v", node)
return a, false
}
switch node.Operator.T {
case lex.TokenNegate:
switch argVal := a.(type) {
case value.BoolValue:
//u.Infof("found urnary bool: res=%v expr=%v", !argVal.v, node.StringAST())
return value.NewBoolValue(!argVal.Val()), true
default:
//u.Errorf("urnary type not implementedUnknonwn node type: %T", argVal)
panic(ErrUnknownNodeType)
}
case lex.TokenMinus:
if an, aok := a.(value.NumericValue); aok {
return value.NewNumberValue(-an.Float()), true
}
default:
u.Warnf("urnary not implemented: %#v", node)
}
return value.NewNilValue(), false
}
// Sum
func SumFunc(ctx expr.EvalContext, vals ...value.Value) (value.NumberValue, bool) {
//u.Debugf("Sum: %v", vals)
sumval := float64(0)
for _, val := range vals {
if val == nil || val.Nil() || val.Err() {
// we don't need to evaluate if nil or error
} else {
switch valValue := val.(type) {
case value.StringsValue:
//u.Debugf("Nice, we have strings: %v", valValue)
for _, sv := range valValue.Value().([]string) {
if fv, ok := value.ToFloat64(reflect.ValueOf(sv)); ok && !math.IsNaN(fv) {
sumval += fv
}
}
default:
//u.Debugf("Sum: %T tofloat=%v %v", value.ToFloat64(val.Rv()), value.ToFloat64(val.Rv()), val.Rv())
if fv, ok := value.ToFloat64(val.Rv()); ok && !math.IsNaN(fv) {
sumval += fv
}
}
}
}
if sumval == float64(0) {
return value.NumberNaNValue, false
}
//u.Debugf("Sum() about to return? %v Nan?%v", sumval, sumval == math.NaN())
return value.NewNumberValue(sumval), true
}
// Pow
func PowFunc(ctx EvalContext, val, toPower value.Value) (value.NumberValue, bool) {
//Pow(x, y float64) float64
//u.Infof("powFunc: %T:%v %T:%v ", val, val.Value(), toPower, toPower.Value())
if val.Err() || val.Nil() {
return value.NewNumberValue(0), false
}
if toPower.Err() || toPower.Nil() {
return value.NewNumberValue(0), false
}
fv, pow := value.ToFloat64(val.Rv()), value.ToFloat64(toPower.Rv())
if fv == math.NaN() || pow == math.NaN() {
return value.NewNumberValue(0), false
}
fv = math.Pow(fv, pow)
//u.Infof("pow ??? vals=[%v]", fv, pow)
return value.NewNumberValue(fv), true
}
// creates a new Value with a nil group and given value.
// TODO: convert this to an interface method on nodes called Value()
func numberNodeToValue(t *expr.NumberNode) (v value.Value) {
//u.Debugf("nodeToValue() isFloat?%v", t.IsFloat)
if t.IsInt {
v = value.NewIntValue(t.Int64)
} else if t.IsFloat {
v = value.NewNumberValue(value.ToFloat64(reflect.ValueOf(t.Text)))
} else {
u.Errorf("Could not find type? %v", t.Type())
}
//u.Debugf("return nodeToValue() %v %T arg:%T", v, v, t)
return v
}
func walkUnary(ctx expr.EvalContext, node *expr.UnaryNode) (value.Value, bool) {
a, ok := Eval(ctx, node.Arg)
if !ok {
switch node.Operator.T {
case lex.TokenExists:
return value.NewBoolValue(false), true
case lex.TokenNegate:
return value.NewBoolValue(true), true
}
u.Debugf("unary could not evaluate for[ %s ] and %#v", node.String(), node)
return a, false
}
switch node.Operator.T {
case lex.TokenNegate:
switch argVal := a.(type) {
case value.BoolValue:
//u.Debugf("found unary bool: res=%v expr=%v", !argVal.Val(), node)
return value.NewBoolValue(!argVal.Val()), true
case nil, value.NilValue:
return value.NewBoolValue(false), false
default:
u.LogThrottle(u.WARN, 5, "unary type not implemented. Unknonwn node type: %T:%v node=%s", argVal, argVal, node.String())
return value.NewNilValue(), false
}
case lex.TokenMinus:
if an, aok := a.(value.NumericValue); aok {
return value.NewNumberValue(-an.Float()), true
}
case lex.TokenExists:
switch a.(type) {
case nil, value.NilValue:
return value.NewBoolValue(false), true
}
if a.Nil() {
return value.NewBoolValue(false), true
}
return value.NewBoolValue(true), true
default:
u.Warnf("urnary not implemented for type %s %#v", node.Operator.T.String(), node)
}
return value.NewNilValue(), false
}
// creates a new Value with a nil group and given value.
// TODO: convert this to an interface method on nodes called Value()
func numberNodeToValue(t *expr.NumberNode) (value.Value, bool) {
//u.Debugf("nodeToValue() isFloat?%v", t.IsFloat)
var v value.Value
if t.IsInt {
v = value.NewIntValue(t.Int64)
} else if t.IsFloat {
fv, ok := value.ToFloat64(reflect.ValueOf(t.Text))
if !ok {
u.Warnf("Could not perform numeric conversion for %q", t.Text)
return value.NilValueVal, false
}
v = value.NewNumberValue(fv)
} else {
u.Warnf("Could not find numeric conversion for %v", t.Type())
return value.NilValueVal, false
}
//u.Debugf("return nodeToValue() %v %T arg:%T", v, v, t)
return v, true
}
func walkUnary(ctx expr.EvalContext, node *expr.UnaryNode) (value.Value, bool) {
a, ok := Eval(ctx, node.Arg)
if !ok {
if node.Operator.T == lex.TokenExists {
return value.NewBoolValue(false), true
}
u.Debugf("unary could not evaluate %#v", node)
return a, false
}
switch node.Operator.T {
case lex.TokenNegate:
switch argVal := a.(type) {
case value.BoolValue:
//u.Infof("found unary bool: res=%v expr=%v", !argVal.v, node.StringAST())
return value.NewBoolValue(!argVal.Val()), true
case nil, value.NilValue:
return value.NewBoolValue(false), false
default:
u.Errorf("unary type not implemented. Unknonwn node type: %T:%v", argVal, argVal)
panic(ErrUnknownNodeType)
}
case lex.TokenMinus:
if an, aok := a.(value.NumericValue); aok {
return value.NewNumberValue(-an.Float()), true
}
case lex.TokenExists:
switch a.(type) {
case nil, value.NilValue:
return value.NewBoolValue(false), true
}
return value.NewBoolValue(true), true
default:
u.Warnf("urnary not implemented for type %s %#v", node.Operator.T.String(), node)
}
return value.NewNilValue(), false
}
{`qs("www.Google.com/?q=golang","q")`, value.NewStringValue("golang")},
{`urlminusqs("http://www.Google.com/search?q1=golang&q2=github","q1")`, value.NewStringValue("http://www.Google.com/search?q2=github")},
{`urlminusqs("http://www.Google.com/search?q1=golang&q2=github","q3")`, value.NewStringValue("http://www.Google.com/search?q1=golang&q2=github")},
{`urlminusqs("http://www.Google.com/search?q1=golang","q1")`, value.NewStringValue("http://www.Google.com/search")},
{`urlmain("http://www.Google.com/search?q1=golang&q2=github")`, value.NewStringValue("www.Google.com/search")},
{`toint("5")`, value.NewIntValue(5)},
{`toint("hello")`, value.ErrValue},
{`toint("$ 5.22")`, value.NewIntValue(5)},
{`toint("5.56")`, value.NewIntValue(5)},
{`toint("$5.56")`, value.NewIntValue(5)},
{`toint("5,555.00")`, value.NewIntValue(5555)},
{`toint("€ 5,555.00")`, value.NewIntValue(5555)},
{`seconds("M10:30")`, value.NewNumberValue(630)},
{`seconds(replace("M10:30","M"))`, value.NewNumberValue(630)},
{`seconds("M100:30")`, value.NewNumberValue(6030)},
{`seconds("00:30")`, value.NewNumberValue(30)},
{`seconds("30")`, value.NewNumberValue(30)},
{`seconds(30)`, value.NewNumberValue(30)},
{`seconds("2015/07/04")`, value.NewNumberValue(1435968000)},
{`yy("10/13/2014")`, value.NewIntValue(14)},
{`yy("01/02/2006")`, value.NewIntValue(6)},
{`yy()`, value.NewIntValue(int64(ts.Year() - 2000))},
{`mm("10/13/2014")`, value.NewIntValue(10)},
{`mm("01/02/2006")`, value.NewIntValue(1)},
{`yymm("10/13/2014")`, value.NewStringValue("1410")},
func TimeSeconds(ctx expr.EvalContext, val value.Value) (value.NumberValue, bool) {
switch vt := val.(type) {
case value.StringValue:
ts := vt.ToString()
// First, lets try to treat it as a time/date and
// then extract unix seconds
if tv, err := dateparse.ParseAny(ts); err == nil {
return value.NewNumberValue(float64(tv.In(time.UTC).Unix())), true
}
// Since that didn't work, lets look for a variety of seconds/minutes type
// pseudo standards
// M10:30
// 10:30
// 100:30
//
if strings.HasPrefix(ts, "M") {
ts = ts[1:]
}
if strings.Contains(ts, ":") {
parts := strings.Split(ts, ":")
switch len(parts) {
case 1:
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
return value.NewNumberValue(float64(iv)), true
}
if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
return value.NewNumberValue(fv), true
}
case 2:
min, sec := float64(0), float64(0)
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
min = float64(iv)
} else if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
min = fv
}
if iv, err := strconv.ParseInt(parts[1], 10, 64); err == nil {
sec = float64(iv)
} else if fv, err := strconv.ParseFloat(parts[1], 64); err == nil {
sec = fv
}
if min > 0 || sec > 0 {
return value.NewNumberValue(60*min + sec), true
}
case 3:
}
} else {
parts := strings.Split(ts, ":")
if iv, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
return value.NewNumberValue(float64(iv)), true
}
if fv, err := strconv.ParseFloat(parts[0], 64); err == nil {
return value.NewNumberValue(fv), true
}
}
case value.NumberValue:
return vt, true
case value.IntValue:
return vt.NumberValue(), true
}
return value.NewNumberValue(0), false
}
func operateNumbers(op lex.Token, av, bv value.NumberValue) value.Value {
switch op.T {
case lex.TokenPlus, lex.TokenStar, lex.TokenMultiply, lex.TokenDivide, lex.TokenMinus,
lex.TokenModulus:
if math.IsNaN(av.Val()) || math.IsNaN(bv.Val()) {
return value.NewNumberValue(math.NaN())
}
}
//
a, b := av.Val(), bv.Val()
switch op.T {
case lex.TokenPlus: // +
return value.NewNumberValue(a + b)
case lex.TokenStar, lex.TokenMultiply: // *
return value.NewNumberValue(a * b)
case lex.TokenMinus: // -
return value.NewNumberValue(a - b)
case lex.TokenDivide: // /
return value.NewNumberValue(a / b)
case lex.TokenModulus: // %
// is this even valid? modulus on floats?
return value.NewNumberValue(float64(int64(a) % int64(b)))
// Below here are Boolean Returns
case lex.TokenEqualEqual, lex.TokenEqual: // ==
//u.Infof("==? %v %v", av, bv)
if a == b {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenGT: // >
if a > b {
//r = 1
return value.BoolValueTrue
} else {
//r = 0
return value.BoolValueFalse
}
case lex.TokenNE: // != or <>
if a != b {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenLT: // <
if a < b {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenGE: // >=
if a >= b {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenLE: // <=
if a <= b {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenLogicOr, lex.TokenOr: // ||
if a != 0 || b != 0 {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
case lex.TokenLogicAnd: // &&
if a != 0 && b != 0 {
return value.BoolValueTrue
} else {
return value.BoolValueFalse
}
}
panic(fmt.Errorf("expr: unknown operator %s", op))
}
{`urlminusqs("http://www.Google.com/search?q1=golang","q1")`, value.NewStringValue("http://www.Google.com/search")},
{`urlmain("http://www.Google.com/search?q1=golang&q2=github")`, value.NewStringValue("www.Google.com/search")},
// ts2 = time.Date(2014, 4, 7, 0, 0, 0, 00, time.UTC)
// Eu style
{`todate("02/01/2006","07/04/2014")`, value.NewTimeValue(ts2)},
{`toint("5")`, value.NewIntValue(5)},
{`toint("hello")`, value.ErrValue},
{`toint("$ 5.22")`, value.NewIntValue(5)},
{`toint("5.56")`, value.NewIntValue(5)},
{`toint("$5.56")`, value.NewIntValue(5)},
{`toint("5,555.00")`, value.NewIntValue(5555)},
{`toint("€ 5,555.00")`, value.NewIntValue(5555)},
{`tonumber("5")`, value.NewNumberValue(float64(5))},
{`tonumber("hello")`, value.ErrValue},
{`tonumber("$ 5.22")`, value.NewNumberValue(float64(5.22))},
{`tonumber("5.56")`, value.NewNumberValue(float64(5.56))},
{`tonumber("$5.56")`, value.NewNumberValue(float64(5.56))},
{`tonumber("5,555.00")`, value.NewNumberValue(float64(5555.00))},
{`tonumber("€ 5,555.00")`, value.NewNumberValue(float64(5555.00))},
{`seconds("M10:30")`, value.NewNumberValue(630)},
{`seconds(replace("M10:30","M"))`, value.NewNumberValue(630)},
{`seconds("M100:30")`, value.NewNumberValue(6030)},
{`seconds("00:30")`, value.NewNumberValue(30)},
{`seconds("30")`, value.NewNumberValue(30)},
{`seconds(30)`, value.NewNumberValue(30)},
{`seconds("2015/07/04")`, value.NewNumberValue(1435968000)},
func (m *Sqlbridge) parseValueList() ([][]*ValueColumn, error) {
if m.Cur().T != lex.TokenLeftParenthesis {
return nil, fmt.Errorf("Expecting opening paren ( but got %v", m.Cur())
}
var row []*ValueColumn
values := make([][]*ValueColumn, 0)
for {
//u.Debug(m.Cur().String())
switch m.Cur().T {
case lex.TokenLeftParenthesis:
// start of row
if len(row) > 0 {
values = append(values, row)
}
row = make([]*ValueColumn, 0)
case lex.TokenRightParenthesis:
values = append(values, row)
case lex.TokenFrom, lex.TokenInto, lex.TokenLimit, lex.TokenEOS, lex.TokenEOF:
if len(row) > 0 {
values = append(values, row)
}
return values, nil
case lex.TokenValue:
row = append(row, &ValueColumn{Value: value.NewStringValue(m.Cur().V)})
case lex.TokenInteger:
iv, err := strconv.ParseInt(m.Cur().V, 10, 64)
if err != nil {
return nil, err
}
row = append(row, &ValueColumn{Value: value.NewIntValue(iv)})
case lex.TokenFloat:
fv, err := strconv.ParseFloat(m.Cur().V, 64)
if err != nil {
return nil, err
}
row = append(row, &ValueColumn{Value: value.NewNumberValue(fv)})
case lex.TokenBool:
bv, err := strconv.ParseBool(m.Cur().V)
if err != nil {
return nil, err
}
row = append(row, &ValueColumn{Value: value.NewBoolValue(bv)})
case lex.TokenIdentity:
// TODO: this is a bug in lexer
lv := m.Cur().V
if bv, err := strconv.ParseBool(lv); err == nil {
row = append(row, &ValueColumn{Value: value.NewBoolValue(bv)})
} else {
// error?
u.Warnf("Could not figure out how to use: %v", m.Cur())
}
case lex.TokenLeftBracket:
// an array of values?
m.Next() // Consume the [
arrayVal, err := expr.ValueArray(m.SqlTokenPager)
if err != nil {
return nil, err
}
//n := NewValueNode(arrayVal)
row = append(row, &ValueColumn{Value: arrayVal})
u.Infof("what is token? %v peek:%v", m.Cur(), m.Peek())
//t.Next()
case lex.TokenComma:
// don't need to do anything
case lex.TokenUdfExpr:
tree := expr.NewTreeFuncs(m.SqlTokenPager, m.funcs)
if err := m.parseNode(tree); err != nil {
u.Errorf("could not parse: %v", err)
return nil, err
}
//col.Expr = tree.Root
row = append(row, &ValueColumn{Expr: tree.Root})
default:
u.Warnf("don't know how to handle ? %v", m.Cur())
return nil, fmt.Errorf("expected column but got: %v", m.Cur().String())
}
m.Next()
}
panic("unreachable")
}
{`hourofday("Apr 7, 2014 4:58:55 PM")`, value.NewIntValue(16)},
{`hourofday()`, value.NewIntValue(16)},
{`hourofweek("Apr 7, 2014 4:58:55 PM")`, value.NewIntValue(40)},
{`totimestamp("Apr 7, 2014 4:58:55 PM")`, value.NewIntValue(1396889935)},
{`todate("Apr 7, 2014 4:58:55 PM")`, value.NewTimeValue(ts)},
{`exists(event)`, value.BoolValueTrue},
{`exists(price)`, value.BoolValueTrue},
{`exists(toint(price))`, value.BoolValueTrue},
{`exists(-1)`, value.BoolValueTrue},
{`exists(non_field)`, value.BoolValueFalse},
{`pow(5,2)`, value.NewNumberValue(25)},
{`pow(2,2)`, value.NewNumberValue(4)},
{`pow(NotAField,2)`, value.ErrValue},
{`sqrt(4)`, value.NewNumberValue(2)},
{`sqrt(25)`, value.NewNumberValue(5)},
{`sqrt(NotAField)`, value.ErrValue},
{`count(4)`, value.NewIntValue(1)},
{`count(not_a_field)`, value.ErrValue},
}
// Need to think about this a bit, as expression vm resolves IdentityNodes in advance
// such that we get just value, so exists() doesn't even work
// {`exists(event)`, value.BoolValueTrue},
// {`exists("event")`, value.BoolValueTrue},
