// Extract host from a String (must be urlish), doesn't do much/any validation
func Qs(ctx expr.EvalContext, urlItem, keyItem value.Value) (value.StringValue, bool) {
val, ok := value.ToString(urlItem.Rv())
if !ok {
return value.EmptyStringValue, false
}
if val == "" {
return value.EmptyStringValue, false
}
urlstr := strings.ToLower(val)
if len(urlstr) < 8 {
return value.EmptyStringValue, false
}
keyVal, ok := value.ToString(keyItem.Rv())
if !ok {
return value.EmptyStringValue, false
}
if keyVal == "" {
return value.EmptyStringValue, false
}
if !strings.HasPrefix(urlstr, "http") {
urlstr = "http://" + urlstr
}
if urlParsed, err := url.Parse(urlstr); err == nil {
//u.Infof("url.parse: %#v", urlParsed)
qsval, ok := urlParsed.Query()[keyVal]
if !ok {
return value.EmptyStringValue, false
}
if len(qsval) > 0 {
return value.NewStringValue(qsval[0]), true
}
}
return value.EmptyStringValue, false
}
// todate: convert to Date
//
// todate(field) uses araddon\dateparse util to recognize formats
//
// todate("01/02/2006", field ) uses golang date parse rules
// first parameter is the layout/format
//
//
func ToDate(ctx expr.EvalContext, items ...value.Value) (value.TimeValue, bool) {
if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewTimeValue(t), true
}
} else if len(items) == 2 {
dateStr, ok := value.ToString(items[1].Rv())
if !ok {
return value.TimeZeroValue, false
}
formatStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("hello layout=%v time=%v", formatStr, dateStr)
if t, err := time.Parse(formatStr, dateStr); err == nil {
return value.NewTimeValue(t), true
}
}
return value.TimeZeroValue, false
}
func TimeExtractFunc(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
switch len(items) {
case 0:
// if we have no "items", return time associated with ctx
// This is an alias of now()
t := ctx.Ts()
if !t.IsZero() {
return value.NewStringValue(t.String()), true
}
return value.EmptyStringValue, false
case 1:
// if only 1 item, convert item to time
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
t, err := dateparse.ParseAny(dateStr)
if err != nil {
return value.EmptyStringValue, false
}
return value.NewStringValue(t.String()), true
case 2:
// if we have 2 items, the first is the time string
// and the second is the format string.
// Use leekchan/timeutil package
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
formatStr, ok := value.ToString(items[1].Rv())
if !ok {
return value.EmptyStringValue, false
}
t, err := dateparse.ParseAny(dateStr)
if err != nil {
return value.EmptyStringValue, false
}
formatted := timeutil.Strftime(&t, formatStr)
return value.NewStringValue(formatted), true
default:
return value.EmptyStringValue, false
}
}
// String contains
// Will first convert to string, so may get unexpected results
//
func ContainsFunc(ctx expr.EvalContext, lv, rv value.Value) (value.BoolValue, bool) {
left, leftOk := value.ToString(lv.Rv())
right, rightOk := value.ToString(rv.Rv())
if !leftOk || !rightOk {
return value.BoolValueFalse, false
}
//u.Infof("Contains(%v, %v)", left, right)
if left == "" || right == "" {
return value.BoolValueFalse, false
}
if strings.Contains(left, right) {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
}
// Split a string, accepts an optional with parameter
//
// split(item, ",")
//
func SplitFunc(ctx expr.EvalContext, input value.Value, splitByV value.StringValue) (value.StringsValue, bool) {
sv, ok := value.ToString(input.Rv())
splitBy, splitByOk := value.ToString(splitByV.Rv())
if !ok || !splitByOk {
return value.NewStringsValue(make([]string, 0)), false
}
if sv == "" {
return value.NewStringsValue(make([]string, 0)), false
}
if splitBy == "" {
return value.NewStringsValue(make([]string, 0)), false
}
vals := strings.Split(sv, splitBy)
return value.NewStringsValue(vals), true
}
// Get year in integer from field, must be able to convert to date
//
// yy() => 15, true // assuming it is 2015
// yy("2014-03-01") => 14, true
//
func Yy(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
yy := 0
if len(items) == 0 {
if !ctx.Ts().IsZero() {
yy = ctx.Ts().Year()
} else {
// Do we want to use Now()?
}
} else if len(items) == 1 {
//u.Debugf("has 1 items? %#v", items[0].Rv())
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Debugf("v=%v %v", dateStr, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err != nil {
return value.NewIntValue(0), false
} else {
yy = t.Year()
}
} else {
return value.NewIntValue(0), false
}
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Debugf("yy = %v", yy)
return value.NewIntValue(int64(yy)), true
}
// Get year in integer from date
func Yy(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
yy := 0
if len(items) == 0 {
if !ctx.Ts().IsZero() {
yy = ctx.Ts().Year()
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err != nil {
return value.NewIntValue(0), false
} else {
yy = t.Year()
}
} else {
return value.NewIntValue(0), false
}
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Infof("%v yy = %v", item, yy)
return value.NewIntValue(int64(yy)), true
}
// Join items together (string concatenation)
//
// join("apples","oranges",",") => "apples,oranges"
// join(["apples","oranges"],",") => "apples,oranges"
// join("apples","oranges","") => "applesoranges"
//
func JoinFunc(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
if len(items) <= 1 {
return value.EmptyStringValue, false
}
sep, ok := value.ToString(items[len(items)-1].Rv())
if !ok {
return value.EmptyStringValue, false
}
args := make([]string, 0)
for i := 0; i < len(items)-1; i++ {
switch valTyped := items[i].(type) {
case value.SliceValue:
vals := make([]string, len(valTyped.Val()))
for i, sv := range valTyped.Val() {
vals[i] = sv.ToString()
}
args = append(args, vals...)
case value.StringsValue:
vals := make([]string, len(valTyped.Val()))
for i, sv := range valTyped.Val() {
vals[i] = sv
}
args = append(args, vals...)
case value.StringValue, value.NumberValue, value.IntValue:
val := valTyped.ToString()
if val == "" {
return value.EmptyStringValue, false
}
args = append(args, val)
}
}
return value.NewStringValue(strings.Join(args, sep)), true
}
// String lower function
// must be able to convert to string
//
func Lower(ctx expr.EvalContext, item value.Value) (value.StringValue, bool) {
val, ok := value.ToString(item.Rv())
if !ok {
return value.EmptyStringValue, false
}
return value.NewStringValue(strings.ToLower(val)), true
}
// Example of a custom Function, that we are adding into the Expression VM
//
// select
// user_id AS theuserid, email, item_count * 2, reg_date
// FROM stdio
// WHERE email_is_valid(email)
func EmailIsValid(ctx expr.EvalContext, email value.Value) (value.BoolValue, bool) {
emailstr, ok := value.ToString(email.Rv())
if !ok || emailstr == "" {
return value.BoolValueFalse, true
}
if _, err := mail.ParseAddress(emailstr); err == nil {
return value.BoolValueTrue, true
}
return value.BoolValueFalse, true
}
// Join items
//
// join("applies","oranges",",") => "apples,oranges"
//
func JoinFunc(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
if len(items) <= 1 {
return value.EmptyStringValue, false
}
sep, ok := value.ToString(items[len(items)-1].Rv())
if !ok {
return value.EmptyStringValue, false
}
args := make([]string, 0)
for i := 0; i < len(items)-1; i++ {
val, ok := value.ToString(items[i].Rv())
if !ok {
return value.EmptyStringValue, false
}
if val == "" {
return value.EmptyStringValue, false
}
args = append(args, val)
}
return value.NewStringValue(strings.Join(args, sep)), true
}
// totimestamp: convert to date, then to unix Seconds
//
func ToTimestamp(ctx expr.EvalContext, item value.Value) (value.IntValue, bool) {
dateStr, ok := value.ToString(item.Rv())
if !ok {
return value.NewIntValue(0), false
}
if t, err := dateparse.ParseAny(dateStr); err == nil {
//u.Infof("v=%v %v unix=%v", item, item.Rv(), t.Unix())
return value.NewIntValue(int64(t.Unix())), true
}
return value.NewIntValue(0), false
}
// todate
func ToDate(ctx expr.EvalContext, item value.Value) (value.TimeValue, bool) {
dateStr, ok := value.ToString(item.Rv())
if !ok {
return value.TimeZeroValue, false
}
//u.Infof("v=%v %v ", v, item.Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewTimeValue(t), true
}
return value.TimeZeroValue, false
}
// emailname a string, parses email
//
// emailname("Bob <*****@*****.**>") => Bob
//
func EmailNameFunc(ctx expr.EvalContext, item value.Value) (value.StringValue, bool) {
val, ok := value.ToString(item.Rv())
if !ok {
return value.EmptyStringValue, false
}
if val == "" {
return value.EmptyStringValue, false
}
if len(val) < 6 {
return value.EmptyStringValue, false
}
if em, err := mail.ParseAddress(val); err == nil {
return value.NewStringValue(em.Name), true
}
return value.EmptyStringValue, false
}
// hour of day [0-23]
func HourOfDay(ctx expr.EvalContext, items ...value.Value) (value.IntValue, bool) {
if len(items) == 0 {
if !ctx.Ts().IsZero() {
return value.NewIntValue(int64(ctx.Ts().Hour())), true
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", v, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewIntValue(int64(t.Hour())), true
}
}
return value.NewIntValue(0), false
}
// Get yymm in 4 digits from argument if supplied, else uses message context ts
//
func YyMm(ctx expr.EvalContext, items ...value.Value) (value.StringValue, bool) {
if len(items) == 0 {
if !ctx.Ts().IsZero() {
t := ctx.Ts()
return value.NewStringValue(t.Format(yymmTimeLayout)), true
}
} else if len(items) == 1 {
dateStr, ok := value.ToString(items[0].Rv())
if !ok {
return value.EmptyStringValue, false
}
//u.Infof("v=%v %v ", v, items[0].Rv())
if t, err := dateparse.ParseAny(dateStr); err == nil {
return value.NewStringValue(t.Format(yymmTimeLayout)), true
}
}
return value.EmptyStringValue, false
}
func Yy(ctx expr.EvalContext, item value.Value) (value.IntValue, bool) {
//u.Info("yy: %T", item)
val, ok := value.ToString(item.Rv())
if !ok || val == "" {
return value.NewIntValue(0), false
}
//u.Infof("v=%v %v ", val, item.Rv())
if t, err := dateparse.ParseAny(val); err == nil {
yy := t.Year()
if yy >= 2000 {
yy = yy - 2000
} else if yy >= 1900 {
yy = yy - 1900
}
//u.Infof("Yy = %v yy = %v", item, yy)
return value.NewIntValue(int64(yy)), true
}
return value.NewIntValue(0), false
}
// email a string, parses email
//
// email("Bob <*****@*****.**>") => [email protected]
//
func EmailDomainFunc(ctx expr.EvalContext, item value.Value) (value.StringValue, bool) {
val, ok := value.ToString(item.Rv())
if !ok {
return value.EmptyStringValue, false
}
if val == "" {
return value.EmptyStringValue, false
}
if len(val) < 6 {
return value.EmptyStringValue, false
}
if em, err := mail.ParseAddress(strings.ToLower(val)); err == nil {
parts := strings.SplitN(strings.ToLower(em.Address), "@", 2)
if len(parts) == 2 {
return value.NewStringValue(parts[1]), true
}
}
return value.EmptyStringValue, false
}
// urlminusqs removes a specific query parameter and its value from a url
//
// urlminusqs("http://www.lytics.io/?q1=google&q2=123", "q1") => "http://www.lytics.io/?q2=123", true
//
func UrlMinusQs(ctx expr.EvalContext, urlItem, keyItem value.Value) (value.StringValue, bool) {
val := ""
switch itemT := urlItem.(type) {
case value.StringValue:
val = itemT.Val()
case value.StringsValue:
if len(itemT.Val()) == 0 {
return value.EmptyStringValue, false
}
val = itemT.Val()[0]
}
if val == "" {
return value.EmptyStringValue, false
}
if !strings.HasPrefix(val, "http") {
val = "http://" + val
}
keyVal, ok := value.ToString(keyItem.Rv())
if !ok {
return value.EmptyStringValue, false
}
if keyVal == "" {
return value.EmptyStringValue, false
}
if up, err := url.Parse(val); err == nil {
qsval := up.Query()
_, ok := qsval[keyVal]
if !ok {
return value.NewStringValue(fmt.Sprintf("%s://%s%s?%s", up.Scheme, up.Host, up.Path, up.RawQuery)), true
}
qsval.Del(keyVal)
up.RawQuery = qsval.Encode()
if up.RawQuery == "" {
return value.NewStringValue(fmt.Sprintf("%s://%s%s", up.Scheme, up.Host, up.Path)), true
}
return value.NewStringValue(fmt.Sprintf("%s://%s%s?%s", up.Scheme, up.Host, up.Path, up.RawQuery)), true
}
return value.EmptyStringValue, false
}
// Extract url path from a String (must be urlish), doesn't do much/any validation
func UrlPath(ctx expr.EvalContext, item value.Value) (value.StringValue, bool) {
val, ok := value.ToString(item.Rv())
if !ok {
return value.EmptyStringValue, false
}
if val == "" {
return value.EmptyStringValue, false
}
urlstr := strings.ToLower(val)
if len(urlstr) < 8 {
return value.EmptyStringValue, false
}
if !strings.HasPrefix(urlstr, "http") {
urlstr = "http://" + urlstr
}
if urlParsed, err := url.Parse(urlstr); err == nil {
//u.Infof("url.parse: %#v", urlParsed)
return value.NewStringValue(urlParsed.Path), true
}
return value.EmptyStringValue, false
}