// Write - Writes body to a log - `log.write`.
func Write(ctx action.Map) (action.Map, error) {
// Get string body from arguments
if body, ok := ctx.Get("body").String(); ok {
glog.Info(body)
}
return ctx, nil
}
// Extract - extracts pieces of html in `body` using `selectors`
func Extract(ctx action.Map) (action.Map, error) {
body, ok := ctx.Pop("body").Reader()
if !ok {
return ctx, selector.ErrNoBody
}
defer body.Close()
selectors, ok := ctx.Pop("selectors").Map()
if !ok {
return ctx, ErrNoSelectors
}
glog.V(2).Infof("html.extract selectors=%v\n", selectors)
// Extract from body
result, err := ExtractSelectors(selectors, body)
if err != nil {
glog.V(2).Infof("html.extract err=%v\n", err)
return ctx, err
}
glog.V(2).Infof("html.extract result=%v\n", result)
return result, nil
}
// Run - runs command. It does not support streams yet!
func Run(ctx action.Map) (action.Map, error) {
var bin string // command name
var args []string // command arguments
// Command name with arguments
name := ctx.Pop("name")
// Check if name is not empty
if name.IsNil() {
return ctx, ErrNoCommand
}
// Split cmd to a name and args
var words []string
if cmdname, ok := name.String(); ok {
words = strings.Split(cmdname, " ")
} else {
words, _ = name.StringList()
}
if len(words) > 0 {
bin = words[0]
args = words[1:]
}
// Get command arguments
a := ctx.Pop("args")
if arg, ok := a.String(); ok {
args = append(args, strings.Split(arg, " ")...)
} else if arg, ok := a.StringList(); ok {
args = append(args, arg...)
}
// Execute command
cmd := exec.Command(bin, args...)
// Run command and get output
output, err := cmd.CombinedOutput()
// Action result
result := action.Map{
"success": cmd.ProcessState.Success(),
"output": output,
"time": action.Map{
"system": cmd.ProcessState.SystemTime(),
"user": cmd.ProcessState.UserTime(),
},
}
if err != nil {
result["error"] = err.Error()
return result, err
}
return result, nil
}
// mapToRegexpMap - Converts a map to RegexpMap.
func mapToRegexpMap(m action.Map) (remap RegexpMap) {
remap = make(RegexpMap)
for name := range m {
// Get and convert value to a Map
if m, ok := m.Get(name).Map(); ok {
// Convert Map to Regexp
re := mapToRegexp(m)
// If Regexp name is empty use key value
if re.Name == "" {
re.Name = name
}
// Add Regexp to RegexpMap
remap[name] = re
}
}
return
}
// mapToSelector - Creates an Extractor ouf of action.Map.
func mapToSelector(m action.Map) selector.Extractor {
// Selector path
path, _ := m.Get("$path").String()
// Selector extractor (it can be a string (attribute extractor), empty (text extractor))
// it can be another selector or map of selectors
extract := m.Get("$extract")
// If $extract is empty - extractor is TextExtractor
if extract.IsNil() {
return &selector.Selector{
Extractor: selector.TextExtractor,
Path: path,
}
}
// If $extract is a string, extractor is an attribute Extractor
if name, ok := extract.String(); ok {
return &selector.Selector{
Path: path,
Extractor: selector.GetExtractor(name),
}
}
// If $extract is a selector we have to create deep selector
if IsSelector(extract) {
return &selector.Selector{
Path: path,
Extractor: ToSelector(extract),
}
}
// Or a map of selectors
if m, ok := extract.Map(); ok {
return &selector.Selector{
Path: path,
Extractor: MapSelectors(m),
}
}
// $extract is not expected value
return nil
}
// Extract - Extracts pattern from value under Name key in ctx.
func (r *Regexp) Extract(ctx action.Map) interface{} {
// Get value `r.Name` from context
value := ctx.Get(r.Name)
// Pass if empty value
if value.IsNil() {
return nil
}
// Compile regular expression
re, err := r.Compile()
if err != nil {
return nil
}
// If it's a string - use in it
if strvalue, ok := value.String(); ok {
// Find all
if r.All {
return re.FindAllString(strvalue, -1)
}
// Find one
return re.FindString(strvalue)
}
// If it's a byte array - use in it
if bytes, ok := value.Bytes(); ok {
// Find all
if r.All {
return re.FindAll(bytes, -1)
}
// Find one
return re.Find(bytes)
}
// If it's nothing else - pass
return nil
}
// Extract - Extracts patterns from context.
func Extract(ctx action.Map) (action.Map, error) {
// Pop regexp value containig RegexpMap
value := ctx.Pop("regexp")
// If value is nil return error
if value.IsNil() {
return nil, fmt.Errorf("Regexp value can not be empty.")
}
// Convert value to a map
remap, ok := value.Map()
if !ok {
return nil, fmt.Errorf("Regexp value `%#v` is not valid.", ctx.Get("regexp").Value)
}
// Convert a map to RegexpMap
re := mapToRegexpMap(remap)
// Extract all Regexps
result := re.Extract(ctx)
// Merge result into context
ctx.Merge(result)
// Return context
return ctx, nil
}
// MapSelectors - Iterates over map of selectors initializaing proper structures.
func MapSelectors(selectors action.Map) (res selector.SelectorsMap) {
res = selector.SelectorsMap{}
// Map all values in selectors map
for name := range selectors {
// Get value by name
value := selectors.Get(name)
// If value is a selector create structures and continue
if IsSelector(value) {
res[name] = ToSelector(value)
continue
}
// If it's not a selector but a map it's a map of selectors
if m, ok := value.Map(); ok {
res[name] = MapSelectors(m)
}
}
// Return result
return
}
// mapToRegexp - Converts a map to Regexp.
func mapToRegexp(m action.Map) *Regexp {
var (
pattern, _ = m.Get("pattern").String()
name, _ = m.Get("name").String()
all, _ = m.Get("all").Bool()
)
return &Regexp{
Pattern: pattern,
Name: name,
All: all,
}
}
// mapToAction - Converts a map to action.
func mapToAction(m action.Map) *action.Action {
if m == nil {
return nil
}
var (
name, _ = m.Get("name").String()
context, _ = m.Get("ctx").Map()
next, _ = m.Get("next").Map()
)
return &action.Action{
Name: name,
Ctx: context,
Next: mapToAction(next),
}
}
// Push - Pushes values from `push` context map to context.
func Push(ctx action.Map) (action.Map, error) {
p := ctx.Pop("push")
// If it is a string push current context under `push` name to a new one
if push, ok := p.String(); ok {
clean := make(action.Map)
clean.Push(push, ctx)
return clean, nil
}
// If it is a map iterate through `push` map and pull values by `key` from context and push under `value`
if push, ok := p.Map(); ok {
for key := range push {
if value := ctx.Pop(key); !value.IsNil() {
name, _ := push.Get(key).String()
ctx.Push(name, value.Value)
glog.V(3).Infof("Pushing %#v value %#v under name %#v", key, value, name)
}
}
return ctx, nil
}
return nil, ErrPushUnexpected
}
// Request - Sends a http request.
func Request(ctx action.Map) (action.Map, error) {
var (
uri, _ = ctx.Get("url").String()
query = ctx.Pop("query")
method = "GET"
)
// Parse URL and join query
u, err := joinURL(uri, query)
if err != nil {
return ctx, err
}
// Get method from context
if m, ok := ctx.Pop("method").String(); ok {
method = strings.ToUpper(m)
}
// Set scheme from context
if scheme, ok := ctx.Pop("scheme").String(); ok {
u.Scheme = scheme
}
// Set pathname from context
if pathname, ok := ctx.Pop("pathname").String(); ok {
u.Path = pathname
}
// Set hostname from context
if hostname, ok := ctx.Pop("hostname").String(); ok {
u.Host = hostname
}
// Set http scheme if empty
if u.Scheme == "" {
u.Scheme = "http"
}
// HTTP Request structure
req := &http.Request{
URL: u,
Method: method,
}
// Set header from context
if reqhead, ok := ctx.Pop("header").Map(); ok {
req.Header = make(http.Header)
for key := range reqhead {
if strvalue, ok := reqhead.Get(key).String(); ok {
req.Header.Add(key, strvalue)
} else if listvalue, ok := reqhead.Get(key).StringList(); ok {
req.Header[key] = listvalue
}
}
}
if glog.V(3) {
glog.Infof("http.request %s %s\n", method, u)
}
// Request body
body := ctx.Pop("body")
// Set request body if any and method is not GET
if reqbody, ok := body.Reader(); ok && method != "GET" {
defer func() {
err := reqbody.Close()
if err != nil && glog.V(2) {
glog.Infof("error closing request body %v\n", err)
}
}()
switch body.Value.(type) {
case []byte:
v, _ := body.Value.([]byte)
req.ContentLength = int64(len(v))
case string:
v, _ := body.Value.(string)
req.ContentLength = int64(len(v))
default:
if length, ok := ctx.Get("Content-Length").Int(); ok {
req.ContentLength = int64(length)
} else {
// TODO: HTTP CHUNK
return nil, ErrEmptyContentLength
}
}
req.Body = reqbody
}
// Send request
resp, err := http.DefaultClient.Do(req)
if err != nil {
if glog.V(2) {
glog.Infof("http.response error %s %s - %v\n", method, u, err)
}
return ctx, err
}
// Response
ctx.Add("status", resp.StatusCode)
ctx.Add("header", headerToMap(resp.Header))
ctx.Add("body", resp.Body)
ctx.Add("url", u.String())
// Log response
if glog.V(3) {
glog.Infof("http.response %s %s - %s\n", method, u, resp.Status)
}
// Return result
return ctx, err
}
// TrimSpaces - filter.trim - trims all string values in context from `trim` list.
// Example:
// {
// "name": "filter.trim",
// "ctx": {
// "trim": ["content", "other"],
// "content": " some content to trim out of white space ",
// "other": " some other content to trim out of white space "
// }
// }
func TrimSpaces(ctx action.Map) (action.Map, error) {
// Pop trim value - context keys to trim
ctx.Transform(ctx.Pop("trim"), trimFunc)
return ctx, nil
}
// Decode - json.decode - decodes all JSON values in context from `decode` list.
// Example:
// {
// "name": "json.decode",
// "ctx": {
// "decode": ["content"],
// "content": "[1, 2, 3]"
// }
// }
func Decode(ctx action.Map) (action.Map, error) {
// Pop decode value - context keys to decode
ctx.Transform(ctx.Pop("decode"), decodeFunc)
return ctx, nil
}
// Encode - json.encode - encodes all values in context from `encode` list to JSON.
// Example:
// {
// "name": "json.encode",
// "ctx": {
// "encode": ["content"],
// "content": [1, 2, 3]
// }
// }
func Encode(ctx action.Map) (action.Map, error) {
// Pop encode value - context keys to encode
ctx.Transform(ctx.Pop("encode"), encodeFunc)
return ctx, nil
}
// Clear - Removes all context variables except those defined in `context` argument.
func Clear(ctx action.Map) (action.Map, error) {
leave := ctx.Pull("context")
// If `context` argument is empty -- clear entire context
if leave.IsNil() {
return action.Map{}, nil
}
// If `context` is a map -- it's a new context
if clean, ok := leave.Map(); ok {
return clean, nil
}
// If `context` is a string -- leave one variable
if name, ok := leave.String(); ok {
// Pop value
value := ctx.Pop(name).Value
// Close old context
ctx.Close()
return action.Map{name: value}, nil
}
// If `context` is a list -- create a new context and close old one
list, _ := ctx.Pull("context").StringList()
clean := make(action.Map)
// Save all values from list
for _, name := range list {
value := ctx.Pop(name).Value
clean.Add(name, value)
}
// Close old context
ctx.Close()
return clean, nil
}
// Split - actions.split
func Split(ctx action.Map) (action.Map, error) {
glog.V(2).Info("actions.split start\n")
split, ok := ctx.Pop("split").Map()
if !ok {
return nil, fmt.Errorf("Split value %#v is not valid", split)
}
// Create a running group
g := group.New()
// Split execution
for key := range split {
// What to do
do, _ := split.Pop(key).Map()
// Action name
name, _ := do.Get("name").String()
actx, _ := do.Get("ctx").Map()
if actx == nil {
actx = make(action.Map)
}
// Name of pushed iterated value
as, _ := do.Get("as").String()
// Value to split
value := ctx.Pop(key)
// If its a value -- iterate on it and run action on every element
if list, ok := value.List(); ok {
for _, v := range list {
// run `do` action with `v` as `do.as`
c := actx.Clone()
c[as] = v
g.Add(&action.Action{Name: name, Ctx: c})
glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, c)
}
} else {
// run `do` action with `value` as `do.as`
actx[as] = value.Value
g.Add(&action.Action{Name: name, Ctx: actx})
glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, actx)
}
}
err := <-g.Wait()
if err != nil {
glog.V(2).Infof("actions.split run err=%v \n", err)
return nil, err
}
glog.V(3).Infof("actions.split run results=%v\n", g.Results)
// What to collect
c := ctx.Pop("collect")
// If `collect` is a string -- add it to list
var collect []string // context variables to collect
if collectone, ok := c.String(); ok {
collect = append(collect, collectone)
} else if collect, ok = c.StringList(); !ok {
return action.Map{"results": g.Results}, nil
}
// Collect results
results := make(action.Map)
for _, result := range g.Results {
for _, key := range collect {
var list []interface{}
if results[key] != nil {
list, _ = results[key].([]interface{})
}
results[key] = append(list, result.Value(key))
}
}
glog.V(3).Infof("actions.split collect results=%v\n", results)
return results, nil
}