// Write - Writes body to a log - `log.write`. func Write(ctx action.Map) (action.Map, error) { // Get string body from arguments if body, ok := ctx.Get("body").String(); ok { glog.Info(body) } return ctx, nil }
// Extract - extracts pieces of html in `body` using `selectors` func Extract(ctx action.Map) (action.Map, error) { body, ok := ctx.Pop("body").Reader() if !ok { return ctx, selector.ErrNoBody } defer body.Close() selectors, ok := ctx.Pop("selectors").Map() if !ok { return ctx, ErrNoSelectors } glog.V(2).Infof("html.extract selectors=%v\n", selectors) // Extract from body result, err := ExtractSelectors(selectors, body) if err != nil { glog.V(2).Infof("html.extract err=%v\n", err) return ctx, err } glog.V(2).Infof("html.extract result=%v\n", result) return result, nil }
// Run - runs command. It does not support streams yet! func Run(ctx action.Map) (action.Map, error) { var bin string // command name var args []string // command arguments // Command name with arguments name := ctx.Pop("name") // Check if name is not empty if name.IsNil() { return ctx, ErrNoCommand } // Split cmd to a name and args var words []string if cmdname, ok := name.String(); ok { words = strings.Split(cmdname, " ") } else { words, _ = name.StringList() } if len(words) > 0 { bin = words[0] args = words[1:] } // Get command arguments a := ctx.Pop("args") if arg, ok := a.String(); ok { args = append(args, strings.Split(arg, " ")...) } else if arg, ok := a.StringList(); ok { args = append(args, arg...) } // Execute command cmd := exec.Command(bin, args...) // Run command and get output output, err := cmd.CombinedOutput() // Action result result := action.Map{ "success": cmd.ProcessState.Success(), "output": output, "time": action.Map{ "system": cmd.ProcessState.SystemTime(), "user": cmd.ProcessState.UserTime(), }, } if err != nil { result["error"] = err.Error() return result, err } return result, nil }
// mapToRegexpMap - Converts a map to RegexpMap. func mapToRegexpMap(m action.Map) (remap RegexpMap) { remap = make(RegexpMap) for name := range m { // Get and convert value to a Map if m, ok := m.Get(name).Map(); ok { // Convert Map to Regexp re := mapToRegexp(m) // If Regexp name is empty use key value if re.Name == "" { re.Name = name } // Add Regexp to RegexpMap remap[name] = re } } return }
// mapToSelector - Creates an Extractor ouf of action.Map. func mapToSelector(m action.Map) selector.Extractor { // Selector path path, _ := m.Get("$path").String() // Selector extractor (it can be a string (attribute extractor), empty (text extractor)) // it can be another selector or map of selectors extract := m.Get("$extract") // If $extract is empty - extractor is TextExtractor if extract.IsNil() { return &selector.Selector{ Extractor: selector.TextExtractor, Path: path, } } // If $extract is a string, extractor is an attribute Extractor if name, ok := extract.String(); ok { return &selector.Selector{ Path: path, Extractor: selector.GetExtractor(name), } } // If $extract is a selector we have to create deep selector if IsSelector(extract) { return &selector.Selector{ Path: path, Extractor: ToSelector(extract), } } // Or a map of selectors if m, ok := extract.Map(); ok { return &selector.Selector{ Path: path, Extractor: MapSelectors(m), } } // $extract is not expected value return nil }
// Extract - Extracts pattern from value under Name key in ctx. func (r *Regexp) Extract(ctx action.Map) interface{} { // Get value `r.Name` from context value := ctx.Get(r.Name) // Pass if empty value if value.IsNil() { return nil } // Compile regular expression re, err := r.Compile() if err != nil { return nil } // If it's a string - use in it if strvalue, ok := value.String(); ok { // Find all if r.All { return re.FindAllString(strvalue, -1) } // Find one return re.FindString(strvalue) } // If it's a byte array - use in it if bytes, ok := value.Bytes(); ok { // Find all if r.All { return re.FindAll(bytes, -1) } // Find one return re.Find(bytes) } // If it's nothing else - pass return nil }
// Extract - Extracts patterns from context. func Extract(ctx action.Map) (action.Map, error) { // Pop regexp value containig RegexpMap value := ctx.Pop("regexp") // If value is nil return error if value.IsNil() { return nil, fmt.Errorf("Regexp value can not be empty.") } // Convert value to a map remap, ok := value.Map() if !ok { return nil, fmt.Errorf("Regexp value `%#v` is not valid.", ctx.Get("regexp").Value) } // Convert a map to RegexpMap re := mapToRegexpMap(remap) // Extract all Regexps result := re.Extract(ctx) // Merge result into context ctx.Merge(result) // Return context return ctx, nil }
// MapSelectors - Iterates over map of selectors initializaing proper structures. func MapSelectors(selectors action.Map) (res selector.SelectorsMap) { res = selector.SelectorsMap{} // Map all values in selectors map for name := range selectors { // Get value by name value := selectors.Get(name) // If value is a selector create structures and continue if IsSelector(value) { res[name] = ToSelector(value) continue } // If it's not a selector but a map it's a map of selectors if m, ok := value.Map(); ok { res[name] = MapSelectors(m) } } // Return result return }
// mapToRegexp - Converts a map to Regexp. func mapToRegexp(m action.Map) *Regexp { var ( pattern, _ = m.Get("pattern").String() name, _ = m.Get("name").String() all, _ = m.Get("all").Bool() ) return &Regexp{ Pattern: pattern, Name: name, All: all, } }
// mapToAction - Converts a map to action. func mapToAction(m action.Map) *action.Action { if m == nil { return nil } var ( name, _ = m.Get("name").String() context, _ = m.Get("ctx").Map() next, _ = m.Get("next").Map() ) return &action.Action{ Name: name, Ctx: context, Next: mapToAction(next), } }
// Push - Pushes values from `push` context map to context. func Push(ctx action.Map) (action.Map, error) { p := ctx.Pop("push") // If it is a string push current context under `push` name to a new one if push, ok := p.String(); ok { clean := make(action.Map) clean.Push(push, ctx) return clean, nil } // If it is a map iterate through `push` map and pull values by `key` from context and push under `value` if push, ok := p.Map(); ok { for key := range push { if value := ctx.Pop(key); !value.IsNil() { name, _ := push.Get(key).String() ctx.Push(name, value.Value) glog.V(3).Infof("Pushing %#v value %#v under name %#v", key, value, name) } } return ctx, nil } return nil, ErrPushUnexpected }
// Request - Sends a http request. func Request(ctx action.Map) (action.Map, error) { var ( uri, _ = ctx.Get("url").String() query = ctx.Pop("query") method = "GET" ) // Parse URL and join query u, err := joinURL(uri, query) if err != nil { return ctx, err } // Get method from context if m, ok := ctx.Pop("method").String(); ok { method = strings.ToUpper(m) } // Set scheme from context if scheme, ok := ctx.Pop("scheme").String(); ok { u.Scheme = scheme } // Set pathname from context if pathname, ok := ctx.Pop("pathname").String(); ok { u.Path = pathname } // Set hostname from context if hostname, ok := ctx.Pop("hostname").String(); ok { u.Host = hostname } // Set http scheme if empty if u.Scheme == "" { u.Scheme = "http" } // HTTP Request structure req := &http.Request{ URL: u, Method: method, } // Set header from context if reqhead, ok := ctx.Pop("header").Map(); ok { req.Header = make(http.Header) for key := range reqhead { if strvalue, ok := reqhead.Get(key).String(); ok { req.Header.Add(key, strvalue) } else if listvalue, ok := reqhead.Get(key).StringList(); ok { req.Header[key] = listvalue } } } if glog.V(3) { glog.Infof("http.request %s %s\n", method, u) } // Request body body := ctx.Pop("body") // Set request body if any and method is not GET if reqbody, ok := body.Reader(); ok && method != "GET" { defer func() { err := reqbody.Close() if err != nil && glog.V(2) { glog.Infof("error closing request body %v\n", err) } }() switch body.Value.(type) { case []byte: v, _ := body.Value.([]byte) req.ContentLength = int64(len(v)) case string: v, _ := body.Value.(string) req.ContentLength = int64(len(v)) default: if length, ok := ctx.Get("Content-Length").Int(); ok { req.ContentLength = int64(length) } else { // TODO: HTTP CHUNK return nil, ErrEmptyContentLength } } req.Body = reqbody } // Send request resp, err := http.DefaultClient.Do(req) if err != nil { if glog.V(2) { glog.Infof("http.response error %s %s - %v\n", method, u, err) } return ctx, err } // Response ctx.Add("status", resp.StatusCode) ctx.Add("header", headerToMap(resp.Header)) ctx.Add("body", resp.Body) ctx.Add("url", u.String()) // Log response if glog.V(3) { glog.Infof("http.response %s %s - %s\n", method, u, resp.Status) } // Return result return ctx, err }
// TrimSpaces - filter.trim - trims all string values in context from `trim` list. // Example: // { // "name": "filter.trim", // "ctx": { // "trim": ["content", "other"], // "content": " some content to trim out of white space ", // "other": " some other content to trim out of white space " // } // } func TrimSpaces(ctx action.Map) (action.Map, error) { // Pop trim value - context keys to trim ctx.Transform(ctx.Pop("trim"), trimFunc) return ctx, nil }
// Decode - json.decode - decodes all JSON values in context from `decode` list. // Example: // { // "name": "json.decode", // "ctx": { // "decode": ["content"], // "content": "[1, 2, 3]" // } // } func Decode(ctx action.Map) (action.Map, error) { // Pop decode value - context keys to decode ctx.Transform(ctx.Pop("decode"), decodeFunc) return ctx, nil }
// Encode - json.encode - encodes all values in context from `encode` list to JSON. // Example: // { // "name": "json.encode", // "ctx": { // "encode": ["content"], // "content": [1, 2, 3] // } // } func Encode(ctx action.Map) (action.Map, error) { // Pop encode value - context keys to encode ctx.Transform(ctx.Pop("encode"), encodeFunc) return ctx, nil }
// Clear - Removes all context variables except those defined in `context` argument. func Clear(ctx action.Map) (action.Map, error) { leave := ctx.Pull("context") // If `context` argument is empty -- clear entire context if leave.IsNil() { return action.Map{}, nil } // If `context` is a map -- it's a new context if clean, ok := leave.Map(); ok { return clean, nil } // If `context` is a string -- leave one variable if name, ok := leave.String(); ok { // Pop value value := ctx.Pop(name).Value // Close old context ctx.Close() return action.Map{name: value}, nil } // If `context` is a list -- create a new context and close old one list, _ := ctx.Pull("context").StringList() clean := make(action.Map) // Save all values from list for _, name := range list { value := ctx.Pop(name).Value clean.Add(name, value) } // Close old context ctx.Close() return clean, nil }
// Split - actions.split func Split(ctx action.Map) (action.Map, error) { glog.V(2).Info("actions.split start\n") split, ok := ctx.Pop("split").Map() if !ok { return nil, fmt.Errorf("Split value %#v is not valid", split) } // Create a running group g := group.New() // Split execution for key := range split { // What to do do, _ := split.Pop(key).Map() // Action name name, _ := do.Get("name").String() actx, _ := do.Get("ctx").Map() if actx == nil { actx = make(action.Map) } // Name of pushed iterated value as, _ := do.Get("as").String() // Value to split value := ctx.Pop(key) // If its a value -- iterate on it and run action on every element if list, ok := value.List(); ok { for _, v := range list { // run `do` action with `v` as `do.as` c := actx.Clone() c[as] = v g.Add(&action.Action{Name: name, Ctx: c}) glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, c) } } else { // run `do` action with `value` as `do.as` actx[as] = value.Value g.Add(&action.Action{Name: name, Ctx: actx}) glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, actx) } } err := <-g.Wait() if err != nil { glog.V(2).Infof("actions.split run err=%v \n", err) return nil, err } glog.V(3).Infof("actions.split run results=%v\n", g.Results) // What to collect c := ctx.Pop("collect") // If `collect` is a string -- add it to list var collect []string // context variables to collect if collectone, ok := c.String(); ok { collect = append(collect, collectone) } else if collect, ok = c.StringList(); !ok { return action.Map{"results": g.Results}, nil } // Collect results results := make(action.Map) for _, result := range g.Results { for _, key := range collect { var list []interface{} if results[key] != nil { list, _ = results[key].([]interface{}) } results[key] = append(list, result.Value(key)) } } glog.V(3).Infof("actions.split collect results=%v\n", results) return results, nil }