Exemple #1
0
// Extract - extracts pieces of html in `body` using `selectors`
func Extract(ctx action.Map) (action.Map, error) {
	body, ok := ctx.Pop("body").Reader()
	if !ok {
		return ctx, selector.ErrNoBody
	}
	defer body.Close()

	selectors, ok := ctx.Pop("selectors").Map()
	if !ok {
		return ctx, ErrNoSelectors
	}

	glog.V(2).Infof("html.extract selectors=%v\n", selectors)

	// Extract from body
	result, err := ExtractSelectors(selectors, body)
	if err != nil {
		glog.V(2).Infof("html.extract err=%v\n", err)
		return ctx, err
	}

	glog.V(2).Infof("html.extract result=%v\n", result)

	return result, nil
}
Exemple #2
0
// Extract - Extracts patterns from context.
func Extract(ctx action.Map) (action.Map, error) {
	// Pop regexp value containig RegexpMap
	value := ctx.Pop("regexp")

	// If value is nil return error
	if value.IsNil() {
		return nil, fmt.Errorf("Regexp value can not be empty.")
	}

	// Convert value to a map
	remap, ok := value.Map()
	if !ok {
		return nil, fmt.Errorf("Regexp value `%#v` is not valid.", ctx.Get("regexp").Value)
	}

	// Convert a map to RegexpMap
	re := mapToRegexpMap(remap)

	// Extract all Regexps
	result := re.Extract(ctx)

	// Merge result into context
	ctx.Merge(result)

	// Return context
	return ctx, nil
}
Exemple #3
0
// Run - runs command. It does not support streams yet!
func Run(ctx action.Map) (action.Map, error) {
	var bin string    // command name
	var args []string // command arguments

	// Command name with arguments
	name := ctx.Pop("name")

	// Check if name is not empty
	if name.IsNil() {
		return ctx, ErrNoCommand
	}

	// Split cmd to a name and args
	var words []string
	if cmdname, ok := name.String(); ok {
		words = strings.Split(cmdname, " ")
	} else {
		words, _ = name.StringList()
	}

	if len(words) > 0 {
		bin = words[0]
		args = words[1:]
	}

	// Get command arguments
	a := ctx.Pop("args")
	if arg, ok := a.String(); ok {
		args = append(args, strings.Split(arg, " ")...)
	} else if arg, ok := a.StringList(); ok {
		args = append(args, arg...)
	}

	// Execute command
	cmd := exec.Command(bin, args...)

	// Run command and get output
	output, err := cmd.CombinedOutput()

	// Action result
	result := action.Map{
		"success": cmd.ProcessState.Success(),
		"output":  output,
		"time": action.Map{
			"system": cmd.ProcessState.SystemTime(),
			"user":   cmd.ProcessState.UserTime(),
		},
	}

	if err != nil {
		result["error"] = err.Error()
		return result, err
	}

	return result, nil
}
Exemple #4
0
// Clear - Removes all context variables except those defined in `context` argument.
func Clear(ctx action.Map) (action.Map, error) {
	leave := ctx.Pull("context")

	// If `context` argument is empty -- clear entire context
	if leave.IsNil() {
		return action.Map{}, nil
	}

	// If `context` is a map -- it's a new context
	if clean, ok := leave.Map(); ok {
		return clean, nil
	}

	// If `context` is a string -- leave one variable
	if name, ok := leave.String(); ok {
		// Pop value
		value := ctx.Pop(name).Value

		// Close old context
		ctx.Close()
		return action.Map{name: value}, nil
	}

	// If `context` is a list -- create a new context and close old one
	list, _ := ctx.Pull("context").StringList()
	clean := make(action.Map)

	// Save all values from list
	for _, name := range list {
		value := ctx.Pop(name).Value
		clean.Add(name, value)
	}

	// Close old context
	ctx.Close()

	return clean, nil
}
Exemple #5
0
// Push - Pushes values from `push` context map to context.
func Push(ctx action.Map) (action.Map, error) {
	p := ctx.Pop("push")

	// If it is a string push current context under `push` name to a new one
	if push, ok := p.String(); ok {
		clean := make(action.Map)
		clean.Push(push, ctx)
		return clean, nil
	}

	// If it is a map iterate through `push` map and pull values by `key` from context and push under `value`
	if push, ok := p.Map(); ok {
		for key := range push {
			if value := ctx.Pop(key); !value.IsNil() {
				name, _ := push.Get(key).String()
				ctx.Push(name, value.Value)
				glog.V(3).Infof("Pushing %#v value %#v under name %#v", key, value, name)
			}
		}
		return ctx, nil
	}

	return nil, ErrPushUnexpected
}
Exemple #6
0
// TrimSpaces - filter.trim - trims all string values in context from `trim` list.
// Example:
// 	{
//		"name": "filter.trim",
//		"ctx": {
//			"trim": ["content", "other"],
//			"content": "   some content to trim out of white space    ",
//			"other": "   some other content to trim out of white space    "
//		}
//	}
func TrimSpaces(ctx action.Map) (action.Map, error) {
	// Pop trim value - context keys to trim
	ctx.Transform(ctx.Pop("trim"), trimFunc)

	return ctx, nil
}
Exemple #7
0
// Request - Sends a http request.
func Request(ctx action.Map) (action.Map, error) {
	var (
		uri, _ = ctx.Get("url").String()
		query  = ctx.Pop("query")
		method = "GET"
	)

	// Parse URL and join query
	u, err := joinURL(uri, query)
	if err != nil {
		return ctx, err
	}

	// Get method from context
	if m, ok := ctx.Pop("method").String(); ok {
		method = strings.ToUpper(m)
	}

	// Set scheme from context
	if scheme, ok := ctx.Pop("scheme").String(); ok {
		u.Scheme = scheme
	}

	// Set pathname from context
	if pathname, ok := ctx.Pop("pathname").String(); ok {
		u.Path = pathname
	}

	// Set hostname from context
	if hostname, ok := ctx.Pop("hostname").String(); ok {
		u.Host = hostname
	}

	// Set http scheme if empty
	if u.Scheme == "" {
		u.Scheme = "http"
	}

	// HTTP Request structure
	req := &http.Request{
		URL:    u,
		Method: method,
	}

	// Set header from context
	if reqhead, ok := ctx.Pop("header").Map(); ok {
		req.Header = make(http.Header)
		for key := range reqhead {
			if strvalue, ok := reqhead.Get(key).String(); ok {
				req.Header.Add(key, strvalue)
			} else if listvalue, ok := reqhead.Get(key).StringList(); ok {
				req.Header[key] = listvalue
			}
		}
	}

	if glog.V(3) {
		glog.Infof("http.request %s %s\n", method, u)
	}

	// Request body
	body := ctx.Pop("body")

	// Set request body if any and method is not GET
	if reqbody, ok := body.Reader(); ok && method != "GET" {
		defer func() {
			err := reqbody.Close()
			if err != nil && glog.V(2) {
				glog.Infof("error closing request body %v\n", err)
			}
		}()

		switch body.Value.(type) {
		case []byte:
			v, _ := body.Value.([]byte)
			req.ContentLength = int64(len(v))
		case string:
			v, _ := body.Value.(string)
			req.ContentLength = int64(len(v))
		default:
			if length, ok := ctx.Get("Content-Length").Int(); ok {
				req.ContentLength = int64(length)
			} else {
				// TODO: HTTP CHUNK
				return nil, ErrEmptyContentLength
			}
		}

		req.Body = reqbody
	}

	// Send request
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		if glog.V(2) {
			glog.Infof("http.response error %s %s - %v\n", method, u, err)
		}
		return ctx, err
	}

	// Response
	ctx.Add("status", resp.StatusCode)
	ctx.Add("header", headerToMap(resp.Header))
	ctx.Add("body", resp.Body)
	ctx.Add("url", u.String())

	// Log response
	if glog.V(3) {
		glog.Infof("http.response %s %s - %s\n", method, u, resp.Status)
	}

	// Return result
	return ctx, err
}
Exemple #8
0
// Split - actions.split
func Split(ctx action.Map) (action.Map, error) {
	glog.V(2).Info("actions.split start\n")
	split, ok := ctx.Pop("split").Map()
	if !ok {
		return nil, fmt.Errorf("Split value %#v is not valid", split)
	}

	// Create a running group
	g := group.New()

	// Split execution
	for key := range split {
		// What to do
		do, _ := split.Pop(key).Map()
		// Action name
		name, _ := do.Get("name").String()
		actx, _ := do.Get("ctx").Map()
		if actx == nil {
			actx = make(action.Map)
		}

		// Name of pushed iterated value
		as, _ := do.Get("as").String()

		// Value to split
		value := ctx.Pop(key)

		// If its a value -- iterate on it and run action on every element
		if list, ok := value.List(); ok {
			for _, v := range list {
				// run `do` action with `v` as `do.as`
				c := actx.Clone()
				c[as] = v
				g.Add(&action.Action{Name: name, Ctx: c})
				glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, c)
			}
		} else {
			// run `do` action with `value` as `do.as`
			actx[as] = value.Value
			g.Add(&action.Action{Name: name, Ctx: actx})
			glog.V(2).Infof("actions.split run name=%s ctx=%#v\n", name, actx)
		}
	}

	err := <-g.Wait()
	if err != nil {
		glog.V(2).Infof("actions.split run err=%v \n", err)
		return nil, err
	}

	glog.V(3).Infof("actions.split run results=%v\n", g.Results)

	// What to collect
	c := ctx.Pop("collect")

	// If `collect` is a string -- add it to list
	var collect []string // context variables to collect
	if collectone, ok := c.String(); ok {
		collect = append(collect, collectone)
	} else if collect, ok = c.StringList(); !ok {
		return action.Map{"results": g.Results}, nil
	}

	// Collect results
	results := make(action.Map)
	for _, result := range g.Results {
		for _, key := range collect {
			var list []interface{}
			if results[key] != nil {
				list, _ = results[key].([]interface{})
			}
			results[key] = append(list, result.Value(key))
		}
	}

	glog.V(3).Infof("actions.split collect results=%v\n", results)

	return results, nil
}
Exemple #9
0
// Encode - json.encode - encodes all values in context from `encode` list to JSON.
// Example:
// 	{
//		"name": "json.encode",
//		"ctx": {
//			"encode": ["content"],
//			"content": [1, 2, 3]
//		}
//	}
func Encode(ctx action.Map) (action.Map, error) {
	// Pop encode value - context keys to encode
	ctx.Transform(ctx.Pop("encode"), encodeFunc)
	return ctx, nil
}
Exemple #10
0
// Decode - json.decode - decodes all JSON values in context from `decode` list.
// Example:
// 	{
//		"name": "json.decode",
//		"ctx": {
//			"decode": ["content"],
//			"content": "[1, 2, 3]"
//		}
//	}
func Decode(ctx action.Map) (action.Map, error) {
	// Pop decode value - context keys to decode
	ctx.Transform(ctx.Pop("decode"), decodeFunc)

	return ctx, nil
}