func mergeAllDocs(root map[interface{}]interface{}, paths []string) error {
m := &Merger{}
for _, path := range paths {
DEBUG("Processing file '%s'", path)
data, err := ioutil.ReadFile(path)
if err != nil {
return ansi.Errorf("@R{Error reading file} @m{%s}: %s\n", path, err.Error())
}
if handleConcourseQuoting {
data = quoteConcourse(data)
}
doc, err := parseYAML(data)
if err != nil {
return ansi.Errorf("@m{%s}: @R{%s}\n", path, err.Error())
}
m.Merge(root, doc)
tmpYaml, _ := yaml.Marshal(root) // we don't care about errors for debugging
TRACE("Current data after processing '%s':\n%s", path, tmpYaml)
}
return m.Error()
}
// Resolve ...
func (e *Expr) Resolve(tree map[interface{}]interface{}) (*Expr, error) {
switch e.Type {
case Literal:
return e, nil
case EnvVar:
v := os.Getenv(e.Name)
if v == "" {
return nil, ansi.Errorf("@R{Environment variable} @c{$%s} @R{is not set}", e.Name)
}
return &Expr{Type: Literal, Literal: v}, nil
case Reference:
if _, err := e.Reference.Resolve(tree); err != nil {
return nil, ansi.Errorf("@R{Unable to resolve `}@c{%s}@R{`: %s}", e.Reference, err)
}
return e, nil
case LogicalOr:
if o, err := e.Left.Resolve(tree); err == nil {
return o, nil
}
return e.Right.Resolve(tree)
}
return nil, ansi.Errorf("@R{unknown expression operand type (}@c{%d}@R{)}", e.Type)
}
// Reduce ...
func (e *Expr) Reduce() (*Expr, error) {
var reduce func(*Expr) (*Expr, *Expr, bool)
reduce = func(e *Expr) (*Expr, *Expr, bool) {
switch e.Type {
case Literal:
return e, e, false
case EnvVar:
return e, nil, false
case Reference:
return e, nil, false
case LogicalOr:
l, short, _ := reduce(e.Left)
if short != nil {
return l, short, true
}
r, short, more := reduce(e.Right)
return &Expr{
Type: LogicalOr,
Left: l,
Right: r,
}, short, more
}
return nil, nil, false
}
reduced, short, more := reduce(e)
if more && short != nil {
return reduced, ansi.Errorf("@R{literal} @c{%v} @R{short-circuits expression (}@c{%s}@R{)}", short, e)
}
return reduced, nil
}
func canKeyMergeArray(disp string, array []interface{}, node string, key string) error {
// ensure that all elements of `array` are maps,
// and that they contain the key `key`
for i, o := range array {
if reflect.TypeOf(o).Kind() != reflect.Map {
return ansi.Errorf("@m{%s.%d}: @R{%s object is a} @c{%s}@R{, not a} @c{map} @R{- cannot merge using keys}", node, i, disp, reflect.TypeOf(o).Kind().String())
}
obj := o.(map[interface{}]interface{})
if _, ok := obj[key]; !ok {
return ansi.Errorf("@m{%s.%d}: @R{%s object does not contain the key} @c{'%s'}@R{ - cannot merge}", node, i, disp, key)
}
}
return nil
}
func JSONifyIO(in io.Reader) (string, error) {
data, err := ioutil.ReadAll(in)
if err != nil {
return "", ansi.Errorf("@R{Error reading input}: %s", err)
}
return jsonifyData(data)
}
func JSONifyFiles(paths []string) ([]string, error) {
l := make([]string, len(paths))
for i, path := range paths {
DEBUG("Processing file '%s'", path)
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, ansi.Errorf("@R{Error reading file} @m{%s}: %s", path, err)
}
if l[i], err = jsonifyData(data); err != nil {
return nil, ansi.Errorf("%s: %s", path, err)
}
}
return l, nil
}
// Run ...
func (op *Opcall) Run(ev *Evaluator) (*Response, error) {
was := ev.Here
ev.Here = op.where
r, err := op.op.Run(ev, op.args)
ev.Here = was
if err != nil {
return nil, ansi.Errorf("@m{$.%s}: @R{%s}", op.where, err)
}
return r, nil
}
func parseYAML(data []byte) (map[interface{}]interface{}, error) {
y, err := simpleyaml.NewYaml(data)
if err != nil {
return nil, err
}
doc, err := y.Map()
if err != nil {
return nil, ansi.Errorf("@R{Root of YAML document is not a hash/map}: %s\n", err.Error())
}
return doc, nil
}
func (m *Merger) mergeMap(orig map[interface{}]interface{}, n map[interface{}]interface{}, node string) {
mergeRx := regexp.MustCompile(`^\s*\Q((\E\s*merge\s*.*\Q))\E`)
for k, val := range n {
path := fmt.Sprintf("%s.%v", node, k)
if s, ok := val.(string); ok && mergeRx.MatchString(s) {
m.Errors.Append(ansi.Errorf("@m{%s}: @R{inappropriate use of} @c{(( merge ))} @R{operator outside of a list} (this is @G{spruce}, after all)", path))
}
if _, exists := orig[k]; exists {
DEBUG("%s: found upstream, merging it", path)
orig[k] = m.mergeObj(orig[k], val, path)
} else {
DEBUG("%s: not found upstream, adding it", path)
orig[k] = m.mergeObj(nil, deepCopy(val), path)
}
}
}
func jsonifyData(data []byte) (string, error) {
y, err := simpleyaml.NewYaml(data)
if err != nil {
return "", err
}
doc, err := y.Map()
if err != nil {
return "", ansi.Errorf("@R{Root of YAML document is not a hash/map}: %s\n", err.Error())
}
b, err := json.Marshal(deinterface(doc))
if err != nil {
return "", err
}
return string(b), nil
}
// CheckForCycles ...
func (ev *Evaluator) CheckForCycles(maxDepth int) error {
DEBUG("checking for cycles in final YAML structure")
var check func(o interface{}, depth int) error
check = func(o interface{}, depth int) error {
if depth == 0 {
return ansi.Errorf("@*{Hit max recursion depth. You seem to have a self-referencing dataset}")
}
switch o.(type) {
case []interface{}:
for _, v := range o.([]interface{}) {
if err := check(v, depth-1); err != nil {
return err
}
}
case map[interface{}]interface{}:
for _, v := range o.(map[interface{}]interface{}) {
if err := check(v, depth-1); err != nil {
return err
}
}
}
return nil
}
err := check(ev.Tree, maxDepth)
if err != nil {
DEBUG("error: %s\n", err)
return err
}
DEBUG("no cycles detected.\n")
return nil
}
// Run ...
func (CartesianProductOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) {
DEBUG("running (( cartesian-product ... )) operation at $.%s", ev.Here)
defer DEBUG("done with (( cartesian-product ... )) operation at $%s\n", ev.Here)
var vals [][]string
for i, arg := range args {
v, err := arg.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed\n error: %s", i, err)
return nil, err
}
switch v.Type {
case Literal:
DEBUG(" arg[%d]: found string literal '%s'", i, v.Literal)
vals = append(vals, []string{v.Literal.(string)})
case Reference:
DEBUG(" arg[%d]: trying to resolve reference $.%s", i, v.Reference)
s, err := v.Reference.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed\n error: %s", i, err)
return nil, ansi.Errorf("Unable to resolve `@m{%s}`: %s", v.Reference, err)
}
switch s.(type) {
case []interface{}:
var strs []string
DEBUG(" [%d]: resolved to a list; verifying", i)
for j, sub := range s.([]interface{}) {
if _, ok := sub.([]interface{}); ok {
DEBUG(" list[%d]: list item is itself a list; error!", j)
return nil, fmt.Errorf("cartesian-product operator can only operate on lists of scalar values")
} else if _, ok := sub.(map[interface{}]interface{}); ok {
DEBUG(" list[%d]: list item is a map; error!", j)
return nil, fmt.Errorf("cartesian-product operator can only operate on lists of scalar values")
}
DEBUG(" list[%d]: list item is a scalar: %v", j, sub)
strs = append(strs, fmt.Sprintf("%v", sub))
}
vals = append(vals, strs)
case map[interface{}]interface{}:
DEBUG(" [%d]: resolved to a map; error!", i)
return nil, fmt.Errorf("cartesian-product operator only accepts arrays and string values")
default:
DEBUG(" [%d]: resolved to a scalar; appending", i)
vals = append(vals, []string{fmt.Sprintf("%v", s)})
}
default:
DEBUG(" arg[%d]: I don't know what to do with '%v'", i, arg)
return nil, fmt.Errorf("cartesian-product operator only accepts key reference arguments")
}
DEBUG("")
}
switch len(args) {
case 0:
DEBUG(" no arguments supplied to (( cartesian-product ... )) operation. oops.")
return nil, ansi.Errorf("no arguments specified to @c{(( cartesian-product ... ))}")
case 1:
DEBUG(" called with only one argument; returning value as-is")
return &Response{
Type: Replace,
Value: vals[0],
}, nil
default:
DEBUG(" called with more than one arguments; combining into a single list of strings")
lst := vals[0]
for _, l := range vals[1:] {
lst = cartesian(lst, l)
}
return &Response{
Type: Replace,
Value: lst,
}, nil
}
}
func (m *Merger) mergeArray(orig []interface{}, n []interface{}, node string) []interface{} {
if shouldInlineMergeArray(n) {
DEBUG("%s: performing explicit inline array merge", node)
return m.mergeArrayInline(orig, n[1:], node)
} else if shouldReplaceArray(n) {
DEBUG("%s: replacing with new data", node)
return n[1:]
} else if should, key := shouldKeyMergeArray(n); should {
DEBUG("%s: performing key-based array merge, using key '%s'", node, key)
if err := canKeyMergeArray("new", n[1:], node, key); err != nil {
m.Errors.Append(err)
return nil
}
if err := canKeyMergeArray("original", orig, node, key); err != nil {
m.Errors.Append(err)
return nil
}
return m.mergeArrayByKey(orig, n[1:], node, key)
}
modificationDefinitions := getArrayModifications(n)
DEBUG("%s: performing %d modification operations against list", node, len(modificationDefinitions))
DEBUG("%+v", modificationDefinitions)
// Create a copy of orig for the (multiple) modifications that are about to happen
result := make([]interface{}, len(orig))
copy(result, orig)
var idx int
// Process the modifications definitions that were found in the new list
for i := range modificationDefinitions {
//Special tag for default behavior. Cannot be invoked explicitly by users
if modificationDefinitions[i].defaultMerge {
result = m.mergeArrayDefault(orig, modificationDefinitions[0].list, node)
continue
}
if modificationDefinitions[i].key == "" && modificationDefinitions[i].name == "" { // Index comes directly from operation definition
idx = modificationDefinitions[i].index
// Replace the -1 marker with the actual 'end' index of the array
if idx == -1 {
idx = len(result)
}
} else { // Index look-up based on key and name
key := modificationDefinitions[i].key
name := modificationDefinitions[i].name
delete := modificationDefinitions[i].delete
// Sanity check original list, list must contain key/id based entries
if err := canKeyMergeArray("original", result, node, key); err != nil {
m.Errors.Append(err)
return nil
}
// Sanity check new list, depending on the operation type (delete or insert)
if delete == false {
// Sanity check new list, list must contain key/id based entries
if err := canKeyMergeArray("new", modificationDefinitions[i].list, node, key); err != nil {
m.Errors.Append(err)
return nil
}
// Since we have a way to identify indiviual entries based on their key/id, we can sanity check for possible duplicates
for _, entry := range modificationDefinitions[i].list {
obj := entry.(map[interface{}]interface{})
entryName := obj[key].(string)
if getIndexOfEntry(result, key, entryName) > 0 {
m.Errors.Append(ansi.Errorf("@m{%s}: @R{unable to insert, because new list entry} @c{'%s: %s'} @R{is detected multiple times}", node, key, entryName))
return nil
}
}
} else {
// Sanity check for delete operation, ensure no orphan entries follow the operator definition
if len(modificationDefinitions[i].list) > 0 {
m.Errors.Append(ansi.Errorf("@m{%s}: @R{item in array directly after} @c{(( delete %s: \"%s\" ))} @r{must be one of the array operators 'append', 'prepend', 'delete', or 'insert'}", node, key, name))
return nil
}
}
// Look up the index of the specified insertion point (based on its key/name)
idx = getIndexOfEntry(result, key, name)
if idx < 0 {
m.Errors.Append(ansi.Errorf("@m{%s}: @R{unable to find specified modification point with} @c{'%s: %s'}", node, key, name))
return nil
}
}
// If after is specified, add one to the index to actually put the entry where it is expected
if modificationDefinitions[i].relative == "after" {
idx++
}
// Back out if idx is smaller than 0, or greater than the length (for inserts), or greater/equal than the length (for deletes)
if (idx < 0) || (modificationDefinitions[i].delete == false && idx > len(result)) || (modificationDefinitions[i].delete == true && idx >= len(result)) {
m.Errors.Append(ansi.Errorf("@m{%s}: @R{unable to modify the list, because specified index} @c{%d} @R{is out of bounds}", node, idx))
return nil
}
if modificationDefinitions[i].delete == false {
DEBUG("%s: inserting %d new elements to existing array at index %d", node, len(modificationDefinitions[i].list), idx)
result = insertIntoList(result, idx, modificationDefinitions[i].list)
} else {
DEBUG("%s: deleting element at array index %d", node, idx)
result = deleteIndexFromList(result, idx)
}
}
return result
}
// Run ...
func (JoinOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) {
DEBUG("running (( join ... )) operation at $.%s", ev.Here)
defer DEBUG("done with (( join ... )) operation at $%s\n", ev.Here)
if len(args) == 0 {
DEBUG(" no arguments supplied to (( join ... )) operation.")
return nil, ansi.Errorf("no arguments specified to @c{(( join ... ))}")
}
if len(args) == 1 {
DEBUG(" too few arguments supplied to (( join ... )) operation.")
return nil, ansi.Errorf("too few arguments supplied to @c{(( join ... ))}")
}
var seperator string
var list []string
for i, arg := range args {
if i == 0 { // argument #0: seperator
sep, err := arg.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed\n error: %s", i, err)
return nil, err
}
if sep.Type != Literal {
DEBUG(" [%d]: unsupported type for join operator seperator argument: '%v'", i, sep)
return nil, fmt.Errorf("join operator only accepts literal argument for the seperator")
}
DEBUG(" [%d]: list seperator will be: %s", i, sep)
seperator = sep.Literal.(string)
} else { // argument #1..n: list, or literal
ref, err := arg.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed\n error: %s", i, err)
return nil, err
}
switch ref.Type {
case Literal:
DEBUG(" [%d]: adding literal %s to the list", i, ref)
list = append(list, fmt.Sprintf("%v", ref.Literal))
case Reference:
DEBUG(" [%d]: trying to resolve reference $.%s", i, ref.Reference)
s, err := ref.Reference.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed with error: %s", i, err)
return nil, fmt.Errorf("Unable to resolve `%s`: %s", ref.Reference, err)
}
switch s.(type) {
case []interface{}:
DEBUG(" [%d]: $.%s is a list", i, ref.Reference)
for idx, entry := range s.([]interface{}) {
switch entry.(type) {
case []interface{}:
DEBUG(" [%d]: entry #%d in list is a list (not a literal)", i, idx)
return nil, ansi.Errorf("entry #%d in list is not compatible for @c{(( join ... ))}", idx)
case map[interface{}]interface{}:
DEBUG(" [%d]: entry #%d in list is a map (not a literal)", i, idx)
return nil, ansi.Errorf("entry #%d in list is not compatible for @c{(( join ... ))}", idx)
default:
list = append(list, fmt.Sprintf("%v", entry))
}
}
case map[interface{}]interface{}:
DEBUG(" [%d]: $.%s is a map (not a list or a literal)", i, ref.Reference)
return nil, ansi.Errorf("referenced entry is not a list or string for @c{(( join ... ))}")
default:
DEBUG(" [%d]: $.%s is a literal", i, ref.Reference)
list = append(list, fmt.Sprintf("%v", s))
}
default:
DEBUG(" [%d]: unsupported type for join operator: '%v'", i, ref)
return nil, fmt.Errorf("join operator only lists with string entries, and literals as data arguments")
}
}
}
// finally, join and return
DEBUG(" joined list: %s", strings.Join(list, seperator))
return &Response{
Type: Replace,
Value: strings.Join(list, seperator),
}, nil
}
// DataFlow ...
func (ev *Evaluator) DataFlow(phase OperatorPhase) ([]*Opcall, error) {
ev.Here = &tree.Cursor{}
all := map[string]*Opcall{}
locs := []*tree.Cursor{}
errors := MultiError{Errors: []error{}}
// forward decls of co-recursive function
var check func(interface{})
var scan func(interface{})
check = func(v interface{}) {
if s, ok := v.(string); ok {
op, err := ParseOpcall(phase, s)
if err != nil {
errors.Append(err)
} else if op != nil {
op.where = ev.Here.Copy()
if canon, err := op.where.Canonical(ev.Tree); err == nil {
op.canonical = canon
} else {
op.canonical = op.where
}
all[op.canonical.String()] = op
TRACE("found an operation at %s: %s", op.where.String(), op.src)
TRACE(" (canonical at %s)", op.canonical.String())
locs = append(locs, op.canonical)
}
} else {
scan(v)
}
}
scan = func(o interface{}) {
switch o.(type) {
case map[interface{}]interface{}:
for k, v := range o.(map[interface{}]interface{}) {
ev.Here.Push(fmt.Sprintf("%v", k))
check(v)
ev.Here.Pop()
}
case []interface{}:
for i, v := range o.([]interface{}) {
name := nameOfObj(v, fmt.Sprintf("%d", i))
op, _ := ParseOpcall(phase, name)
if op == nil {
ev.Here.Push(name)
} else {
ev.Here.Push(fmt.Sprintf("%d", i))
}
check(v)
ev.Here.Pop()
}
}
}
scan(ev.Tree)
// construct the data flow graph, where a -> b = b calls/requires a
// represent the graph as list of adjancies, that is [a,b] = a -> b
var g [][2]*Opcall
for _, a := range all {
for _, path := range a.Dependencies(ev, locs) {
if b, found := all[path.String()]; found {
g = append(g, [2]*Opcall{b, a})
}
}
}
// construct a sorted list of keys in $all, so that we
// can reliably generate the same DFA every time
var sortedKeys []string
for k := range all {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
// find all nodes in g that are free (no futher dependencies)
freeNodes := func(g [][2]*Opcall) []*Opcall {
l := []*Opcall{}
for _, k := range sortedKeys {
node, ok := all[k]
if !ok {
continue
}
called := false
for _, pair := range g {
if pair[1] == node {
called = true
break
}
}
if !called {
delete(all, k)
l = append(l, node)
}
}
return l
}
// removes (nullifies) all dependencies on n in g
remove := func(old [][2]*Opcall, n *Opcall) [][2]*Opcall {
l := [][2]*Opcall{}
for _, pair := range old {
if pair[0] != n {
l = append(l, pair)
}
}
return l
}
// Kahn topological sort
ops := []*Opcall{} // order in which to call the ops
wave := 0
for len(all) > 0 {
wave++
free := freeNodes(g)
if len(free) == 0 {
return nil, ansi.Errorf("@*{cycle detected in operator data-flow graph}")
}
for _, node := range free {
TRACE("data flow: [%d] wave %d, op %s: %s", len(ops), wave, node.where, node.src)
ops = append(ops, node)
g = remove(g, node)
}
}
if len(errors.Errors) > 0 {
return nil, errors
}
return ops, nil
}
// Run ...
func (InjectOperator) Run(ev *Evaluator, args []*Expr) (*Response, error) {
DEBUG("running (( inject ... )) operation at $.%s", ev.Here)
defer DEBUG("done with (( inject ... )) operation at $%s\n", ev.Here)
var vals []map[interface{}]interface{}
for i, arg := range args {
v, err := arg.Resolve(ev.Tree)
if err != nil {
DEBUG(" arg[%d]: failed to resolve expression to a concrete value", i)
DEBUG(" [%d]: error was: %s", i, err)
return nil, err
}
switch v.Type {
case Literal:
DEBUG(" arg[%d]: found string literal '%s'", i, v.Literal)
DEBUG(" (inject operator only handles references to other parts of the YAML tree)")
return nil, fmt.Errorf("inject operator only accepts key reference arguments")
case Reference:
DEBUG(" arg[%d]: trying to resolve reference $.%s", i, v.Reference)
s, err := v.Reference.Resolve(ev.Tree)
if err != nil {
DEBUG(" [%d]: resolution failed\n error: %s", i, err)
return nil, err
}
m, ok := s.(map[interface{}]interface{})
if !ok {
DEBUG(" [%d]: resolved to something that is not a map. that is unacceptable.", i)
return nil, ansi.Errorf("@c{%s} @R{is not a map}", v.Reference)
}
DEBUG(" [%d]: resolved to a map; appending to the list of maps to merge/inject", i)
vals = append(vals, m)
default:
DEBUG(" arg[%d]: I don't know what to do with '%v'", i, arg)
return nil, fmt.Errorf("inject operator only accepts key reference arguments")
}
DEBUG("")
}
switch len(vals) {
case 0:
DEBUG(" no arguments supplied to (( inject ... )) operation. oops.")
return nil, ansi.Errorf("no arguments specified to @c{(( inject ... ))}")
default:
DEBUG(" merging found maps into a single map to be injected")
merged, err := Merge(vals...)
if err != nil {
DEBUG(" failed: %s\n", err)
return nil, err
}
return &Response{
Type: Inject,
Value: merged,
}, nil
}
}
// Cherry-pick ...
func (ev *Evaluator) CherryPick(paths []string) error {
DEBUG("cherry-picking %d paths from the final YAML structure", len(paths))
if len(paths) > 0 {
// This will serve as the replacement tree ...
replacement := make(map[interface{}]interface{})
for _, path := range paths {
cursor, err := tree.ParseCursor(path)
if err != nil {
return err
}
// These variables will potentially be modified (depending on the structure)
var cherryName string
var cherryValue interface{}
// Resolve the value that needs to be cherry picked
cherryValue, err = cursor.Resolve(ev.Tree)
if err != nil {
return err
}
// Name of the parameter of the to-be-picked value
cherryName = cursor.Nodes[len(cursor.Nodes)-1]
// Since the cherry can be deep down the structure, we need to go down
// (or up, depending how you read it) the structure to include the parent
// names of the respective cherry. The pointer will be reassigned with
// each level.
pointer := cursor
for pointer != nil {
parent := pointer.Copy()
parent.Pop()
if parent.String() == "" {
// Empty parent string means we reached the root, setting the pointer nil to stop processing ...
pointer = nil
// ... create the final cherry wrapped in its container ...
tmp := make(map[interface{}]interface{})
tmp[cherryName] = cherryValue
// ... and add it to the replacement map
DEBUG("Merging '%s' into the replacement tree", path)
merger := &Merger{AppendByDefault: true}
merged := merger.mergeObj(tmp, replacement, path)
if err := merger.Error(); err != nil {
return err
}
replacement = merged.(map[interface{}]interface{})
} else {
// Reassign the pointer to the parent and restructre the current cherry value to address the parent structure and name
pointer = parent
// Depending on the type of the parent, either a map or a list is created for the new parent of the cherry value
if obj, err := parent.Resolve(ev.Tree); err == nil {
switch obj.(type) {
case map[interface{}]interface{}:
tmp := make(map[interface{}]interface{})
tmp[cherryName] = cherryValue
cherryName = parent.Nodes[len(parent.Nodes)-1]
cherryValue = tmp
case []interface{}:
tmp := make([]interface{}, 0, 0)
tmp = append(tmp, cherryValue)
cherryName = parent.Nodes[len(parent.Nodes)-1]
cherryValue = tmp
default:
return ansi.Errorf("@*{Unsupported type detected, %s is neither a map nor a list}", parent.String())
}
} else {
return err
}
}
}
}
// replace the existing tree with a new one that contain the cherry-picks
ev.Tree = replacement
}
DEBUG("")
return nil
}