func (c Config) Copy(v interface{}) (interface{}, error) { if c.Lock && reflect.ValueOf(v).Kind() != reflect.Ptr { return nil, errPointerRequired } w := new(walker) if c.Lock { w.useLocks = true } if c.Copiers == nil { c.Copiers = Copiers } err := reflectwalk.Walk(v, w) if err != nil { return nil, err } // Get the result. If the result is nil, then we want to turn it // into a typed nil if we can. result := w.Result if result == nil { val := reflect.ValueOf(v) result = reflect.Indirect(reflect.New(val.Type())).Interface() } return result, nil }
func (r *RawConfig) init() error { r.config = r.Raw r.Interpolations = nil r.Variables = nil fn := func(i Interpolation) (string, error) { r.Interpolations = append(r.Interpolations, i) for k, v := range i.Variables() { if r.Variables == nil { r.Variables = make(map[string]InterpolatedVariable) } r.Variables[k] = v } return "", nil } walker := &interpolationWalker{F: fn} if err := reflectwalk.Walk(r.Raw, walker); err != nil { return err } return nil }
func (r *RawConfig) init() error { r.lock.Lock() defer r.lock.Unlock() r.config = r.Raw r.Interpolations = nil r.Variables = nil fn := func(node ast.Node) (interface{}, error) { r.Interpolations = append(r.Interpolations, node) vars, err := DetectVariables(node) if err != nil { return "", err } for _, v := range vars { if r.Variables == nil { r.Variables = make(map[string]InterpolatedVariable) } r.Variables[v.FullKey()] = v } return "", nil } walker := &interpolationWalker{F: fn} if err := reflectwalk.Walk(r.Raw, walker); err != nil { return err } return nil }
// applyOverride applies the given override to the specified object. func applyOverride(object interface{}, o Override) (Element, error) { if err := o.Validate(); err != nil { return Element{}, errors.Wrap(err, "invalid override") } walker := newOverrideWalker(o) // walk the copy and apply the updates if err := reflectwalk.Walk(object, walker); err != nil { return Element{}, err } unused := walker.unused() if len(unused) > 0 { return Element{}, fmt.Errorf("unknown options %v in section %s", unused, o.Section) } // Return the modified copy element := walker.elementObject() if element.value == nil && !o.Delete { return Element{}, fmt.Errorf("unknown section %s", o.Section) } // Validate new value if v, ok := element.value.(Validator); ok { if err := v.Validate(); err != nil { return Element{}, errors.Wrap(err, "failed validation") } } return element, nil }
// Redacted returns the options for the element in a map. // Any fields with the `override:",redact"` tag set will be replaced // with a boolean value indicating whether a non-zero value was set. func (e Element) Redacted() (map[string]interface{}, []string, error) { walker := newRedactWalker() // walk the section and collect redacted options if err := reflectwalk.Walk(e.value, walker); err != nil { return nil, nil, errors.Wrap(err, "failed to redact section") } return walker.optionsMap(), walker.redactedList(), nil }
// ElementKeys returns a map of section name to element key for each section. func ElementKeys(config interface{}) (map[string]string, error) { // walk the config and read all sections walker := newSectionWalker() if err := reflectwalk.Walk(config, walker); err != nil { return nil, errors.Wrap(err, "failed to read sections from configuration object") } return walker.elementKeysMap(), nil }
// Copy returns a deep copy of v. func Copy(v interface{}) (interface{}, error) { w := new(walker) err := reflectwalk.Walk(v, w) if err != nil { return nil, err } return w.Result, nil }
func TestInterpolationWalker_replace(t *testing.T) { cases := []struct { Input interface{} Output interface{} }{ { Input: map[string]interface{}{ "foo": "$${var.foo}", }, Output: map[string]interface{}{ "foo": "$${var.foo}", }, }, { Input: map[string]interface{}{ "foo": "hello, ${var.foo}", }, Output: map[string]interface{}{ "foo": "hello, bar", }, }, { Input: map[string]interface{}{ "foo": map[string]interface{}{ "${var.foo}": "bar", }, }, Output: map[string]interface{}{ "foo": map[string]interface{}{ "bar": "bar", }, }, }, } for i, tc := range cases { fn := func(i Interpolation) (string, error) { return "bar", nil } w := &interpolationWalker{F: fn, Replace: true} if err := reflectwalk.Walk(tc.Input, w); err != nil { t.Fatalf("err: %s", err) } if !reflect.DeepEqual(tc.Input, tc.Output) { t.Fatalf("%d: bad:\n\n%#v", i, tc.Input) } } }
// HashStructure takes an interface and hashes all the values within // the structure. Only _values_ are hashed: keys of objects are not. // // For the HashCallback, see the built-in HashCallbacks below. func HashStructure(s interface{}, cb HashCallback) (interface{}, error) { s, err := copystructure.Copy(s) if err != nil { return nil, err } walker := &hashWalker{Callback: cb} if err := reflectwalk.Walk(s, walker); err != nil { return nil, err } return s, nil }
// Close will close all shadow values within the given structure. // // This uses reflection to walk the structure, find all shadow elements, // and close them. Currently this will only find struct fields that are // shadow values, and not slice elements, etc. func Close(v interface{}) error { // We require a pointer so we can address the internal fields val := reflect.ValueOf(v) if val.Kind() != reflect.Ptr { return fmt.Errorf("value must be a pointer") } // Walk and close var w closeWalker if err := reflectwalk.Walk(v, &w); err != nil { return err } return w.Err }
// ValidateInterface renders any value and returns the resulting value. func ValidateInterface(v interface{}, ctx *Context) error { f := func(v string) (string, error) { return v, Validate(v, ctx) } walker := &renderWalker{ F: f, Replace: false, } err := reflectwalk.Walk(v, walker) if err != nil { return err } return nil }
func (r *RawConfig) interpolate(fn interpolationWalkerFunc) error { config, err := copystructure.Copy(r.Raw) if err != nil { return err } r.config = config.(map[string]interface{}) w := &interpolationWalker{F: fn, Replace: true} err = reflectwalk.Walk(r.config, w) if err != nil { return err } r.unknownKeys = w.unknownKeys return nil }
func TestUnknownCheckWalker(t *testing.T) { cases := []struct { Name string Input interface{} Result bool }{ { "primitive", 42, false, }, { "primitive computed", unknownValue(), true, }, { "list", []interface{}{"foo", unknownValue()}, true, }, { "nested list", []interface{}{ "foo", []interface{}{unknownValue()}, }, true, }, } for i, tc := range cases { t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) { var w unknownCheckWalker if err := reflectwalk.Walk(tc.Input, &w); err != nil { t.Fatalf("err: %s", err) } if w.Unknown != tc.Result { t.Fatalf("bad: %v", w.Unknown) } }) } }
// Copy returns a deep copy of v. func Copy(v interface{}) (interface{}, error) { w := new(walker) err := reflectwalk.Walk(v, w) if err != nil { return nil, err } // Get the result. If the result is nil, then we want to turn it // into a typed nil if we can. result := w.Result if result == nil { val := reflect.ValueOf(v) result = reflect.Indirect(reflect.New(val.Type())).Interface() } return result, nil }
// RenderInterface renders any value and returns the resulting value. func RenderInterface(v interface{}, ctx *Context) (interface{}, error) { f := func(v string) (string, error) { return Render(v, ctx) } walker := &renderWalker{ F: f, Replace: true, } err := reflectwalk.Walk(v, walker) if err != nil { return nil, err } if walker.Top != nil { v = walker.Top } return v, nil }
// Interpolate uses the given mapping of variable values and uses // those as the values to replace any variables in this raw // configuration. // // Any prior calls to Interpolate are replaced with this one. // // If a variable key is missing, this will panic. func (r *RawConfig) Interpolate(vs map[string]string) error { config, err := copystructure.Copy(r.Raw) if err != nil { return err } r.config = config.(map[string]interface{}) fn := func(i Interpolation) (string, error) { return i.Interpolate(vs) } w := &interpolationWalker{F: fn, Replace: true} err = reflectwalk.Walk(r.config, w) if err != nil { return err } r.unknownKeys = w.unknownKeys return nil }
// IsComputed returns whether the given key is computed or not. func (c *ResourceConfig) IsComputed(k string) bool { // The next thing we do is check the config if we get a computed // value out of it. v, ok := c.get(k, c.Config) if !ok { return false } // If value is nil, then it isn't computed if v == nil { return false } // Test if the value contains an unknown value var w unknownCheckWalker if err := reflectwalk.Walk(v, &w); err != nil { panic(err) } return w.Unknown }
// OverrideConfig applies all given overrides and returns a map of all configuration sections, even if they were not overridden. // The overrides are all applied to the same object and the original configuration object remains unmodified. // // Values must be of the same type as the named option field, or have another means of converting the value. // // Numeric types will be converted to the absolute type using Go's default conversion mechanisms. // Strings and Stringer types will be parsed for numeric values if possible. // TextUnmarshaler types will attempt to unmarshal string values. // // Mismatched types or failure to convert the value will result in an error. // // An element value that is a Validator will be validated and any encounted error returned. // // When a new element is being created if the element type is a Initer, then the zero value of the // element will first have defaults set before the overrides are applied. // // The underlying configuration object is not modified, but rather a copy is returned via the Element type. func OverrideConfig(config interface{}, os []Override) (map[string]Section, error) { // First make a copy into which we can apply the updates. copy, err := copystructure.Copy(config) if err != nil { return nil, errors.Wrap(err, "failed to copy configuration object") } // Apply all overrides to the same copy for _, o := range os { // We do not need to keep a reference to the section since we are going to walk the entire copy next _, err := applyOverride(copy, o) if err != nil { return nil, errors.Wrapf(err, "failed to override configuration %s/%s", o.Section, o.Element) } } // Walk the copy to return all sections walker := newSectionWalker() if err := reflectwalk.Walk(copy, walker); err != nil { return nil, errors.Wrap(err, "failed to read sections from configuration object") } return walker.sectionsMap(), nil }
// Walk will walk an arbitrary Go structure and parse any string as an // HIL program and call the callback cb to determine what to replace it // with. // // This function is very useful for arbitrary HIL program interpolation // across a complex configuration structure. Due to the heavy use of // reflection in this function, it is recommend to write many unit tests // with your typical configuration structures to hilp mitigate the risk // of panics. func Walk(v interface{}, cb WalkFn) error { walker := &interpolationWalker{F: cb} return reflectwalk.Walk(v, walker) }
// Validate does some basic semantic checking of the configuration. func (c *Config) Validate() error { var errs []error for _, k := range c.unknownKeys { errs = append(errs, fmt.Errorf( "Unknown root level key: %s", k)) } vars := c.allVariables() varMap := make(map[string]*Variable) for _, v := range c.Variables { varMap[v.Name] = v } for _, v := range c.Variables { if v.Type() == VariableTypeUnknown { errs = append(errs, fmt.Errorf( "Variable '%s': must be string or mapping", v.Name)) continue } interp := false fn := func(i Interpolation) (string, error) { interp = true return "", nil } w := &interpolationWalker{F: fn} if err := reflectwalk.Walk(v.Default, w); err == nil { if interp { errs = append(errs, fmt.Errorf( "Variable '%s': cannot contain interpolations", v.Name)) } } } // Check for references to user variables that do not actually // exist and record those errors. for source, vs := range vars { for _, v := range vs { uv, ok := v.(*UserVariable) if !ok { continue } if _, ok := varMap[uv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown variable referenced: %s", source, uv.Name)) } } } // Check that all references to resources are valid resources := make(map[string]*Resource) dupped := make(map[string]struct{}) for _, r := range c.Resources { if _, ok := resources[r.Id()]; ok { if _, ok := dupped[r.Id()]; !ok { dupped[r.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: resource repeated multiple times", r.Id())) } } resources[r.Id()] = r } dupped = nil // Validate resources for n, r := range resources { if r.Count < 1 { errs = append(errs, fmt.Errorf( "%s: count must be greater than or equal to 1", n)) } for _, d := range r.DependsOn { if _, ok := resources[d]; !ok { errs = append(errs, fmt.Errorf( "%s: resource depends on non-existent resource '%s'", n, d)) } } } for source, vs := range vars { for _, v := range vs { rv, ok := v.(*ResourceVariable) if !ok { continue } id := fmt.Sprintf("%s.%s", rv.Type, rv.Name) r, ok := resources[id] if !ok { errs = append(errs, fmt.Errorf( "%s: unknown resource '%s' referenced in variable %s", source, id, rv.FullKey())) continue } // If it is a multi reference and resource has a single // count, it is an error. if r.Count > 1 && !rv.Multi { errs = append(errs, fmt.Errorf( "%s: variable '%s' must specify index for multi-count "+ "resource %s", source, rv.FullKey(), id)) continue } } } // Check that all outputs are valid for _, o := range c.Outputs { invalid := false for k, _ := range o.RawConfig.Raw { if k != "value" { invalid = true break } } if invalid { errs = append(errs, fmt.Errorf( "%s: output should only have 'value' field", o.Name)) } } if len(errs) > 0 { return &multierror.Error{Errors: errs} } return nil }
func TestInterpolationWalker_detect(t *testing.T) { cases := []struct { Input interface{} Result []Interpolation }{ { Input: map[string]interface{}{ "foo": "$${var.foo}", }, Result: nil, }, { Input: map[string]interface{}{ "foo": "${var.foo}", }, Result: []Interpolation{ &VariableInterpolation{ Variable: &UserVariable{ Name: "foo", key: "var.foo", }, }, }, }, { Input: map[string]interface{}{ "foo": "${lookup(var.foo)}", }, Result: []Interpolation{ &FunctionInterpolation{ Func: nil, Args: []Interpolation{ &VariableInterpolation{ Variable: &UserVariable{ Name: "foo", key: "var.foo", }, }, }, }, }, }, } for i, tc := range cases { var actual []Interpolation detectFn := func(i Interpolation) (string, error) { actual = append(actual, i) return "", nil } w := &interpolationWalker{F: detectFn} if err := reflectwalk.Walk(tc.Input, w); err != nil { t.Fatalf("err: %s", err) } for _, a := range actual { // This is jank, but reflect.DeepEqual never has functions // being the same. if f, ok := a.(*FunctionInterpolation); ok { f.Func = nil } } if !reflect.DeepEqual(actual, tc.Result) { t.Fatalf("%d: bad:\n\n%#v", i, actual) } } }
// Validate does some basic semantic checking of the configuration. func (c *Config) Validate() error { if c == nil { return nil } var errs []error for _, k := range c.unknownKeys { errs = append(errs, fmt.Errorf( "Unknown root level key: %s", k)) } vars := c.InterpolatedVariables() varMap := make(map[string]*Variable) for _, v := range c.Variables { varMap[v.Name] = v } for _, v := range c.Variables { if v.Type() == VariableTypeUnknown { errs = append(errs, fmt.Errorf( "Variable '%s': must be string or mapping", v.Name)) continue } interp := false fn := func(ast.Node) (string, error) { interp = true return "", nil } w := &interpolationWalker{F: fn} if v.Default != nil { if err := reflectwalk.Walk(v.Default, w); err == nil { if interp { errs = append(errs, fmt.Errorf( "Variable '%s': cannot contain interpolations", v.Name)) } } } } // Check for references to user variables that do not actually // exist and record those errors. for source, vs := range vars { for _, v := range vs { uv, ok := v.(*UserVariable) if !ok { continue } if _, ok := varMap[uv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown variable referenced: '%s'. define it with 'variable' blocks", source, uv.Name)) } } } // Check that all count variables are valid. for source, vs := range vars { for _, rawV := range vs { switch v := rawV.(type) { case *CountVariable: if v.Type == CountValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid count variable: %s", source, v.FullKey())) } case *PathVariable: if v.Type == PathValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid path variable: %s", source, v.FullKey())) } } } } // Check that providers aren't declared multiple times. providerSet := make(map[string]struct{}) for _, p := range c.ProviderConfigs { name := p.FullName() if _, ok := providerSet[name]; ok { errs = append(errs, fmt.Errorf( "provider.%s: declared multiple times, you can only declare a provider once", name)) continue } providerSet[name] = struct{}{} } // Check that all references to modules are valid modules := make(map[string]*Module) dupped := make(map[string]struct{}) for _, m := range c.Modules { // Check for duplicates if _, ok := modules[m.Id()]; ok { if _, ok := dupped[m.Id()]; !ok { dupped[m.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: module repeated multiple times", m.Id())) } // Already seen this module, just skip it continue } modules[m.Id()] = m // Check that the source has no interpolations rc, err := NewRawConfig(map[string]interface{}{ "root": m.Source, }) if err != nil { errs = append(errs, fmt.Errorf( "%s: module source error: %s", m.Id(), err)) } else if len(rc.Interpolations) > 0 { errs = append(errs, fmt.Errorf( "%s: module source cannot contain interpolations", m.Id())) } // Check that the name matches our regexp if !NameRegexp.Match([]byte(m.Name)) { errs = append(errs, fmt.Errorf( "%s: module name can only contain letters, numbers, "+ "dashes, and underscores", m.Id())) } // Check that the configuration can all be strings raw := make(map[string]interface{}) for k, v := range m.RawConfig.Raw { var strVal string if err := mapstructure.WeakDecode(v, &strVal); err != nil { errs = append(errs, fmt.Errorf( "%s: variable %s must be a string value", m.Id(), k)) } raw[k] = strVal } // Check for invalid count variables for _, v := range m.RawConfig.Variables { switch v.(type) { case *CountVariable: errs = append(errs, fmt.Errorf( "%s: count variables are only valid within resources", m.Name)) case *SelfVariable: errs = append(errs, fmt.Errorf( "%s: self variables are only valid within resources", m.Name)) } } // Update the raw configuration to only contain the string values m.RawConfig, err = NewRawConfig(raw) if err != nil { errs = append(errs, fmt.Errorf( "%s: can't initialize configuration: %s", m.Id(), err)) } } dupped = nil // Check that all variables for modules reference modules that // exist. for source, vs := range vars { for _, v := range vs { mv, ok := v.(*ModuleVariable) if !ok { continue } if _, ok := modules[mv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown module referenced: %s", source, mv.Name)) } } } // Check that all references to resources are valid resources := make(map[string]*Resource) dupped = make(map[string]struct{}) for _, r := range c.Resources { if _, ok := resources[r.Id()]; ok { if _, ok := dupped[r.Id()]; !ok { dupped[r.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: resource repeated multiple times", r.Id())) } } resources[r.Id()] = r } dupped = nil // Validate resources for n, r := range resources { // Verify count variables for _, v := range r.RawCount.Variables { switch v.(type) { case *CountVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference count variable: %s", n, v.FullKey())) case *ModuleVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference module variable: %s", n, v.FullKey())) case *ResourceVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference resource variable: %s", n, v.FullKey())) case *UserVariable: // Good default: panic("Unknown type in count var: " + n) } } // Interpolate with a fixed number to verify that its a number. r.RawCount.interpolate(func(root ast.Node) (string, error) { // Execute the node but transform the AST so that it returns // a fixed value of "5" for all interpolations. out, _, err := lang.Eval( lang.FixedValueTransform( root, &ast.LiteralNode{Value: "5", Typex: ast.TypeString}), nil) if err != nil { return "", err } return out.(string), nil }) _, err := strconv.ParseInt(r.RawCount.Value().(string), 0, 0) if err != nil { errs = append(errs, fmt.Errorf( "%s: resource count must be an integer", n)) } r.RawCount.init() // Verify depends on points to resources that all exist for _, d := range r.DependsOn { // Check if we contain interpolations rc, err := NewRawConfig(map[string]interface{}{ "value": d, }) if err == nil && len(rc.Variables) > 0 { errs = append(errs, fmt.Errorf( "%s: depends on value cannot contain interpolations: %s", n, d)) continue } if _, ok := resources[d]; !ok { errs = append(errs, fmt.Errorf( "%s: resource depends on non-existent resource '%s'", n, d)) } } // Verify provider points to a provider that is configured if r.Provider != "" { if _, ok := providerSet[r.Provider]; !ok { errs = append(errs, fmt.Errorf( "%s: resource depends on non-configured provider '%s'", n, r.Provider)) } } // Verify provisioners don't contain any splats for _, p := range r.Provisioners { // This validation checks that there are now splat variables // referencing ourself. This currently is not allowed. for _, v := range p.ConnInfo.Variables { rv, ok := v.(*ResourceVariable) if !ok { continue } if rv.Multi && rv.Index == -1 && rv.Type == r.Type && rv.Name == r.Name { errs = append(errs, fmt.Errorf( "%s: connection info cannot contain splat variable "+ "referencing itself", n)) break } } for _, v := range p.RawConfig.Variables { rv, ok := v.(*ResourceVariable) if !ok { continue } if rv.Multi && rv.Index == -1 && rv.Type == r.Type && rv.Name == r.Name { errs = append(errs, fmt.Errorf( "%s: connection info cannot contain splat variable "+ "referencing itself", n)) break } } } } for source, vs := range vars { for _, v := range vs { rv, ok := v.(*ResourceVariable) if !ok { continue } id := fmt.Sprintf("%s.%s", rv.Type, rv.Name) if _, ok := resources[id]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown resource '%s' referenced in variable %s", source, id, rv.FullKey())) continue } } } // Check that all outputs are valid for _, o := range c.Outputs { invalid := false for k, _ := range o.RawConfig.Raw { if k != "value" { invalid = true break } } if invalid { errs = append(errs, fmt.Errorf( "%s: output should only have 'value' field", o.Name)) } for _, v := range o.RawConfig.Variables { if _, ok := v.(*CountVariable); ok { errs = append(errs, fmt.Errorf( "%s: count variables are only valid within resources", o.Name)) } } } // Check that all variables are in the proper context for source, rc := range c.rawConfigs() { walker := &interpolationWalker{ ContextF: c.validateVarContextFn(source, &errs), } if err := reflectwalk.Walk(rc.Raw, walker); err != nil { errs = append(errs, fmt.Errorf( "%s: error reading config: %s", source, err)) } } // Validate the self variable for source, rc := range c.rawConfigs() { // Ignore provisioners. This is a pretty brittle way to do this, // but better than also repeating all the resources. if strings.Contains(source, "provision") { continue } for _, v := range rc.Variables { if _, ok := v.(*SelfVariable); ok { errs = append(errs, fmt.Errorf( "%s: cannot contain self-reference %s", source, v.FullKey())) } } } if len(errs) > 0 { return &multierror.Error{Errors: errs} } return nil }
// Validate does some basic semantic checking of the configuration. func (c *Config) Validate() error { if c == nil { return nil } var errs []error for _, k := range c.unknownKeys { errs = append(errs, fmt.Errorf( "Unknown root level key: %s", k)) } vars := c.InterpolatedVariables() varMap := make(map[string]*Variable) for _, v := range c.Variables { varMap[v.Name] = v } for _, v := range c.Variables { if v.Type() == VariableTypeUnknown { errs = append(errs, fmt.Errorf( "Variable '%s': must be string or mapping", v.Name)) continue } interp := false fn := func(i Interpolation) (string, error) { interp = true return "", nil } w := &interpolationWalker{F: fn} if v.Default != nil { if err := reflectwalk.Walk(v.Default, w); err == nil { if interp { errs = append(errs, fmt.Errorf( "Variable '%s': cannot contain interpolations", v.Name)) } } } } // Check for references to user variables that do not actually // exist and record those errors. for source, vs := range vars { for _, v := range vs { uv, ok := v.(*UserVariable) if !ok { continue } if _, ok := varMap[uv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown variable referenced: %s", source, uv.Name)) } } } // Check that all count variables are valid. for source, vs := range vars { for _, rawV := range vs { switch v := rawV.(type) { case *CountVariable: if v.Type == CountValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid count variable: %s", source, v.FullKey())) } case *PathVariable: if v.Type == PathValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid path variable: %s", source, v.FullKey())) } } } } // Check that all references to modules are valid modules := make(map[string]*Module) dupped := make(map[string]struct{}) for _, m := range c.Modules { // Check for duplicates if _, ok := modules[m.Id()]; ok { if _, ok := dupped[m.Id()]; !ok { dupped[m.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: module repeated multiple times", m.Id())) } // Already seen this module, just skip it continue } modules[m.Id()] = m // Check that the source has no interpolations rc, err := NewRawConfig(map[string]interface{}{ "root": m.Source, }) if err != nil { errs = append(errs, fmt.Errorf( "%s: module source error: %s", m.Id(), err)) } else if len(rc.Interpolations) > 0 { errs = append(errs, fmt.Errorf( "%s: module source cannot contain interpolations", m.Id())) } // Check that the name matches our regexp if !NameRegexp.Match([]byte(m.Name)) { errs = append(errs, fmt.Errorf( "%s: module name can only contain letters, numbers, "+ "dashes, and underscores", m.Id())) } // Check that the configuration can all be strings raw := make(map[string]interface{}) for k, v := range m.RawConfig.Raw { var strVal string if err := mapstructure.WeakDecode(v, &strVal); err != nil { errs = append(errs, fmt.Errorf( "%s: variable %s must be a string value", m.Id(), k)) } raw[k] = strVal } // Update the raw configuration to only contain the string values m.RawConfig, err = NewRawConfig(raw) if err != nil { errs = append(errs, fmt.Errorf( "%s: can't initialize configuration: %s", m.Id(), err)) } } dupped = nil // Check that all variables for modules reference modules that // exist. for source, vs := range vars { for _, v := range vs { mv, ok := v.(*ModuleVariable) if !ok { continue } if _, ok := modules[mv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown module referenced: %s", source, mv.Name)) } } } // Check that all references to resources are valid resources := make(map[string]*Resource) dupped = make(map[string]struct{}) for _, r := range c.Resources { if _, ok := resources[r.Id()]; ok { if _, ok := dupped[r.Id()]; !ok { dupped[r.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: resource repeated multiple times", r.Id())) } } resources[r.Id()] = r } dupped = nil // Validate resources for n, r := range resources { // Verify count variables for _, v := range r.RawCount.Variables { switch v.(type) { case *CountVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference count variable: %s", n, v.FullKey())) case *ModuleVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference module variable: %s", n, v.FullKey())) case *ResourceVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference resource variable: %s", n, v.FullKey())) case *UserVariable: // Good default: panic("Unknown type in count var: " + n) } } // Interpolate with a fixed number to verify that its a number r.RawCount.interpolate(func(Interpolation) (string, error) { return "5", nil }) _, err := strconv.ParseInt(r.RawCount.Value().(string), 0, 0) if err != nil { errs = append(errs, fmt.Errorf( "%s: resource count must be an integer", n)) } r.RawCount.init() // Verify depends on points to resources that all exist for _, d := range r.DependsOn { if _, ok := resources[d]; !ok { errs = append(errs, fmt.Errorf( "%s: resource depends on non-existent resource '%s'", n, d)) } } } for source, vs := range vars { for _, v := range vs { rv, ok := v.(*ResourceVariable) if !ok { continue } id := fmt.Sprintf("%s.%s", rv.Type, rv.Name) if _, ok := resources[id]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown resource '%s' referenced in variable %s", source, id, rv.FullKey())) continue } } } // Check that all outputs are valid for _, o := range c.Outputs { invalid := false for k, _ := range o.RawConfig.Raw { if k != "value" { invalid = true break } } if invalid { errs = append(errs, fmt.Errorf( "%s: output should only have 'value' field", o.Name)) } } // Check that all variables are in the proper context for source, rc := range c.rawConfigs() { walker := &interpolationWalker{ ContextF: c.validateVarContextFn(source, &errs), } if err := reflectwalk.Walk(rc.Raw, walker); err != nil { errs = append(errs, fmt.Errorf( "%s: error reading config: %s", source, err)) } } if len(errs) > 0 { return &multierror.Error{Errors: errs} } return nil }
func TestInterpolationWalker_detect(t *testing.T) { cases := []struct { Input interface{} Result []string }{ { Input: map[string]interface{}{ "foo": "$${var.foo}", }, Result: []string{ "Literal(TypeString, ${var.foo})", }, }, { Input: map[string]interface{}{ "foo": "${var.foo}", }, Result: []string{ "Variable(var.foo)", }, }, { Input: map[string]interface{}{ "foo": "${aws_instance.foo.*.num}", }, Result: []string{ "Variable(aws_instance.foo.*.num)", }, }, { Input: map[string]interface{}{ "foo": "${lookup(var.foo)}", }, Result: []string{ "Call(lookup, Variable(var.foo))", }, }, { Input: map[string]interface{}{ "foo": `${file("test.txt")}`, }, Result: []string{ "Call(file, Literal(TypeString, test.txt))", }, }, { Input: map[string]interface{}{ "foo": `${file("foo/bar.txt")}`, }, Result: []string{ "Call(file, Literal(TypeString, foo/bar.txt))", }, }, { Input: map[string]interface{}{ "foo": `${join(",", foo.bar.*.id)}`, }, Result: []string{ "Call(join, Literal(TypeString, ,), Variable(foo.bar.*.id))", }, }, { Input: map[string]interface{}{ "foo": `${concat("localhost", ":8080")}`, }, Result: []string{ "Call(concat, Literal(TypeString, localhost), Literal(TypeString, :8080))", }, }, } for i, tc := range cases { var actual []string detectFn := func(root ast.Node) (string, error) { actual = append(actual, fmt.Sprintf("%s", root)) return "", nil } w := &interpolationWalker{F: detectFn} if err := reflectwalk.Walk(tc.Input, w); err != nil { t.Fatalf("err: %s", err) } if !reflect.DeepEqual(actual, tc.Result) { t.Fatalf("%d: bad:\n\n%#v", i, actual) } } }
func TestInterpolationWalker_replace(t *testing.T) { cases := []struct { Input interface{} Output interface{} Value string }{ { Input: map[string]interface{}{ "foo": "$${var.foo}", }, Output: map[string]interface{}{ "foo": "bar", }, Value: "bar", }, { Input: map[string]interface{}{ "foo": "hello, ${var.foo}", }, Output: map[string]interface{}{ "foo": "bar", }, Value: "bar", }, { Input: map[string]interface{}{ "foo": map[string]interface{}{ "${var.foo}": "bar", }, }, Output: map[string]interface{}{ "foo": map[string]interface{}{ "bar": "bar", }, }, Value: "bar", }, { Input: map[string]interface{}{ "foo": []interface{}{ "${var.foo}", "bing", }, }, Output: map[string]interface{}{ "foo": []interface{}{ "bar", "baz", "bing", }, }, Value: NewStringList([]string{"bar", "baz"}).String(), }, { Input: map[string]interface{}{ "foo": []interface{}{ "${var.foo}", "bing", }, }, Output: map[string]interface{}{}, Value: NewStringList([]string{UnknownVariableValue, "baz"}).String(), }, } for i, tc := range cases { fn := func(ast.Node) (string, error) { return tc.Value, nil } w := &interpolationWalker{F: fn, Replace: true} if err := reflectwalk.Walk(tc.Input, w); err != nil { t.Fatalf("err: %s", err) } if !reflect.DeepEqual(tc.Input, tc.Output) { t.Fatalf("%d: bad:\n\nexpected:%#v\ngot:%#v", i, tc.Output, tc.Input) } } }
// Validate does some basic semantic checking of the configuration. func (c *Config) Validate() error { if c == nil { return nil } var errs []error for _, k := range c.unknownKeys { errs = append(errs, fmt.Errorf( "Unknown root level key: %s", k)) } // Validate the Terraform config if tf := c.Terraform; tf != nil { if raw := tf.RequiredVersion; raw != "" { // Check that the value has no interpolations rc, err := NewRawConfig(map[string]interface{}{ "root": raw, }) if err != nil { errs = append(errs, fmt.Errorf( "terraform.required_version: %s", err)) } else if len(rc.Interpolations) > 0 { errs = append(errs, fmt.Errorf( "terraform.required_version: cannot contain interpolations")) } else { // Check it is valid _, err := version.NewConstraint(raw) if err != nil { errs = append(errs, fmt.Errorf( "terraform.required_version: invalid syntax: %s", err)) } } } } vars := c.InterpolatedVariables() varMap := make(map[string]*Variable) for _, v := range c.Variables { if _, ok := varMap[v.Name]; ok { errs = append(errs, fmt.Errorf( "Variable '%s': duplicate found. Variable names must be unique.", v.Name)) } varMap[v.Name] = v } for k, _ := range varMap { if !NameRegexp.MatchString(k) { errs = append(errs, fmt.Errorf( "variable %q: variable name must match regular expresion %s", k, NameRegexp)) } } for _, v := range c.Variables { if v.Type() == VariableTypeUnknown { errs = append(errs, fmt.Errorf( "Variable '%s': must be a string or a map", v.Name)) continue } interp := false fn := func(ast.Node) (interface{}, error) { interp = true return "", nil } w := &interpolationWalker{F: fn} if v.Default != nil { if err := reflectwalk.Walk(v.Default, w); err == nil { if interp { errs = append(errs, fmt.Errorf( "Variable '%s': cannot contain interpolations", v.Name)) } } } } // Check for references to user variables that do not actually // exist and record those errors. for source, vs := range vars { for _, v := range vs { uv, ok := v.(*UserVariable) if !ok { continue } if _, ok := varMap[uv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown variable referenced: '%s'. define it with 'variable' blocks", source, uv.Name)) } } } // Check that all count variables are valid. for source, vs := range vars { for _, rawV := range vs { switch v := rawV.(type) { case *CountVariable: if v.Type == CountValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid count variable: %s", source, v.FullKey())) } case *PathVariable: if v.Type == PathValueInvalid { errs = append(errs, fmt.Errorf( "%s: invalid path variable: %s", source, v.FullKey())) } } } } // Check that providers aren't declared multiple times. providerSet := make(map[string]struct{}) for _, p := range c.ProviderConfigs { name := p.FullName() if _, ok := providerSet[name]; ok { errs = append(errs, fmt.Errorf( "provider.%s: declared multiple times, you can only declare a provider once", name)) continue } providerSet[name] = struct{}{} } // Check that all references to modules are valid modules := make(map[string]*Module) dupped := make(map[string]struct{}) for _, m := range c.Modules { // Check for duplicates if _, ok := modules[m.Id()]; ok { if _, ok := dupped[m.Id()]; !ok { dupped[m.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: module repeated multiple times", m.Id())) } // Already seen this module, just skip it continue } modules[m.Id()] = m // Check that the source has no interpolations rc, err := NewRawConfig(map[string]interface{}{ "root": m.Source, }) if err != nil { errs = append(errs, fmt.Errorf( "%s: module source error: %s", m.Id(), err)) } else if len(rc.Interpolations) > 0 { errs = append(errs, fmt.Errorf( "%s: module source cannot contain interpolations", m.Id())) } // Check that the name matches our regexp if !NameRegexp.Match([]byte(m.Name)) { errs = append(errs, fmt.Errorf( "%s: module name can only contain letters, numbers, "+ "dashes, and underscores", m.Id())) } // Check that the configuration can all be strings, lists or maps raw := make(map[string]interface{}) for k, v := range m.RawConfig.Raw { var strVal string if err := hilmapstructure.WeakDecode(v, &strVal); err == nil { raw[k] = strVal continue } var mapVal map[string]interface{} if err := hilmapstructure.WeakDecode(v, &mapVal); err == nil { raw[k] = mapVal continue } var sliceVal []interface{} if err := hilmapstructure.WeakDecode(v, &sliceVal); err == nil { raw[k] = sliceVal continue } errs = append(errs, fmt.Errorf( "%s: variable %s must be a string, list or map value", m.Id(), k)) } // Check for invalid count variables for _, v := range m.RawConfig.Variables { switch v.(type) { case *CountVariable: errs = append(errs, fmt.Errorf( "%s: count variables are only valid within resources", m.Name)) case *SelfVariable: errs = append(errs, fmt.Errorf( "%s: self variables are only valid within resources", m.Name)) } } // Update the raw configuration to only contain the string values m.RawConfig, err = NewRawConfig(raw) if err != nil { errs = append(errs, fmt.Errorf( "%s: can't initialize configuration: %s", m.Id(), err)) } } dupped = nil // Check that all variables for modules reference modules that // exist. for source, vs := range vars { for _, v := range vs { mv, ok := v.(*ModuleVariable) if !ok { continue } if _, ok := modules[mv.Name]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown module referenced: %s", source, mv.Name)) } } } // Check that all references to resources are valid resources := make(map[string]*Resource) dupped = make(map[string]struct{}) for _, r := range c.Resources { if _, ok := resources[r.Id()]; ok { if _, ok := dupped[r.Id()]; !ok { dupped[r.Id()] = struct{}{} errs = append(errs, fmt.Errorf( "%s: resource repeated multiple times", r.Id())) } } resources[r.Id()] = r } dupped = nil // Validate resources for n, r := range resources { // Verify count variables for _, v := range r.RawCount.Variables { switch v.(type) { case *CountVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference count variable: %s", n, v.FullKey())) case *ModuleVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference module variable: %s", n, v.FullKey())) case *ResourceVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference resource variable: %s", n, v.FullKey())) case *SimpleVariable: errs = append(errs, fmt.Errorf( "%s: resource count can't reference variable: %s", n, v.FullKey())) case *UserVariable: // Good default: panic(fmt.Sprintf("Unknown type in count var in %s: %T", n, v)) } } // Interpolate with a fixed number to verify that its a number. r.RawCount.interpolate(func(root ast.Node) (interface{}, error) { // Execute the node but transform the AST so that it returns // a fixed value of "5" for all interpolations. result, err := hil.Eval( hil.FixedValueTransform( root, &ast.LiteralNode{Value: "5", Typex: ast.TypeString}), nil) if err != nil { return "", err } return result.Value, nil }) _, err := strconv.ParseInt(r.RawCount.Value().(string), 0, 0) if err != nil { errs = append(errs, fmt.Errorf( "%s: resource count must be an integer", n)) } r.RawCount.init() // Validate DependsOn errs = append(errs, c.validateDependsOn(n, r.DependsOn, resources, modules)...) // Verify provisioners don't contain any splats for _, p := range r.Provisioners { // This validation checks that there are now splat variables // referencing ourself. This currently is not allowed. for _, v := range p.ConnInfo.Variables { rv, ok := v.(*ResourceVariable) if !ok { continue } if rv.Multi && rv.Index == -1 && rv.Type == r.Type && rv.Name == r.Name { errs = append(errs, fmt.Errorf( "%s: connection info cannot contain splat variable "+ "referencing itself", n)) break } } for _, v := range p.RawConfig.Variables { rv, ok := v.(*ResourceVariable) if !ok { continue } if rv.Multi && rv.Index == -1 && rv.Type == r.Type && rv.Name == r.Name { errs = append(errs, fmt.Errorf( "%s: connection info cannot contain splat variable "+ "referencing itself", n)) break } } } // Verify ignore_changes contains valid entries for _, v := range r.Lifecycle.IgnoreChanges { if strings.Contains(v, "*") && v != "*" { errs = append(errs, fmt.Errorf( "%s: ignore_changes does not support using a partial string "+ "together with a wildcard: %s", n, v)) } } // Verify ignore_changes has no interpolations rc, err := NewRawConfig(map[string]interface{}{ "root": r.Lifecycle.IgnoreChanges, }) if err != nil { errs = append(errs, fmt.Errorf( "%s: lifecycle ignore_changes error: %s", n, err)) } else if len(rc.Interpolations) > 0 { errs = append(errs, fmt.Errorf( "%s: lifecycle ignore_changes cannot contain interpolations", n)) } // If it is a data source then it can't have provisioners if r.Mode == DataResourceMode { if _, ok := r.RawConfig.Raw["provisioner"]; ok { errs = append(errs, fmt.Errorf( "%s: data sources cannot have provisioners", n)) } } } for source, vs := range vars { for _, v := range vs { rv, ok := v.(*ResourceVariable) if !ok { continue } id := rv.ResourceId() if _, ok := resources[id]; !ok { errs = append(errs, fmt.Errorf( "%s: unknown resource '%s' referenced in variable %s", source, id, rv.FullKey())) continue } } } // Check that all outputs are valid { found := make(map[string]struct{}) for _, o := range c.Outputs { // Verify the output is new if _, ok := found[o.Name]; ok { errs = append(errs, fmt.Errorf( "%s: duplicate output. output names must be unique.", o.Name)) continue } found[o.Name] = struct{}{} var invalidKeys []string valueKeyFound := false for k := range o.RawConfig.Raw { if k == "value" { valueKeyFound = true continue } if k == "sensitive" { if sensitive, ok := o.RawConfig.config[k].(bool); ok { if sensitive { o.Sensitive = true } continue } errs = append(errs, fmt.Errorf( "%s: value for 'sensitive' must be boolean", o.Name)) continue } if k == "description" { if desc, ok := o.RawConfig.config[k].(string); ok { o.Description = desc continue } errs = append(errs, fmt.Errorf( "%s: value for 'description' must be string", o.Name)) continue } invalidKeys = append(invalidKeys, k) } if len(invalidKeys) > 0 { errs = append(errs, fmt.Errorf( "%s: output has invalid keys: %s", o.Name, strings.Join(invalidKeys, ", "))) } if !valueKeyFound { errs = append(errs, fmt.Errorf( "%s: output is missing required 'value' key", o.Name)) } for _, v := range o.RawConfig.Variables { if _, ok := v.(*CountVariable); ok { errs = append(errs, fmt.Errorf( "%s: count variables are only valid within resources", o.Name)) } } } } // Check that all variables are in the proper context for source, rc := range c.rawConfigs() { walker := &interpolationWalker{ ContextF: c.validateVarContextFn(source, &errs), } if err := reflectwalk.Walk(rc.Raw, walker); err != nil { errs = append(errs, fmt.Errorf( "%s: error reading config: %s", source, err)) } } // Validate the self variable for source, rc := range c.rawConfigs() { // Ignore provisioners. This is a pretty brittle way to do this, // but better than also repeating all the resources. if strings.Contains(source, "provision") { continue } for _, v := range rc.Variables { if _, ok := v.(*SelfVariable); ok { errs = append(errs, fmt.Errorf( "%s: cannot contain self-reference %s", source, v.FullKey())) } } } if len(errs) > 0 { return &multierror.Error{Errors: errs} } return nil }
func DumpConfig(config interface{}, say func(string)) { walker := newDumpConfig(say) reflectwalk.Walk(config, walker) }
func TestInterpolationWalker_detect(t *testing.T) { cases := []struct { Input interface{} Result []Interpolation }{ { Input: map[string]interface{}{ "foo": "$${var.foo}", }, Result: nil, }, { Input: map[string]interface{}{ "foo": "${var.foo}", }, Result: []Interpolation{ &VariableInterpolation{ Variable: &UserVariable{ Name: "foo", key: "var.foo", }, }, }, }, { Input: map[string]interface{}{ "foo": "${aws_instance.foo.*.num}", }, Result: []Interpolation{ &VariableInterpolation{ Variable: &ResourceVariable{ Type: "aws_instance", Name: "foo", Field: "num", Multi: true, Index: -1, key: "aws_instance.foo.*.num", }, }, }, }, { Input: map[string]interface{}{ "foo": "${lookup(var.foo)}", }, Result: []Interpolation{ &FunctionInterpolation{ Func: nil, Args: []Interpolation{ &VariableInterpolation{ Variable: &UserVariable{ Name: "foo", key: "var.foo", }, }, }, }, }, }, { Input: map[string]interface{}{ "foo": `${file("test.txt")}`, }, Result: []Interpolation{ &FunctionInterpolation{ Func: nil, Args: []Interpolation{ &LiteralInterpolation{ Literal: "test.txt", }, }, }, }, }, { Input: map[string]interface{}{ "foo": `${file("foo/bar.txt")}`, }, Result: []Interpolation{ &FunctionInterpolation{ Func: nil, Args: []Interpolation{ &LiteralInterpolation{ Literal: "foo/bar.txt", }, }, }, }, }, { Input: map[string]interface{}{ "foo": `${join(",", foo.bar.*.id)}`, }, Result: []Interpolation{ &FunctionInterpolation{ Func: nil, Args: []Interpolation{ &LiteralInterpolation{ Literal: ",", }, &VariableInterpolation{ Variable: &ResourceVariable{ Type: "foo", Name: "bar", Field: "id", Multi: true, Index: -1, key: "foo.bar.*.id", }, }, }, }, }, }, } for i, tc := range cases { var actual []Interpolation detectFn := func(i Interpolation) (string, error) { actual = append(actual, i) return "", nil } w := &interpolationWalker{F: detectFn} if err := reflectwalk.Walk(tc.Input, w); err != nil { t.Fatalf("err: %s", err) } for _, a := range actual { // This is jank, but reflect.DeepEqual never has functions // being the same. if f, ok := a.(*FunctionInterpolation); ok { f.Func = nil } } if !reflect.DeepEqual(actual, tc.Result) { t.Fatalf("%d: bad:\n\n%#v", i, actual) } } }