// DecodeConfig is a helper that handles decoding raw configuration using
// mapstructure. It returns the metadata and any errors that may happen.
// If you need extra configuration for mapstructure, you should configure
// it manually and not use this helper function.
func DecodeConfig(target interface{}, raws ...interface{}) (*mapstructure.Metadata, error) {
decodeHook, err := decodeConfigHook(raws)
if err != nil {
return nil, err
}
var md mapstructure.Metadata
decoderConfig := &mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
decodeHook,
mapstructure.StringToSliceHookFunc(","),
),
Metadata: &md,
Result: target,
WeaklyTypedInput: true,
}
decoder, err := mapstructure.NewDecoder(decoderConfig)
if err != nil {
return nil, err
}
for _, raw := range raws {
err := decoder.Decode(raw)
if err != nil {
return nil, err
}
}
return &md, nil
}
// NewMapDecoder returns decoder configured for decoding data into result with all registered hooks.
func newMapDecoder(result interface{}) (*mapstructure.Decoder, error) {
return mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(mapDecoderHooks...),
Metadata: nil,
Result: result,
TagName: TagName,
})
}
func decodeResult(m, result interface{}) error {
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
ZeroFields: true,
WeaklyTypedInput: true,
Result: result,
TagName: rreflect.TagName,
DecodeHook: mapstructure.ComposeDecodeHookFunc(
decodeBigIntHook,
),
})
if err != nil {
return err
}
return dec.Decode(m)
}
func transform(source map[string]interface{}, target interface{}) error {
data := mapstructure.Metadata{}
config := &mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
transformHook,
mapstructure.StringToTimeDurationHookFunc()),
Result: target,
Metadata: &data,
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
return err
}
err = decoder.Decode(source)
// TODO: log unused keys
return err
}
// decodeJSON is used to decode an HTTP response body into an interface as JSON.
func decodeJSON(out interface{}, body io.ReadCloser) error {
defer body.Close()
var parsed interface{}
dec := json.NewDecoder(body)
if err := dec.Decode(&parsed); err != nil {
return err
}
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
mapToHTTPHeaderHookFunc(),
stringToTimeHookFunc(),
),
WeaklyTypedInput: true,
Result: out,
})
if err != nil {
return err
}
return decoder.Decode(parsed)
}
// ParseConfig reads the configuration file at the given path and returns a new
// Config struct with the data populated.
func ParseConfig(path string) (*Config, error) {
var errs *multierror.Error
// Read the contents of the file
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("error reading config at %q: %s", path, err)
}
// Parse the file (could be HCL or JSON)
var shadow interface{}
if err := hcl.Decode(&shadow, string(contents)); err != nil {
return nil, fmt.Errorf("error decoding config at %q: %s", path, err)
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error converting config at %q", path)
}
flattenKeys(parsed, []string{"auth", "ssl", "syslog", "vault", "deduplicate"})
// Create a new, empty config
config := new(Config)
// Use mapstructure to populate the basic config fields
metadata := new(mapstructure.Metadata)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
watch.StringToWaitDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: metadata,
Result: config,
})
if err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
if err := decoder.Decode(parsed); err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
// Store a reference to the path where this config was read from
config.Path = path
// Ensure there's a default value for the template's file permissions
for _, t := range config.ConfigTemplates {
if t.Perms == 0000 {
t.Perms = defaultFilePerms
}
}
// Update the list of set keys
if config.setKeys == nil {
config.setKeys = make(map[string]struct{})
}
for _, key := range metadata.Keys {
if _, ok := config.setKeys[key]; !ok {
config.setKeys[key] = struct{}{}
}
}
config.setKeys["path"] = struct{}{}
d := DefaultConfig()
d.Merge(config)
config = d
return config, errs.ErrorOrNil()
}
// ParseConfig reads the configuration file at the given path and returns a new
// Config struct with the data populated.
func ParseConfig(path string) (*Config, error) {
var errs *multierror.Error
// Read the contents of the file
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("error reading config at %q: %s", path, err)
}
// Parse the file (could be HCL or JSON)
var shadow interface{}
if err := hcl.Decode(&shadow, string(contents)); err != nil {
return nil, fmt.Errorf("error decoding config at %q: %s", path, err)
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error converting config at %q", path)
}
flattenKeys(parsed, []string{"auth", "ssl", "syslog"})
// Parse the prefixes
if raw, ok := parsed["prefixes"]; ok {
if typed, ok := raw.([]interface{}); !ok {
err = fmt.Errorf("error converting prefixes to []interface{} at %q, was %T", path, raw)
errs = multierror.Append(errs, err)
delete(parsed, "prefixes")
} else {
prefixes := make([]*dep.StoreKeyPrefix, 0, len(typed))
for _, p := range typed {
if s, ok := p.(string); ok {
if prefix, err := dep.ParseStoreKeyPrefix(s); err != nil {
err = fmt.Errorf("error parsing prefix %q at %q: %s", p, path, err)
errs = multierror.Append(errs, err)
} else {
prefixes = append(prefixes, prefix)
}
} else {
err = fmt.Errorf("error converting %T to string", p)
errs = multierror.Append(errs, err)
delete(parsed, "prefixes")
}
}
parsed["prefixes"] = prefixes
}
}
// Parse the wait component
if raw, ok := parsed["wait"]; ok {
if typed, ok := raw.(string); !ok {
err = fmt.Errorf("error converting wait to string at %q", path)
errs = multierror.Append(errs, err)
delete(parsed, "wait")
} else {
if wait, err := watch.ParseWait(typed); err != nil {
err = fmt.Errorf("error parsing wait at %q: %s", path, err)
errs = multierror.Append(errs, err)
delete(parsed, "wait")
} else {
parsed["wait"] = map[string]time.Duration{
"min": wait.Min,
"max": wait.Max,
}
}
}
}
// Create a new, empty config
config := new(Config)
// Use mapstructure to populate the basic config fields
metadata := new(mapstructure.Metadata)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: metadata,
Result: config,
})
if err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
if err := decoder.Decode(parsed); err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
// Store a reference to the path where this config was read from
config.Path = path
// Update the list of set keys
if config.setKeys == nil {
config.setKeys = make(map[string]struct{})
}
for _, key := range metadata.Keys {
if _, ok := config.setKeys[key]; !ok {
config.setKeys[key] = struct{}{}
}
}
config.setKeys["path"] = struct{}{}
d := DefaultConfig()
d.Merge(config)
config = d
return config, errs.ErrorOrNil()
}
// Parse parses the given string contents as a config
func Parse(s string) (*Config, error) {
var errs *multierror.Error
// Parse the file (could be HCL or JSON)
var shadow interface{}
if err := hcl.Decode(&shadow, s); err != nil {
return nil, fmt.Errorf("error decoding config: %s", err)
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error converting config")
}
flattenKeys(parsed, []string{
"auth",
"ssl",
"syslog",
"exec",
"vault",
"deduplicate",
})
// Deprecations
if vault, ok := parsed["vault"].(map[string]interface{}); ok {
if val, ok := vault["renew"]; ok {
log.Println(`[WARN] vault.renew has been renamed to vault.renew_token. ` +
`Update your configuration files and change "renew" to "renew_token".`)
vault["renew_token"] = val
delete(vault, "renew")
}
}
// Create a new, empty config
config := new(Config)
// Use mapstructure to populate the basic config fields
metadata := new(mapstructure.Metadata)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
StringToFileModeFunc(),
signals.StringToSignalFunc(),
watch.StringToWaitDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: metadata,
Result: config,
})
if err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
if err := decoder.Decode(parsed); err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
// Explicitly check for the nil signal and set the value back to nil
if config.ReloadSignal == signals.SIGNIL {
config.ReloadSignal = nil
}
if config.DumpSignal == signals.SIGNIL {
config.DumpSignal = nil
}
if config.KillSignal == signals.SIGNIL {
config.KillSignal = nil
}
if config.Exec != nil {
if config.Exec.ReloadSignal == signals.SIGNIL {
config.Exec.ReloadSignal = nil
}
if config.Exec.KillSignal == signals.SIGNIL {
config.Exec.KillSignal = nil
}
}
// Setup default values for templates
for _, t := range config.ConfigTemplates {
// Ensure there's a default value for the template's file permissions
if t.Perms == 0000 {
t.Perms = DefaultFilePerms
}
// Ensure we have a default command timeout
if t.CommandTimeout == 0 {
t.CommandTimeout = DefaultCommandTimeout
}
// Set up a default zero wait, which disables it for this
// template.
if t.Wait == nil {
t.Wait = &watch.Wait{}
}
}
// Update the list of set keys
if config.setKeys == nil {
config.setKeys = make(map[string]struct{})
}
for _, key := range metadata.Keys {
if _, ok := config.setKeys[key]; !ok {
config.setKeys[key] = struct{}{}
}
}
config.setKeys["path"] = struct{}{}
d := DefaultConfig()
d.Merge(config)
config = d
return config, errs.ErrorOrNil()
}
// ParseConfig reads the configuration file at the given path and returns a new
// Config struct with the data populated.
func ParseConfig(path string) (*Config, error) {
var errs *multierror.Error
// Read the contents of the file
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("error reading config at %q: %s", path, err)
}
// Parse the file (could be HCL or JSON)
var shadow interface{}
if err := hcl.Decode(&shadow, string(contents)); err != nil {
return nil, fmt.Errorf("error decoding config at %q: %s", path, err)
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error converting config at %q", path)
}
flattenKeys(parsed, []string{"auth", "ssl", "syslog", "vault"})
// Create a new, empty config
config := new(Config)
// Use mapstructure to populate the basic config fields
metadata := new(mapstructure.Metadata)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
watch.StringToWaitDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: metadata,
Result: config,
})
if err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
if err := decoder.Decode(parsed); err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
// Store a reference to the path where this config was read from
config.Path = path
// Handle deprecations
if len(config.PrefixesOld) > 0 {
log.Printf(`[WARN] Specifying the key "prefixes" in the configuration is `+
`no longer supported. Please specify each prefix individually using `+
`the key "prefix" (config at %s)`, path)
prefixes := make([]*ConfigPrefix, 0, len(config.PrefixesOld))
for _, prefix := range config.PrefixesOld {
prefixes = append(prefixes, &ConfigPrefix{
Path: prefix,
})
}
config.Prefixes = append(prefixes, config.Prefixes...)
config.PrefixesOld = nil
}
// Update the list of set keys
if config.setKeys == nil {
config.setKeys = make(map[string]struct{})
}
for _, key := range metadata.Keys {
if _, ok := config.setKeys[key]; !ok {
config.setKeys[key] = struct{}{}
}
}
config.setKeys["path"] = struct{}{}
d := DefaultConfig()
d.Merge(config)
config = d
return config, errs.ErrorOrNil()
}
// GetWorkUnits retrieves the keys and data dictionaries for some number
// of work units. If options contains "work_unit_keys", those specific
// work units are retrieved; otherwise the work units are based on
// which of GetWorkUnitsOptions are present.
//
// On success, the return value is a slice of cborrpc.PythonTuple
// objects where each contains the work unit key as a byte slice and
// the data dictionary.
func (jobs *JobServer) GetWorkUnits(workSpecName string, options map[string]interface{}) ([]interface{}, string, error) {
var workUnits map[string]coordinate.WorkUnit
gwuOptions := GetWorkUnitsOptions{
Limit: 1000,
}
spec, err := jobs.Namespace.WorkSpec(workSpecName)
var decoder *mapstructure.Decoder
if err == nil {
config := mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(gwuStateHook, cborrpc.DecodeBytesAsString),
Result: &gwuOptions,
}
decoder, err = mapstructure.NewDecoder(&config)
}
if err == nil {
err = decoder.Decode(options)
}
if err == nil {
query := coordinate.WorkUnitQuery{
Names: gwuOptions.WorkUnitKeys,
}
if gwuOptions.WorkUnitKeys == nil {
query.PreviousName = gwuOptions.Start
query.Limit = gwuOptions.Limit
}
if gwuOptions.WorkUnitKeys == nil && gwuOptions.State != nil {
query.Statuses = make([]coordinate.WorkUnitStatus, len(gwuOptions.State))
for i, state := range gwuOptions.State {
query.Statuses[i], err = translateWorkUnitStatus(state)
if err != nil {
break
}
}
}
if err == nil {
workUnits, err = spec.WorkUnits(query)
}
}
if err != nil {
return nil, "", err
}
// The marshalled result is a list of pairs of (key, data).
var result []interface{}
for name, unit := range workUnits {
var data map[string]interface{}
attempt, err := unit.ActiveAttempt()
if err == nil && attempt != nil {
data, err = attempt.Data()
}
if err == nil && data == nil {
data, err = unit.Data()
}
if err != nil {
return nil, "", err
}
tuple := cborrpc.PythonTuple{Items: []interface{}{[]byte(name), data}}
result = append(result, tuple)
}
return result, "", nil
}
// Parse parses the given string contents as a config
func Parse(s string) (*Config, error) {
var shadow interface{}
if err := hcl.Decode(&shadow, s); err != nil {
return nil, errors.Wrap(err, "error decoding config")
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, errors.New("error converting config")
}
flattenKeys(parsed, []string{
"auth",
"consul",
"consul.auth",
"consul.retry",
"consul.ssl",
"deduplicate",
"env",
"exec",
"exec.env",
"ssl",
"syslog",
"vault",
"vault.retry",
"vault.ssl",
"wait",
})
// FlattenFlatten keys belonging to the templates. We cannot do this above
// because it is an array of tmeplates.
if templates, ok := parsed["template"].([]map[string]interface{}); ok {
for _, template := range templates {
flattenKeys(template, []string{
"env",
"exec",
"exec.env",
"wait",
})
}
}
// TODO: Deprecations
if vault, ok := parsed["vault"].(map[string]interface{}); ok {
if val, ok := vault["renew"]; ok {
log.Println(`[WARN] vault.renew has been renamed to vault.renew_token. ` +
`Update your configuration files and change "renew" to "renew_token".`)
vault["renew_token"] = val
delete(vault, "renew")
}
}
if auth, ok := parsed["auth"].(map[string]interface{}); ok {
log.Println("[WARN] auth has been moved under the consul stanza. " +
"Update your configuration files and place auth inside consul { }.")
if _, ok := parsed["consul"]; !ok {
parsed["consul"] = make(map[string]interface{})
}
parsed["consul"].(map[string]interface{})["auth"] = auth
delete(parsed, "auth")
}
if retry, ok := parsed["retry"].(string); ok {
log.Println("[WARN] retry has been moved under the consul stanza. " +
"Update your configuration files and place retry inside consul { }.")
if _, ok := parsed["consul"]; !ok {
parsed["consul"] = make(map[string]interface{})
}
parsed["consul"].(map[string]interface{})["retry"] = map[string]interface{}{
"backoff": retry,
}
delete(parsed, "retry")
}
if ssl, ok := parsed["ssl"].(map[string]interface{}); ok {
log.Println("[WARN] ssl has been moved under the consul stanza. " +
"Update your configuration files and place ssl inside consul { }.")
if _, ok := parsed["consul"]; !ok {
parsed["consul"] = make(map[string]interface{})
}
parsed["consul"].(map[string]interface{})["ssl"] = ssl
delete(parsed, "ssl")
}
if token, ok := parsed["token"].(string); ok {
log.Println("[WARN] token has been moved under the consul stanza. " +
"Update your configuration files and place token inside consul { }.")
if _, ok := parsed["consul"]; !ok {
parsed["consul"] = make(map[string]interface{})
}
parsed["consul"].(map[string]interface{})["token"] = token
delete(parsed, "token")
}
// Create a new, empty config
var c Config
// Use mapstructure to populate the basic config fields
var md mapstructure.Metadata
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
ConsulStringToStructFunc(),
StringToFileModeFunc(),
signals.StringToSignalFunc(),
StringToWaitDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: &md,
Result: &c,
})
if err != nil {
return nil, errors.Wrap(err, "mapstructure decoder creation failed")
}
if err := decoder.Decode(parsed); err != nil {
return nil, errors.Wrap(err, "mapstructure decode failed")
}
return &c, nil
}
// g reads the configuration file at the given path and returns a new
// Config struct with the data populated.
func ParseConfig(path string) (*Config, error) {
var errs *multierror.Error
// Read the contents of the file
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("error reading config at %q: %s", path, err)
}
// Parse the file (could be HCL or JSON)
var shadow interface{}
if err := hcl.Decode(&shadow, string(contents)); err != nil {
return nil, fmt.Errorf("error decoding config at %q: %s", path, err)
}
// Convert to a map and flatten the keys we want to flatten
parsed, ok := shadow.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error converting config at %q", path)
}
flattenKeys(parsed, []string{"auth", "ssl", "syslog"})
// Create a new, empty config
c := new(Config)
// Use mapstructure to populate the basic config fields
metadata := new(mapstructure.Metadata)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
config.StringToWaitDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
mapstructure.StringToTimeDurationHookFunc(),
),
ErrorUnused: true,
Metadata: metadata,
Result: c,
})
if err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
if err := decoder.Decode(parsed); err != nil {
errs = multierror.Append(errs, err)
return nil, errs.ErrorOrNil()
}
// Store a reference to the path where this config was read from
c.Path = path
// Parse the prefix sources
for _, prefix := range c.Prefixes {
parsed, err := dep.NewKVListQuery(prefix.Source)
if err != nil {
errs = multierror.Append(errs, err)
continue
}
prefix.Dependency = parsed
// If no destination was given, default to the prefix
if prefix.Destination == "" {
prefix.Destination = prefix.Source
}
}
// Update the list of set keys
if c.setKeys == nil {
c.setKeys = make(map[string]struct{})
}
for _, key := range metadata.Keys {
if _, ok := c.setKeys[key]; !ok {
c.setKeys[key] = struct{}{}
}
}
c.setKeys["path"] = struct{}{}
d := DefaultConfig()
d.Merge(c)
c = d
return c, errs.ErrorOrNil()
}
// Decode decodes the configuration into the target and optionally
// automatically interpolates all the configuration as it goes.
func Decode(target interface{}, config *DecodeOpts, raws ...interface{}) error {
if config == nil {
config = &DecodeOpts{Interpolate: true}
}
// Interpolate first
if config.Interpolate {
// Detect user variables from the raws and merge them into our context
ctx, err := DetectContext(raws...)
if err != nil {
return err
}
if config.InterpolateContext == nil {
config.InterpolateContext = ctx
} else {
config.InterpolateContext.UserVariables = ctx.UserVariables
}
ctx = config.InterpolateContext
// Render everything
for i, raw := range raws {
m, err := interpolate.RenderMap(raw, ctx, config.InterpolateFilter)
if err != nil {
return err
}
raws[i] = m
}
}
// Build our decoder
var md mapstructure.Metadata
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: target,
Metadata: &md,
WeaklyTypedInput: true,
DecodeHook: mapstructure.ComposeDecodeHookFunc(
uint8ToStringHook,
mapstructure.StringToSliceHookFunc(","),
),
})
if err != nil {
return err
}
for _, raw := range raws {
if err := decoder.Decode(raw); err != nil {
return err
}
}
// Set the metadata if it is set
if config.Metadata != nil {
*config.Metadata = md
}
// If we have unused keys, it is an error
if len(md.Unused) > 0 {
var err error
sort.Strings(md.Unused)
for _, unused := range md.Unused {
if unused != "type" && !strings.HasPrefix(unused, "packer_") {
err = multierror.Append(err, fmt.Errorf(
"unknown configuration key: %q", unused))
}
}
if err != nil {
return err
}
}
return nil
}