func diff(aFilePath, bFilePath string, separator string) {
aFile, err := ioutil.ReadFile(aFilePath)
if err != nil {
log.Fatalln(fmt.Sprintf("error reading a [%s]:", path.Clean(aFilePath)), err)
}
aYAML, err := yaml.Parse(aFilePath, aFile)
if err != nil {
log.Fatalln(fmt.Sprintf("error parsing a [%s]:", path.Clean(aFilePath)), err)
}
bFile, err := ioutil.ReadFile(bFilePath)
if err != nil {
log.Fatalln(fmt.Sprintf("error reading b [%s]:", path.Clean(bFilePath)), err)
}
bYAML, err := yaml.Parse(bFilePath, bFile)
if err != nil {
log.Fatalln(fmt.Sprintf("error parsing b [%s]:", path.Clean(bFilePath)), err)
}
diffs := compare.Compare(aYAML, bYAML)
if len(diffs) == 0 {
fmt.Println("no differences!")
return
}
for _, diff := range diffs {
fmt.Println("Difference in", strings.Join(diff.Path, "."))
if diff.A != nil {
ayaml, err := candiedyaml.Marshal(diff.A)
if err != nil {
panic(err)
}
fmt.Printf(" %s has:\n \x1b[31m%s\x1b[0m\n", aFilePath, strings.Replace(string(ayaml), "\n", "\n ", -1))
}
if diff.B != nil {
byaml, err := candiedyaml.Marshal(diff.B)
if err != nil {
panic(err)
}
fmt.Printf(" %s has:\n \x1b[32m%s\x1b[0m\n", bFilePath, strings.Replace(string(byaml), "\n", "\n ", -1))
}
fmt.Printf(separator)
}
}
// composeUp converts given json to yaml, saves to a file on the host and
// uses `docker-compose up -d` to create the containers.
func composeUp(d driver.DistroDriver, json map[string]interface{}) error {
if len(json) == 0 {
log.Println("docker-compose config not specified, noop")
return nil
}
// Convert json to yaml
yaml, err := yaml.Marshal(json)
if err != nil {
return fmt.Errorf("error converting to compose.yml: %v", err)
}
if err := os.MkdirAll(composeYmlDir, 0777); err != nil {
return fmt.Errorf("failed creating %s: %v", composeYmlDir, err)
}
log.Printf("Using compose yaml:>>>>>\n%s\n<<<<<", string(yaml))
ymlPath := filepath.Join(composeYmlDir, composeYml)
if err := ioutil.WriteFile(ymlPath, yaml, 0666); err != nil {
return fmt.Errorf("error writing %s: %v", ymlPath, err)
}
// set timeout for docker-compose -> docker-engine interactions.
// When downloading large images, docker-compose intermittently times out
// (gh#docker/compose/issues/2186).
os.Setenv("COMPOSE_HTTP_TIMEOUT", fmt.Sprintf("%d", composeTimeoutSecs)) // versions <= 1.4.2
os.Setenv("DOCKER_CLIENT_TIMEOUT", fmt.Sprintf("%d", composeTimeoutSecs)) // version >= 1.5.0
defer os.Unsetenv("COMPOSE_HTTP_TIMEOUT")
defer os.Unsetenv("DOCKER_CLIENT_TIMEOUT")
return executil.ExecPipeToFds(executil.Fds{Out: ioutil.Discard}, composeBinPath(d), "-p", composeProject, "-f", ymlPath, "up", "-d")
}
func configGet(c *cli.Context) {
arg := c.Args().Get(0)
if arg == "" {
return
}
cfg, err := config.LoadConfig()
if err != nil {
log.WithFields(log.Fields{"err": err}).Fatal("config get: failed to load config")
}
val, err := cfg.Get(arg)
if err != nil {
log.WithFields(log.Fields{"cfg": cfg, "key": arg, "val": val, "err": err}).Fatal("config get: failed to retrieve value")
}
printYaml := false
switch val.(type) {
case []interface{}:
printYaml = true
case map[interface{}]interface{}:
printYaml = true
}
if printYaml {
bytes, err := yaml.Marshal(val)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bytes))
} else {
fmt.Println(val)
}
}
func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
os.Remove(rancherConfig.CloudConfigScriptFile)
os.Remove(rancherConfig.CloudConfigBootFile)
os.Remove(rancherConfig.MetaDataFile)
if len(scriptBytes) > 0 {
log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
if err := ioutil.WriteFile(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
return err
}
}
if err := ioutil.WriteFile(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
return err
}
log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
metaDataBytes, err := yaml.Marshal(metadata)
if err != nil {
return err
}
if err = ioutil.WriteFile(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
return err
}
log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes))
return nil
}
func (c Client) ResolveManifestVersions(yaml []byte) ([]byte, error) {
m := manifest{}
err := candiedyaml.Unmarshal(yaml, &m)
if err != nil {
return nil, err
}
for i, r := range m.Releases {
if r.Version == "latest" {
release, err := c.Release(r.Name)
if err != nil {
return nil, err
}
r.Version = release.Latest()
m.Releases[i] = r
}
}
for i, pool := range m.ResourcePools {
if pool.Stemcell.Version == "latest" {
stemcell, err := c.Stemcell(pool.Stemcell.Name)
if err != nil {
return nil, err
}
pool.Stemcell.Version = stemcell.Latest()
m.ResourcePools[i] = pool
}
}
return candiedyaml.Marshal(m)
}
func Dump(boot, private, full bool) (string, error) {
var cfg *CloudConfig
var err error
if full {
cfg, err = LoadConfig()
} else {
files := []string{CloudConfigBootFile, CloudConfigPrivateFile, CloudConfigFile}
if !private {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
}
if !boot {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigBootFile })
}
cfg, err = ChainCfgFuncs(nil,
func(_ *CloudConfig) (*CloudConfig, error) { return ReadConfig(nil, true, files...) },
amendNils,
)
}
if err != nil {
return "", err
}
bytes, err := yaml.Marshal(*cfg)
return string(bytes), err
}
func formatYAML(yaml yaml.Node) string {
formatted, err := candiedyaml.Marshal(yaml)
if err != nil {
return fmt.Sprintf("\n\t<%T> %#v", yaml, yaml)
}
return fmt.Sprintf("\n\t%s", strings.Replace(string(formatted), "\n", "\n\t", -1))
}
func WriteToFile(data interface{}, filename string) error {
content, err := yaml.Marshal(data)
if err != nil {
return err
}
return ioutil.WriteFile(filename, content, 400)
}
func Convert(from, to interface{}) error {
bytes, err := yaml.Marshal(from)
if err != nil {
log.WithFields(log.Fields{"from": from, "err": err}).Warn("Error serializing to YML")
return err
}
return yaml.Unmarshal(bytes, to)
}
// Convert converts a struct (src) to another one (target) using yaml marshalling/unmarshalling.
// If the structure are not compatible, this will throw an error as the unmarshalling will fail.
func Convert(src, target interface{}) error {
newBytes, err := yaml.Marshal(src)
if err != nil {
return err
}
err = yaml.Unmarshal(newBytes, target)
if err != nil {
logrus.Errorf("Failed to unmarshall: %v\n%s", err, string(newBytes))
}
return err
}
func TestMarshalConfig(t *testing.T) {
config := newTestConfig()
bytes, err := yaml.Marshal(config)
assert.Nil(t, err)
config2 := TestConfig{}
err = yaml.Unmarshal(bytes, &config2)
assert.Nil(t, err)
assert.Equal(t, config, config2)
}
func TestMarshalServiceConfig(t *testing.T) {
configPtr := newTestConfig().SystemContainers["udev"]
bytes, err := yaml.Marshal(configPtr)
assert.Nil(t, err)
configPtr2 := &ServiceConfig{}
err = yaml.Unmarshal(bytes, configPtr2)
assert.Nil(t, err)
assert.Equal(t, configPtr, configPtr2)
}
func TestStr2SliceOrMapPtrMap(t *testing.T) {
s := map[string]*StructSliceorMap{"udav": {
Foos: SliceorMap{"io.rancher.os.bar": "baz", "io.rancher.os.far": "true"},
Bars: []string{},
}}
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
s2 := map[string]*StructSliceorMap{}
yaml.Unmarshal(d, &s2)
assert.Equal(t, s, s2)
}
func composeToCloudConfig(bytes []byte) ([]byte, error) {
compose := make(map[interface{}]interface{})
err := yaml.Unmarshal(bytes, &compose)
if err != nil {
return nil, err
}
return yaml.Marshal(map[interface{}]interface{}{
"rancher": map[interface{}]interface{}{
"services": compose,
},
})
}
func (matcher *MatchYAMLMatcher) prettyPrint(actual interface{}) (actualFormatted, expectedFormatted string, err error) {
actualString, aok := toString(actual)
expectedString, eok := toString(matcher.YAMLToMatch)
if !(aok && eok) {
return "", "", fmt.Errorf("MatchYAMLMatcher matcher requires a string or stringer. Got:\n%s", format.Object(actual, 1))
}
var adata interface{}
if err := candiedyaml.Unmarshal([]byte(actualString), &adata); err != nil {
return "", "", err
}
abuf, _ := candiedyaml.Marshal(adata)
var edata interface{}
if err := candiedyaml.Unmarshal([]byte(expectedString), &edata); err != nil {
return "", "", err
}
ebuf, _ := candiedyaml.Marshal(edata)
return string(abuf), string(ebuf), nil
}
func TestStringorsliceYaml(t *testing.T) {
str := `{foo: [bar, baz]}`
s := StructStringorslice{}
yaml.Unmarshal([]byte(str), &s)
assert.Equal(t, Stringorslice{"bar", "baz"}, s.Foo)
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
s2 := StructStringorslice{}
yaml.Unmarshal(d, &s2)
assert.Equal(t, Stringorslice{"bar", "baz"}, s2.Foo)
}
// Convert JSON to YAML.
func JSONToYAML(j []byte) ([]byte, error) {
// Convert the JSON to an object.
var jsonObj interface{}
// We are using yaml.Unmarshal here (instead of json.Unmarshal) because the
// Go JSON library doesn't try to pick the right number type (int, float,
// etc.) when unmarshling to interface{}, it just picks float64
// universally. go-yaml does go through the effort of picking the right
// number type, so we can preserve number type throughout this process.
err := yaml.Unmarshal(j, &jsonObj)
if err != nil {
return nil, err
}
// Marshal this object into YAML.
return yaml.Marshal(jsonObj)
}
func TestSliceOrMapYaml(t *testing.T) {
str := `{foos: [bar=baz, far=faz]}`
s := StructSliceorMap{}
yaml.Unmarshal([]byte(str), &s)
assert.Equal(t, SliceorMap{"bar": "baz", "far": "faz"}, s.Foos)
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
s2 := StructSliceorMap{}
yaml.Unmarshal(d, &s2)
assert.Equal(t, SliceorMap{"bar": "baz", "far": "faz"}, s2.Foos)
}
func TestMarshalUlimit(t *testing.T) {
ulimits := []struct {
ulimits *Ulimits
expected string
}{
{
ulimits: &Ulimits{
Elements: []Ulimit{
{
ulimitValues: ulimitValues{
Soft: 65535,
Hard: 65535,
},
Name: "nproc",
},
},
},
expected: `nproc: 65535
`,
},
{
ulimits: &Ulimits{
Elements: []Ulimit{
{
Name: "nofile",
ulimitValues: ulimitValues{
Soft: 20000,
Hard: 40000,
},
},
},
},
expected: `nofile:
soft: 20000
hard: 40000
`,
},
}
for _, ulimit := range ulimits {
bytes, err := yaml.Marshal(ulimit.ulimits)
assert.Nil(t, err)
assert.Equal(t, ulimit.expected, string(bytes), "should be equal")
}
}
func TestUnmarshalEmptyCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(sampleEmptyCommand), s)
assert.Nil(t, err)
assert.Nil(t, s.Command)
bytes, err := yaml.Marshal(s)
assert.Nil(t, err)
assert.Equal(t, "{}", strings.TrimSpace(string(bytes)))
s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)
assert.Nil(t, err)
assert.Nil(t, s2.Command)
}
func TestUnmarshalCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(sampleStructCommand), s)
assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s.Command)
assert.Nil(t, s.Entrypoint)
bytes, err := yaml.Marshal(s)
assert.Nil(t, err)
s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)
assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s2.Command)
assert.Nil(t, s2.Entrypoint)
}
func getArg(i int, value interface{}) (string, bool) {
debug.Debug("arg %d: %+v\n", i, value)
switch value.(type) {
case string:
return value.(string), true
case int64:
return strconv.FormatInt(value.(int64), 10), true
default:
if i == 0 || value == nil {
return "", false
}
yaml, err := candiedyaml.Marshal(node(value))
if err != nil {
log.Fatalln("error marshalling manifest:", err)
}
return "---\n" + string(yaml), true
}
}
func populateManifest(baseManifest string, concourseManifestInputs concourseManifestInputs) (string, error) {
var concourseManifest concourseManifest
err := candiedyaml.Unmarshal([]byte(baseManifest), &concourseManifest)
if err != nil {
return "", err
}
concourseManifest.DirectorUUID = concourseManifestInputs.boshDirectorUUID
concourseManifest.Stemcells[0]["version"] = concourseManifestInputs.stemcellVersion
for releaseIdx, release := range concourseManifest.Releases {
switch release["name"] {
case "concourse":
concourseManifest.Releases[releaseIdx]["version"] = concourseManifestInputs.concourseReleaseVersion
case "garden-runc":
concourseManifest.Releases[releaseIdx]["version"] = concourseManifestInputs.gardenReleaseVersion
}
}
for i, _ := range concourseManifest.InstanceGroups {
concourseManifest.InstanceGroups[i].VMType = "m3.medium"
switch concourseManifest.InstanceGroups[i].Name {
case "web":
concourseManifest.InstanceGroups[i].VMExtensions = []string{"lb"}
concourseManifest.InstanceGroups[i].Jobs[0].Properties.BasicAuthUsername = "******"
concourseManifest.InstanceGroups[i].Jobs[0].Properties.BasicAuthPassword = "******"
concourseManifest.InstanceGroups[i].Jobs[0].Properties.ExternalURL = concourseManifestInputs.webExternalURL
case "worker":
concourseManifest.InstanceGroups[i].VMExtensions = []string{"50GB_ephemeral_disk"}
case "db":
concourseManifest.InstanceGroups[i].PersistentDiskType = "1GB"
concourseManifest.InstanceGroups[i].Jobs[0].Properties.Databases[0].Role = "admin"
concourseManifest.InstanceGroups[i].Jobs[0].Properties.Databases[0].Password = "******"
}
}
finalConcourseManifestYAML, err := candiedyaml.Marshal(concourseManifest)
if err != nil {
return "", err
}
return string(finalConcourseManifestYAML), nil
}
func (c CloudConfigManager) Update(input CloudConfigInput, boshClient Client) error {
c.logger.Step("generating cloud config")
cloudConfig, err := c.cloudConfigGenerator.Generate(input)
if err != nil {
return err
}
yaml, err := candiedyaml.Marshal(cloudConfig)
if err != nil {
return err
}
c.logger.Step("applying cloud config")
if err := boshClient.UpdateCloudConfig(yaml); err != nil {
return err
}
return nil
}
// parseYAMLStream takes an encoded YAML stream and turns it into a slice of JSON-marshalable
// objects, one for each document in the stream.
func parseYAMLStream(in io.Reader) ([]interface{}, error) {
// Use candiedyaml because it's the only one that supports streams.
decoder := candiedyaml.NewDecoder(in)
var document interface{}
stream := []interface{}{}
for {
err := decoder.Decode(&document)
if err != nil {
if strings.Contains(err.Error(), "Expected document start at line") {
return stream, nil
}
return nil, err
}
// Now it's held in document but we have to do a bit of a dance to get it in a form that can
// be marshaled as JSON for our API response. The fundamental problem is that YAML is a
// superset of JSON in that it can represent non-string keys, and full IEEE floating point
// values (NaN etc). JSON only allows string keys and its definition of a number is based
// around a sequence of digits.
// Kubernetes does not make use of these features, as it uses YAML as just "pretty JSON".
// Consequently this does not affect Helm either. However, both candiedyaml and go-yaml
// return types that are too wide for JSON marshalling (i.e. map[interface{}]interface{}
// instead of map[string]interface{}), so we have to do some explicit conversion. Luckily,
// ghodss/yaml has code to help with this, since decoding from YAML to JSON-marshalable
// values is exactly the problem that it was designed to solve.
// 1) Marshal it back to YAML string.
yamlBytes, err := candiedyaml.Marshal(document)
if err != nil {
return nil, err
}
// 2) Use ghodss/yaml to unmarshal that string into JSON-compatible data structures.
var jsonObj interface{}
if err := yaml.Unmarshal(yamlBytes, &jsonObj); err != nil {
return nil, err
}
// Now it's suitable for embedding in an API response.
stream = append(stream, jsonObj)
}
}
func TestMaporsliceYaml(t *testing.T) {
str := `{foo: {bar: baz, far: 1}}`
s := StructMaporslice{}
yaml.Unmarshal([]byte(str), &s)
assert.Equal(t, 2, len(s.Foo))
assert.True(t, contains(s.Foo, "bar=baz"))
assert.True(t, contains(s.Foo, "far=1"))
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
s2 := StructMaporslice{}
yaml.Unmarshal(d, &s2)
assert.Equal(t, 2, len(s2.Foo))
assert.True(t, contains(s2.Foo, "bar=baz"))
assert.True(t, contains(s2.Foo, "far=1"))
}
func (cc *GceCloudConfig) getMergedUserData() ([]byte, error) {
var returnUserData []byte
userdata := make(map[string]interface{})
if cc.UserData != "" {
log.Infof("Found UserData Config")
err := yaml.Unmarshal([]byte(cc.UserData), &userdata)
if err != nil {
log.Errorf("Could not unmarshal data: %s", err)
return nil, err
}
}
var auth_keys []string
if _, exists := userdata["ssh_authorized_keys"]; exists {
udSshKeys := userdata["ssh_authorized_keys"].([]interface{})
log.Infof("userdata %s", udSshKeys)
for _, value := range udSshKeys {
auth_keys = append(auth_keys, value.(string))
}
}
if cc.NonUserDataSSHKeys != nil {
for _, value := range cc.NonUserDataSSHKeys {
auth_keys = append(auth_keys, value)
}
}
userdata["ssh_authorized_keys"] = auth_keys
yamlUserData, err := yaml.Marshal(&userdata)
if err != nil {
log.Errorf("Could not Marshal userdata: %s", err)
return nil, err
} else {
returnUserData = append([]byte("#cloud-config\n"), yamlUserData...)
}
return returnUserData, nil
}
func merge(templateFilePath string, stubFilePaths []string) {
templateFile, err := ioutil.ReadFile(templateFilePath)
if err != nil {
log.Fatalln("error reading template:", err)
}
templateYAML, err := yaml.Parse(templateFilePath, templateFile)
if err != nil {
log.Fatalln("error parsing template:", err)
}
stubs := []yaml.Node{}
for _, stubFilePath := range stubFilePaths {
stubFile, err := ioutil.ReadFile(stubFilePath)
if err != nil {
log.Fatalln("error reading stub:", err)
}
stubYAML, err := yaml.Parse(stubFilePath, stubFile)
if err != nil {
log.Fatalln("error parsing stub:", err)
}
stubs = append(stubs, stubYAML)
}
flowed, err := flow.Cascade(templateYAML, stubs...)
if err != nil {
log.Fatalln("error generating manifest:", err)
}
yaml, err := candiedyaml.Marshal(flowed)
if err != nil {
log.Fatalln("error marshalling manifest:", err)
}
fmt.Println(string(yaml))
}
func StageServices(cfg *config.CloudConfig, services ...string) error {
p, err := newProject("stage-services", cfg, nil, nil)
if err != nil {
return err
}
for _, service := range services {
bytes, err := network.LoadServiceResource(service, true, cfg)
if err != nil {
return fmt.Errorf("Failed to load %s : %v", service, err)
}
m := map[interface{}]interface{}{}
if err := yaml.Unmarshal(bytes, &m); err != nil {
return fmt.Errorf("Failed to parse YAML configuration: %s : %v", service, err)
}
bytes, err = yaml.Marshal(m)
if err != nil {
return fmt.Errorf("Failed to marshal YAML configuration: %s : %v", service, err)
}
err = p.Load(bytes)
if err != nil {
return fmt.Errorf("Failed to load %s : %v", service, err)
}
}
// Reduce service configurations to just image and labels
for _, serviceName := range p.ServiceConfigs.Keys() {
serviceConfig, _ := p.ServiceConfigs.Get(serviceName)
p.ServiceConfigs.Add(serviceName, &composeConfig.ServiceConfig{
Image: serviceConfig.Image,
Labels: serviceConfig.Labels,
})
}
return p.Pull(context.Background())
}
func StageServices(cfg *config.CloudConfig, services ...string) error {
p, err := newProject("stage-services", cfg, nil)
if err != nil {
return err
}
for _, service := range services {
bytes, err := LoadServiceResource(service, true, cfg)
if err != nil {
return fmt.Errorf("Failed to load %s : %v", service, err)
}
m := map[interface{}]interface{}{}
if err := yaml.Unmarshal(bytes, &m); err != nil {
return fmt.Errorf("Failed to parse YAML configuration: %s : %v", service, err)
}
bytes, err = yaml.Marshal(config.StringifyValues(m))
if err != nil {
fmt.Errorf("Failed to marshal YAML configuration: %s : %v", service, err)
}
err = p.Load(bytes)
if err != nil {
fmt.Errorf("Failed to load %s : %v", service, err)
}
}
// Reduce service configurations to just image and labels
for serviceName, serviceConfig := range p.Configs {
p.Configs[serviceName] = &project.ServiceConfig{
Image: serviceConfig.Image,
Labels: serviceConfig.Labels,
}
}
return p.Pull()
}