func parseStartCommand(cloudfoundryDropletDir string) (startCommand []string) {
stagingInfoFile, err := os.Open(cloudfoundryDropletDir + "/staging_info.yml")
if err == nil {
stagingInfo := new(StagingInfoYml)
decoder := candiedyaml.NewDecoder(stagingInfoFile)
err = decoder.Decode(stagingInfo)
if err != nil {
log.Fatalf("Failed to decode document: %s", err)
}
startCommand = strings.Split(stagingInfo.StartCommand, " ")
if startCommand[0] != "" {
return
}
procfileFile, err := os.Open(cloudfoundryDropletDir + "/app/Procfile")
if err == nil {
procfileInfo := new(ProcfileYml)
decoder := candiedyaml.NewDecoder(procfileFile)
err = decoder.Decode(procfileInfo)
if err != nil {
log.Fatalf("Failed to decode document: %s", err)
}
startCommand = strings.Split(procfileInfo.Web, " ")
return
}
}
log.Fatal("Unable to find staging_info.yml")
return
}
func generate(source io.Reader) {
var spec builder.DeploymentSpec
var err error
err = candiedyaml.NewDecoder(source).Decode(&spec)
if err != nil {
panic(err)
}
former := aws.New(spec.Description)
region := os.Getenv("AWS_DEFAULT_REGION")
if region == "" {
region = "us-east-1"
}
builder := builder.New(spec, region)
err = builder.Build(former)
if err != nil {
panic(err)
}
err = json.NewEncoder(os.Stdout).Encode(former.Template)
if err != nil {
fatal(err)
}
}
func (p yamlParser) ParseFromIoReader(r io.Reader) ([]models.Plugin, error) {
document := new(interface{})
decoder := candiedyaml.NewDecoder(r)
err := decoder.Decode(document)
if err != nil {
p.logger.Write([]byte("Failed to decode document:" + err.Error()))
return []models.Plugin{}, err
}
output, _ := expandProperties(*document)
plugins := p.pluginsModel.PopulateModel(output)
mirrorsInterface := output.(map[interface{}]interface{})["mirrors"]
if mirrorsInterface == nil {
return plugins, nil
}
for _, mirror := range mirrorsInterface.([]interface{}) {
mirrorPlugins, err := p.ParseFromMirror(mirror.(string))
if err != nil {
return []models.Plugin{}, err
}
plugins = p.mergePlugins(plugins, mirrorPlugins)
}
return plugins, nil
}
func Load(restoreConfigPath string) (Config, error) {
file, err := os.Open(restoreConfigPath)
if err != nil {
return Config{}, err
}
var config Config
if err := candiedyaml.NewDecoder(file).Decode(&config); err != nil {
return Config{}, err
}
return config, nil
}
func Load(path string) (*Config, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
config := &Config{}
if err := candiedyaml.NewDecoder(file).Decode(config); err != nil {
return nil, err
}
return config, nil
}
func ParseConfig(path string) (Config, error) {
file, err := os.Open(path)
if err != nil {
return Config{}, err
}
var config Config
if err := candiedyaml.NewDecoder(file).Decode(&config); err != nil {
return Config{}, err
}
return config, ValidateConfig(config.RedisConfiguration)
}
func parseManifest(file io.Reader) (yamlMap generic.Map, err error) {
decoder := candiedyaml.NewDecoder(file)
yamlMap = generic.NewMap()
err = decoder.Decode(yamlMap)
if err != nil {
return
}
if !generic.IsMappable(yamlMap) {
err = errors.New(T("Invalid manifest. Expected a map"))
return
}
return
}
func ParseConfig(reader io.Reader) (*Config, error) {
var config Config
if err := candiedyaml.NewDecoder(reader).Decode(&config); err != nil {
return nil, err
}
err := ValidateConfig(&config)
if err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(config.Rabbitmq.Policy.EncodedDefinition), &config.Rabbitmq.Policy.Definition); err != nil {
return nil, err
}
return &config, nil
}
func ConvertIgnoreOmitEmpty(from, to interface{}) error {
var buffer bytes.Buffer
encoder := yaml.NewEncoder(&buffer)
encoder.IgnoreOmitEmpty = true
if err := encoder.Encode(from); err != nil {
return err
}
decoder := yaml.NewDecoder(&buffer)
if err := decoder.Decode(to); err != nil {
return err
}
return nil
}
func (runner *Runner) release(buildpackDir string) (Release, error) {
output := new(bytes.Buffer)
err := runner.run(exec.Command(path.Join(buildpackDir, "bin", "release"), runner.config.AppDir()), output)
if err != nil {
return Release{}, err
}
decoder := candiedyaml.NewDecoder(output)
var parsedRelease Release
err = decoder.Decode(&parsedRelease)
if err != nil {
return Release{}, newDescriptiveError(err, "buildpack's release output invalid")
}
return parsedRelease, nil
}
func startCommandFromStagingInfo(stagingInfoPath string) (string, error) {
stagingInfoFile, err := os.Open(stagingInfoPath)
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
defer stagingInfoFile.Close()
info := stagingInfo{}
err = candiedyaml.NewDecoder(stagingInfoFile).Decode(&info)
if err != nil {
return "", errors.New("invalid YAML")
}
return info.StartCommand, nil
}
// parseYAMLStream takes an encoded YAML stream and turns it into a slice of JSON-marshalable
// objects, one for each document in the stream.
func parseYAMLStream(in io.Reader) ([]interface{}, error) {
// Use candiedyaml because it's the only one that supports streams.
decoder := candiedyaml.NewDecoder(in)
var document interface{}
stream := []interface{}{}
for {
err := decoder.Decode(&document)
if err != nil {
if strings.Contains(err.Error(), "Expected document start at line") {
return stream, nil
}
return nil, err
}
// Now it's held in document but we have to do a bit of a dance to get it in a form that can
// be marshaled as JSON for our API response. The fundamental problem is that YAML is a
// superset of JSON in that it can represent non-string keys, and full IEEE floating point
// values (NaN etc). JSON only allows string keys and its definition of a number is based
// around a sequence of digits.
// Kubernetes does not make use of these features, as it uses YAML as just "pretty JSON".
// Consequently this does not affect Helm either. However, both candiedyaml and go-yaml
// return types that are too wide for JSON marshalling (i.e. map[interface{}]interface{}
// instead of map[string]interface{}), so we have to do some explicit conversion. Luckily,
// ghodss/yaml has code to help with this, since decoding from YAML to JSON-marshalable
// values is exactly the problem that it was designed to solve.
// 1) Marshal it back to YAML string.
yamlBytes, err := candiedyaml.Marshal(document)
if err != nil {
return nil, err
}
// 2) Use ghodss/yaml to unmarshal that string into JSON-compatible data structures.
var jsonObj interface{}
if err := yaml.Unmarshal(yamlBytes, &jsonObj); err != nil {
return nil, err
}
// Now it's suitable for embedding in an API response.
stream = append(stream, jsonObj)
}
}
func (bosh *Bosh) GenerateAndSetDeploymentManifest(manifest interface{}, manifestGenerateScripts string, stubs ...string) {
cmd := exec.Command(manifestGenerateScripts, stubs...)
session, err := Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).ToNot(HaveOccurred())
Eventually(session, 10*time.Second).Should(Exit(0))
tmpFile, err := ioutil.TempFile(os.TempDir(), "")
Expect(err).ToNot(HaveOccurred())
_, err = tmpFile.Write(session.Out.Contents())
Expect(err).ToNot(HaveOccurred())
tmpFile.Close()
Expect(bosh.Command("deployment", tmpFile.Name())).To(Exit(0))
tmpFile, err = os.Open(tmpFile.Name())
Expect(err).ToNot(HaveOccurred())
decoder := candiedyaml.NewDecoder(tmpFile)
err = decoder.Decode(manifest)
Expect(err).ToNot(HaveOccurred())
}
func (p yamlParser) Parse() ([]models.Plugin, error) {
file, err := os.Open(p.filePath)
if err != nil {
p.logger.Write([]byte("File does not exist:" + err.Error()))
return []models.Plugin{}, err
}
document := new(interface{})
decoder := candiedyaml.NewDecoder(file)
err = decoder.Decode(document)
if err != nil {
p.logger.Write([]byte("Failed to decode document:" + err.Error()))
return []models.Plugin{}, err
}
output, _ := expandProperties(*document)
plugins := p.pluginsModel.PopulateModel(output)
return plugins, nil
}
func (runner *Runner) detectStartCommandFromProcfile() (string, error) {
procFile, err := os.Open(filepath.Join(runner.config.BuildDir(), "Procfile"))
if err != nil {
if os.IsNotExist(err) {
// Procfiles are optional
return "", nil
}
return "", err
}
defer procFile.Close()
processes := map[string]string{}
err = candiedyaml.NewDecoder(procFile).Decode(&processes)
if err != nil {
// clobber candiedyaml's super low-level error
return "", errors.New("invalid YAML")
}
return processes["web"], nil
}
func (runner *Runner) release(buildpackDir string, webStartCommand string) (Release, error) {
output := new(bytes.Buffer)
err := runner.run(exec.Command(path.Join(buildpackDir, "bin", "release"), runner.config.BuildDir()), output)
if err != nil {
return Release{}, err
}
decoder := candiedyaml.NewDecoder(output)
parsedRelease := Release{}
err = decoder.Decode(&parsedRelease)
if err != nil {
return Release{}, newDescriptiveError(err, "buildpack's release output invalid")
}
procfileContainsWebStartCommand := webStartCommand != ""
if procfileContainsWebStartCommand {
parsedRelease.DefaultProcessTypes.Web = webStartCommand
}
return parsedRelease, nil
}
// ReadCompose for V2
func ReadComposeV2(fn string) *ComposeV2Yaml {
file, err := os.Open(fn)
if err != nil {
log.Println("File does not exist:", err)
return nil
}
defer file.Close()
document := new(interface{})
decoder := candiedyaml.NewDecoder(file)
err = decoder.Decode(document)
if err != nil {
log.Println("Couldn't decode yaml:", err)
return nil
}
c2 := new(ComposeV2Yaml)
cs := make(ComposeServices)
switch comp := (*document).(type) {
case map[interface{}]interface{}:
for label, section := range comp {
switch label {
case "version":
switch version := section.(type) {
case string:
c2.Version = version
default:
log.Println("Version not a string: %v", section)
return nil
}
case "services":
//fmt.Printf("Got services %v\n", section)
switch services := section.(type) {
case map[interface{}]interface{}:
for name, options := range services {
switch optionmap := options.(type) {
case map[interface{}]interface{}:
//fmt.Printf("Service %v:\n", name)
ca := new(ComposeAttributes)
for option, values := range optionmap {
ok := true
switch value := values.(type) {
case []interface{}:
//fmt.Printf(" slice %v:%v\n", option, value)
sv := make([]string, len(value))
for i, v := range value {
sv[i], ok = v.(string)
if !ok {
log.Printf("Can't convert %v:%v to string\n", option, value)
}
}
switch option {
case "volumes":
ca.Volumes = sv
case "ports":
ca.Ports = sv
case "links":
ca.Links = sv
case "networks":
ca.Networks = sv
default:
log.Printf("option ignored %v:%v\n", option, value)
}
case string:
//fmt.Printf(" string %v:%v\n", option, value)
switch option {
case "build":
ca.Build = value
case "image":
ca.Image = value
default:
log.Printf("option ignored %v:%v\n", option, value)
}
default:
log.Printf(" not a string or slice %v:%v\n", option, value)
}
}
cs[name.(string)] = *ca
default:
log.Printf("Couldn't find options in %v", optionmap)
}
}
default:
log.Println("Couldn't find services in %v", services)
}
c2.Services = cs
case "networks":
switch networks := section.(type) {
case map[interface{}]interface{}:
c2.Networks = make(map[string]interface{})
for name, options := range networks {
//fmt.Printf("network %v:%v\n", name, options)
s, ok := name.(string)
if ok {
c2.Networks[s] = options
} else {
log.Printf("Can't convert %v:%v to string\n", name, options)
}
}
}
case "volumes":
switch volumes := section.(type) {
case map[interface{}]interface{}:
c2.Volumes = make(map[string]interface{})
for name, options := range volumes {
//fmt.Printf("volume %v:%v\n", name, options)
s, ok := name.(string)
if ok {
c2.Volumes[s] = options
} else {
log.Printf("Can't convert %v:%v to string\n", name, options)
}
}
}
default:
log.Printf("No matching section: %v\n", label)
}
}
default:
log.Println("Couldn't find sections in compose v2 file")
}
c2.Services = cs
return c2
}
}
//Assert the droplet has the right files in it
Expect(dropletContents).To(HaveKey("./"))
Expect(dropletContents).To(HaveKey("./staging_info.yml"))
Expect(dropletContents).To(HaveKey("./logs/"))
Expect(dropletContents).To(HaveKey("./tmp/"))
Expect(dropletContents).To(HaveKey("./app/"))
Expect(dropletContents).To(HaveKey("./app/my-app"))
Expect(dropletContents).To(HaveKey("./app/compiled"))
//Assert the files contain the right content
Expect(string(dropletContents["./app/my-app"])).To(Equal("scooby-doo"))
//In particular, staging_info.yml should have the correct detected_buildpack and start_command
yamlDecoder := candiedyaml.NewDecoder(bytes.NewReader(dropletContents["./staging_info.yml"]))
stagingInfo := map[string]string{}
err = yamlDecoder.Decode(&stagingInfo)
Expect(err).NotTo(HaveOccurred())
Expect(stagingInfo["detected_buildpack"]).To(Equal("My Buildpack"))
Expect(stagingInfo["start_command"]).To(Equal("the-start-command"))
//Assert nothing else crept into the droplet
Expect(dropletContents).To(HaveLen(7))
})
})
}
stageWith("test-buildpack", "zip-buildpack", buildpack_zip)
stageWith(cc_messages.CUSTOM_BUILDPACK, "custom-zip-buildpack", buildpack_zip)
// test based on https://github.com/b00giZm/docker-compose-nodejs-examples/blob/master/05-nginx-express-redis-nodemon/docker-compose.yml
func TestGraph(t *testing.T) {
testyaml := `
web:
build: ./app
volumes:
- "app:/src/app"
expose:
- "3000"
links:
- "db:redis"
command: nodemon -L app/bin/www
nginx:
restart: always
build: ./nginx/
ports:
- "80:80"
volumes:
- /www/public
volumes_from:
- web
links:
- web:web
db:
image: redis
`
archaius.Conf.Arch = "test"
//archaius.Conf.GraphmlFile = ""
//archaius.Conf.GraphjsonFile = ""
archaius.Conf.RunDuration = 2 * time.Second
archaius.Conf.Dunbar = 50
archaius.Conf.Population = 50
//archaius.Conf.Msglog = false
archaius.Conf.Regions = 1
//archaius.Conf.Collect = false
//archaius.Conf.StopStep = 0
archaius.Conf.EurekaPoll = "1s"
fmt.Println("\nTesting Parser from Docker Compose V1 string")
try(testyaml)
fmt.Println("\nTesting file conversion from Docker Compose V1 compose_yaml/test.yml")
c := ReadCompose("compose_yaml/test.yml")
fmt.Println(c)
fmt.Println("\nTesting Docker Compose V2 format input from compose_yaml/testV2.yml")
file, err := os.Open("compose_yaml/testV2.yml")
if err != nil {
println("File does not exist:", err.Error())
os.Exit(1)
}
defer file.Close()
document := new(interface{})
// too hard to parse V2 yaml with decoder, so use candiedyaml to walk the structure
decoder := candiedyaml.NewDecoder(file)
err = decoder.Decode(document)
if err != nil {
log.Fatal(err)
}
println("parsed yml:")
cs := make(ComposeServices)
switch comp := (*document).(type) {
case map[interface{}]interface{}:
for label, section := range comp {
switch label {
case "version":
switch version := section.(type) {
case string:
fmt.Printf("Got version %v\n", version)
default:
fmt.Printf("Version not a string %v\n", section)
}
case "services":
//fmt.Printf("Got services %v\n", section)
switch services := section.(type) {
case map[interface{}]interface{}:
for name, options := range services {
switch optionmap := options.(type) {
case map[interface{}]interface{}:
fmt.Printf("Service %v:\n", name)
ca := new(ComposeAttributes)
for option, values := range optionmap {
ok := true
switch value := values.(type) {
case []interface{}:
fmt.Printf(" slice %v:%v\n", option, value)
sv := make([]string, len(value))
for i, v := range value {
sv[i], ok = v.(string)
if !ok {
fmt.Printf("not ok %v:%v\n", option, value)
}
}
switch option {
case "volumes":
ca.Volumes = sv
case "ports":
ca.Ports = sv
case "links":
ca.Links = sv
case "networks":
ca.Networks = sv
default:
fmt.Printf("option ignored %v:%v\n", option, value)
}
case string:
fmt.Printf(" string %v:%v\n", option, value)
switch option {
case "build":
ca.Build = value
case "image":
ca.Image = value
default:
fmt.Printf("option ignored %v:%v\n", option, value)
}
default:
fmt.Printf(" no match %v:%v\n", option, value)
}
}
cs[name.(string)] = *ca
default:
fmt.Printf("Couldn't find services in %v", services)
}
}
default:
fmt.Println("Couldn't find services")
}
for k, v := range cs {
fmt.Printf("Service %v:%v\n", k, v)
}
case "networks":
switch networks := section.(type) {
case map[interface{}]interface{}:
for name, options := range networks {
fmt.Printf("network %v:%v\n", name, options)
}
}
case "volumes":
switch volumes := section.(type) {
case map[interface{}]interface{}:
for name, options := range volumes {
fmt.Printf("volume %v:%v\n", name, options)
}
}
default:
fmt.Printf("No matching section: %v\n", label)
}
}
default:
fmt.Println("Couldn't find sections in compose v2 file")
}
fmt.Println("\nTesting ReadComposeV2(compose_yaml/testV2.yml)")
c2 := ReadComposeV2("compose_yaml/testV2.yml")
fmt.Println(*c2)
ComposeArch("composeTestResult", c2)
}
func main() {
boshServerUrl := flag.String("boshUrl", "", "Bosh URL (https://admin:[email protected]:25555)")
outputDir := flag.String("outputDir", "", "Output directory (/tmp/scripts)")
windowsUsername := flag.String("windowsUsername", "", "Windows username")
windowsPassword := flag.String("windowsPassword", "", "Windows password")
machineIp := flag.String("machineIp", "", "(optional) IP address of this cell")
flag.Parse()
if *boshServerUrl == "" || *outputDir == "" {
fmt.Fprintf(os.Stderr, "Usage of generate:\n")
flag.PrintDefaults()
os.Exit(1)
}
_, err := os.Stat(*outputDir)
if err != nil {
if os.IsNotExist(err) {
os.MkdirAll(*outputDir, 0755)
}
}
validateCredentials(*windowsUsername, *windowsPassword)
escapeWindowsPassword(windowsPassword)
response := NewBoshRequest(*boshServerUrl + "/deployments")
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(response.Body)
if err != nil {
fmt.Printf("Could not read response from BOSH director.")
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "Unexpected BOSH director response: %v, %v", response.StatusCode, buf.String())
os.Exit(1)
}
deployments := []models.IndexDeployment{}
json.NewDecoder(response.Body).Decode(&deployments)
idx := GetDiegoDeployment(deployments)
if idx == -1 {
fmt.Fprintf(os.Stderr, "BOSH Director does not have exactly one deployment containing a cf and diego release.")
os.Exit(1)
}
response = NewBoshRequest(*boshServerUrl + "/deployments/" + deployments[idx].Name)
defer response.Body.Close()
deployment := models.ShowDeployment{}
json.NewDecoder(response.Body).Decode(&deployment)
buf := bytes.NewBufferString(deployment.Manifest)
var manifest models.Manifest
decoder := candiedyaml.NewDecoder(buf)
err = decoder.Decode(&manifest)
if err != nil {
FailOnError(err)
}
args := models.InstallerArguments{
Username: *windowsUsername,
Password: *windowsPassword,
}
fillEtcdCluster(&args, manifest)
fillSharedSecret(&args, manifest)
fillMetronAgent(&args, manifest, *outputDir)
fillSyslog(&args, manifest)
fillConsul(&args, manifest, *outputDir)
fillMachineIp(&args, manifest, *machineIp)
fillBBS(&args, manifest, *outputDir)
generateInstallScript(*outputDir, args)
}