func GatherSource(s StackReadWriter, m map[string]interface{}) filepath.WalkFunc {
Debugf("gathering source")
return func(path string, info os.FileInfo, _ error) error {
if info.IsDir() {
return nil // descend
}
switch filepath.Ext(path) {
case ".html":
defaultMetadata := map[string]interface{}{
"source": path,
"target": TargetFileFor(path, filepath.Ext(path)),
"url": "/" + Relative(*targetDir, TargetFileFor(path, filepath.Ext(path))),
"sortkey": filepath.Base(path),
}
fileMetadata := map[string]interface{}{}
fileMetadataBuf, _ := splitMetadata(Read(path))
if len(fileMetadataBuf) > 0 {
fileMetadata = ParseJSON(fileMetadataBuf)
}
inheritedMetadata := s.Get(path)
metadata := mergemap.Merge(defaultMetadata, mergemap.Merge(inheritedMetadata, fileMetadata))
s.Add(path, metadata)
SplatInto(m, Relative(*sourceDir, path), metadata)
Debugf("%s gathered (%d element(s))", path, len(metadata))
case ".md":
defaultMetadata := map[string]interface{}{
"source": path,
"target": TargetFileFor(path, ".html"),
"url": "/" + Relative(*targetDir, TargetFileFor(path, ".html")),
"sortkey": filepath.Base(path),
}
if blogTuple, ok := NewBlogTuple(path, ".html"); ok {
baseDir := filepath.Join(*targetDir, Relative(*sourceDir, filepath.Dir(path)))
defaultMetadata["title"] = blogTuple.Title
defaultMetadata["date"] = blogTuple.DateString()
defaultMetadata["target"] = blogTuple.TargetFileFor(baseDir)
defaultMetadata["url"] = "/" + Relative(*targetDir, blogTuple.TargetFileFor(baseDir))
defaultMetadata["redirects"] = blogTuple.RedirectFromURLs(baseDir)
}
fileMetadata := map[string]interface{}{}
fileMetadataBuf, _ := splitMetadata(Read(path))
if len(fileMetadataBuf) > 0 {
fileMetadata = ParseJSON(fileMetadataBuf)
}
inheritedMetadata := s.Get(path)
metadata := mergemap.Merge(defaultMetadata, mergemap.Merge(inheritedMetadata, fileMetadata))
s.Add(path, metadata)
SplatInto(m, Relative(*sourceDir, path), metadata)
Debugf("%s gathered (%d element(s))", path, len(metadata))
}
return nil
}
}
func overrideWithJsonIfNeeded(overrideEnvVarName string, attributes map[string]interface{}) map[string]interface{} {
if overrideEnvVarName != "" {
if envjson := os.Getenv(overrideEnvVarName); envjson != "" {
if len(envjson) > 8 && envjson[0:7] == "base64," {
logs.WithField("EnvVar", overrideEnvVarName).Debug("Environment variable is base64 encoded")
b64EnvJson := envjson[7:len(envjson)]
envjsonBase64Decoded, err := base64.StdEncoding.DecodeString(b64EnvJson)
if err != nil {
logs.WithE(err).WithField("base64", b64EnvJson).Fatal("Failed to base64 decode")
}
envjson = string(envjsonBase64Decoded)
}
logs.WithField("content", envjson).Debug("Override var content")
var envattr map[string]interface{}
err := json.Unmarshal([]byte(envjson), &envattr)
if err != nil {
logs.WithE(err).
WithField("varName", overrideEnvVarName).
WithField("content", envjson).
Fatal("Invalid format for environment override content")
}
attributes = mergemap.Merge(attributes, envattr)
}
}
return attributes
}
func MergeAttributesFilesForMap(omap map[string]interface{}, files []string) map[string]interface{} {
newMap := make(map[string]interface{})
newMap["default"] = omap
// loop over attributes files
// merge override files to default files
for _, file := range files {
var data interface{}
yml, err := ioutil.ReadFile(file)
if err != nil {
panic(err)
}
// yaml to data
err = yaml.Unmarshal(yml, &data)
if err != nil {
panic(err)
}
data, err = transform(data)
if err != nil {
panic(err)
}
// data to map
json := data.(map[string]interface{})
omap = mergemap.Merge(newMap, json)
}
return ProcessOverride(newMap)
}
func (c *Config) mergeInJSON(data []byte) error {
// This is all HORRIBLE
// but it seems about the only reasonable way to properly merge
// the json schemas such that json objects are recursively merged.
// Steps: convert c to json and then back to a go type, so that
// it is a map[string]interface{} and not a Config type. Get
// the json bytes also into a map[string]interface{} so that
// the two map[string]interface{} objects can be merged. Finally
// convert the merge result to json again so that it can be
// marshaled back into the original Config type... Yuck!
m1 := new(map[string]interface{})
m2 := new(map[string]interface{})
m1bytes, err := json.Marshal(c)
if err != nil {
return err
}
err = json.Unmarshal(m1bytes, m1)
if err != nil {
return err
}
err = json.Unmarshal(data, m2)
if err != nil {
return err
}
merged := mergemap.Merge(*m1, *m2)
mergedBytes, err := json.Marshal(merged)
if err != nil {
return err
}
return json.Unmarshal(mergedBytes, c)
}
func (this *PackerRestAPI) Post(w *rest.ResponseWriter, r *rest.Request) {
userId := r.PathParam("user")
docId := r.PathParam("docId")
// first, locate the existing document
(*this.storage).Open(userId)
doc, err := this.storage.GetDocument(userId, docId)
if err != nil {
rest.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// second, decode the post payload to merge 'into' to prior document
var snippet interface{}
err = r.DecodeJsonPayload(&snippet)
if err != nil {
rest.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// merge the documents
merged := mergemap.Merge(doc, snippet.(map[string]interface{}))
//replace the document in the map
err = this.storage.UpdateDocument(userId, docId, merged)
if err == nil {
w.WriteJson(&IdResponse{docId})
} else {
rest.Error(w, err.Error(), http.StatusInternalServerError)
}
}
// Add merges the given metadata into the Stack element represented by path.
// If no such element exists, Add will create it.
func (s *Stack) Add(path string, m map[string]interface{}) {
key := filepath.Join(SplitPath(path)...)
existing, ok := s.m[key]
if !ok {
existing = map[string]interface{}{}
}
s.m[key] = mergemap.Merge(existing, m)
}
// SplatInto splits the `path` on filepath.Separator, and merges the passed
// `metadata` into the map `m` under the resulting key.
//
// As an example, if path="foo/bar/baz", SplatInto is semantically the same as
// `m = merge(m[foo][bar][baz], metadata)`.
func SplatInto(m map[string]interface{}, path string, metadata map[string]interface{}) {
m0 := m
for _, level := range SplitPath(path) {
if _, ok := m0[level]; !ok {
m0[level] = map[string]interface{}{}
}
m0 = m0[level].(map[string]interface{})
}
m0 = mergemap.Merge(m0, metadata)
}
func TestMergeInto(t *testing.T) {
m := map[string]interface{}{}
m1 := mergemap.Merge(m, map[string]interface{}{"a": "b"})
if m1["a"] != "b" {
t.Fatal("m1[a] != b")
}
m2 := mergemap.Merge(m1, map[string]interface{}{"a": "c"})
if m2["a"] != "c" {
t.Fatal("m2[a] != c")
}
m3 := mergemap.Merge(m2, map[string]interface{}{"b": "d"})
if m3["a"] != "c" {
t.Fatal("m3[a] != c")
}
if m3["b"] != "d" {
t.Fatal("m3[b] != d")
}
}
func overrideWithJsonIfNeeded(overrideEnvVarName string, attributes map[string]interface{}) map[string]interface{} {
if overrideEnvVarName != "" {
if envjson := os.Getenv(overrideEnvVarName); envjson != "" {
var envattr map[string]interface{}
err := json.Unmarshal([]byte(envjson), &envattr)
if err != nil {
panic(err)
}
attributes = mergemap.Merge(attributes, envattr)
}
}
return attributes
}
func ProcessOverride(omap map[string]interface{}) map[string]interface{} {
// merge override to default inside the file
_, okd := omap["default"]
if okd == false {
omap["default"] = make(map[string]interface{}) //init if default doesn't exist
}
_, oko := omap["override"]
if oko == true {
omap = mergemap.Merge(omap["default"].(map[string]interface{}), omap["override"].(map[string]interface{}))
} else {
omap = omap["default"].(map[string]interface{})
}
return omap
}
func overrideWithJsonIfNeeded(overrideEnvVarName string, attributes map[string]interface{}) map[string]interface{} {
if overrideEnvVarName != "" {
if envjson := os.Getenv(overrideEnvVarName); envjson != "" {
logs.WithField("content", envjson).Debug("Override var content")
var envattr map[string]interface{}
err := json.Unmarshal([]byte(envjson), &envattr)
if err != nil {
logs.WithE(err).
WithField("varName", overrideEnvVarName).
WithField("content", envjson).
Fatal("Invalid format for environment override content")
}
attributes = mergemap.Merge(attributes, envattr)
}
}
return attributes
}
// Get returns the aggregate metadata visible from the given path.
func (s *Stack) Get(path string) map[string]interface{} {
list := SplitPath(path)
if len(list) <= 0 {
return map[string]interface{}{}
}
// A weird bit of trickery. We add global metadata with a path of "" (empty
// string) under the expectation that Get will return them for every input
// path. So, we prepend "" to every lookup request. That means 'i' is off-
// by-one, so we can use it directly against the list slice.
m := map[string]interface{}{}
for i, _ := range append([]string{""}, list...) {
key := filepath.Join(list[:i]...)
if m0, ok := s.m[key]; ok {
m = mergemap.Merge(m, m0)
}
}
return m
}
func Merge(envName string, files []string) []byte { // inputDir string,
// "out map" to store merged yamls
omap := MergeAttributesFiles(files)
envjson := os.Getenv(envName)
if envjson != "" {
var envattr map[string]interface{}
err := json.Unmarshal([]byte(envjson), &envattr)
if err != nil {
panic(err)
}
omap = mergemap.Merge(omap, envattr)
}
// map to json
out, err := json.Marshal(omap)
if err != nil {
panic(err)
}
return out
}
func (s Service) writeUnit(i int, node map[string]interface{}, tmpl *Templating, acis string) {
if node[spec.NODE_HOSTNAME].(string) == "" {
s.log.WithField("index", i).Error("hostname is mandatory in node informations")
}
s.log.Debug("Processing node :" + node[spec.NODE_HOSTNAME].(string))
unitName := s.UnitName(node[spec.NODE_HOSTNAME].(string))
data := make(map[string]interface{})
data["node"] = node
data["node"].(map[string]interface{})["acis"] = acis
data["attribute"] = utils.CopyMap(s.attributes)
if data["node"].(map[string]interface{})["attributes"] != nil {
source := utils.CopyMapInterface(data["node"].(map[string]interface{})["attributes"].(map[interface{}]interface{}))
data["attribute"] = mergemap.Merge(data["attribute"].(map[string]interface{}), source.(map[string]interface{}))
}
out, err := json.Marshal(data["attribute"])
if err != nil {
s.log.WithError(err).Panic("Cannot marshall attributes")
}
data["attributes"] = strings.Replace(string(out), "\\\"", "\\\\\\\"", -1)
var b bytes.Buffer
err = tmpl.Execute(&b, data)
if err != nil {
s.log.Error("Failed to run templating for unit "+unitName, err)
}
ok, err := utils.Exists(s.path + "/units")
if !ok || err != nil {
os.Mkdir(s.path+"/units", 0755)
}
err = ioutil.WriteFile(s.path+"/units"+"/"+unitName, b.Bytes(), 0644)
if err != nil {
s.log.WithError(err).WithField("path", s.path+"/units"+"/"+unitName).Error("Cannot writer unit")
}
}
func main() {
var global string
app := cli.NewApp()
app.Name = "rendr"
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "global, g",
Value: "",
Usage: "global json values to be applied to all items in data.json",
Destination: &global,
EnvVar: "RENDR_GLOBAL",
},
}
app.Usage = `
Rendr a template with a list of datasets contained in a json file and output to files
rendr data.json template.tmpl "result.{{ .PossibleDataKey }}.out"
`
app.EnableBashCompletion = true
app.Action = func(c *cli.Context) {
dataFile, tmplFile, outFile := resolveArgs(c.Args())
js, err := ioutil.ReadFile(dataFile)
if err != nil {
fmt.Printf("Unable to read data file '%s' with error '%s'\n", dataFile, err.Error())
return
}
tmpl, err := template.ParseFiles(tmplFile)
if err != nil {
fmt.Printf("Unable to read template file '%s' with error '%s'\n", tmplFile, err.Error())
return
}
dg := make(map[string]interface{})
if len(global) > 0 {
err = json.Unmarshal([]byte(global), &dg)
if err != nil {
fmt.Printf("Unable to unmarshal json with error '%s'\n", err.Error())
return
}
jstmpl, err := template.New("jstmpl").Parse(string(js))
if err != nil {
fmt.Printf("Unable to parse '%s' into a template with error '%s'\n", dataFile, err.Error())
return
}
jsb := bytes.Buffer{}
err = jstmpl.Execute(&jsb, dg)
if err != nil {
fmt.Printf("Unable to execute template with error '%s'\n", err.Error())
}
js = jsb.Bytes()
}
var dl []map[string]interface{}
err = json.Unmarshal(js, &dl)
if err != nil {
fmt.Printf("Unable to unmarshal json with error '%s'\n", err.Error())
return
}
outFileTmpl, err := template.New("outFileTmpl").Parse(outFile)
if err != nil {
fmt.Printf("Unable to parse outfile name as template with error '%s'\n", err.Error())
return
}
fwritten := map[string]bool{}
for _, d := range dl {
if outFile != "" {
d = mergemap.Merge(dg, d)
ofb := bytes.Buffer{}
err := outFileTmpl.Execute(&ofb, d)
if err != nil {
fmt.Printf("Unable to resolve outfile name with error '%s'\n", err.Error())
return
}
fn := ofb.String()
flag := os.O_WRONLY | os.O_CREATE | os.O_APPEND
if !fwritten[fn] {
flag = flag | os.O_TRUNC
}
dir := path.Dir(fn)
err = os.MkdirAll(dir, 0644)
if err != nil {
fmt.Printf("Unable to create path to file '%s' with error '%s'\n", fn, err.Error())
return
}
fp, err := os.OpenFile(fn, flag, 0666)
if err != nil {
fmt.Printf("Unable to write to file '%s' with error '%s'\n", fn, err.Error())
return
}
fwritten[fn] = true
render(tmpl, d, fp)
fp.Close()
} else {
render(tmpl, d, os.Stdout)
}
}
}
app.Run(os.Args)
}
func (u Unit) GenerateAttributes() map[string]interface{} {
data := utils.CopyMap(u.Service.GetAttributes())
data = mergemap.Merge(data, u.Service.NodeAttributes(u.hostname))
return data
}
func Transform(s StackReader) filepath.WalkFunc {
Debugf("transforming")
return func(path string, info os.FileInfo, _ error) error {
if info.IsDir() {
Debugf("descending into %s", path)
return nil // descend
}
Debugf("Transforming %s", path)
switch filepath.Ext(path) {
case ".json":
Debugf("%s ignored for transformation", path)
case ".html":
// read
_, contentBuf := splitMetadata(Read(path))
// render
outputBuf := RenderTemplate(path, contentBuf, s.Get(path))
// write
dst := TargetFileFor(path, filepath.Ext(path))
Write(dst, outputBuf)
Debugf("%s transformed to %s", path, dst)
case ".md":
// read
_, contentBuf := splitMetadata(Read(path))
// render
metadata := mergemap.Merge(s.Get(path), map[string]interface{}{
"content": template.HTML(RenderMarkdown(contentBuf)),
})
templatePath, templateBuf := Template(s, path)
outputBuf := RenderTemplate(templatePath, templateBuf, metadata)
// write file
dst, _ := metadata["target"].(string)
Write(dst, outputBuf)
// write redirects
if redirectsInterface, ok := metadata["redirects"]; ok {
redirectToUrl, _ := metadata["url"].(string)
redirectFromUrls, _ := redirectsInterface.([]string)
for _, redirectFromUrl := range redirectFromUrls {
redirectFromFile := filepath.Join(*targetDir, redirectFromUrl)
Write(redirectFromFile, RedirectTo(redirectToUrl))
}
}
// done
Debugf("%s transformed to %s", path, dst)
case ".source", ".template":
Debugf("%s ignored for transformation", path)
default:
dst := TargetFileFor(path, filepath.Ext(path))
Copy(dst, path)
Debugf("%s transformed to %s verbatim", path, dst)
}
return nil
}
}
