func (m *Machine) cmdLine() []string {
result := []string{"--driver", m.driver}
keys := []string{}
for k := range m.options {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := m.options[k]
switch val := v.(type) {
case string:
result = append(result, "--"+k)
val = os.Expand(val, func(key string) string { return m.expand(key) })
result = append(result, val)
case map[interface{}]interface{}:
keys := []string{}
for kk := range val {
keys = append(keys, kk.(string))
}
sort.Strings(keys)
for _, kk := range keys {
vv := val[kk]
result = append(result, "--"+k)
evv := os.Expand(vv.(string), func(key string) string { return m.expand(key) })
result = append(result, kk+"="+evv)
}
case bool:
if val {
result = append(result, "--"+k)
}
}
}
result = append(result, m.name)
return result
}
func (s *ShellExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) error {
if globalConfig != nil {
s.Shell.User = globalConfig.User
}
// expand environment variables to have current directory
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("Getwd: %v", err)
}
mapping := func(key string) string {
switch key {
case "PWD":
return wd
default:
return ""
}
}
s.DefaultBuildsDir = os.Expand(s.DefaultBuildsDir, mapping)
s.DefaultCacheDir = os.Expand(s.DefaultCacheDir, mapping)
// Pass control to executor
err = s.AbstractExecutor.Prepare(globalConfig, config, build)
if err != nil {
return err
}
s.Println("Using Shell executor...")
return nil
}
func (lw *LogWatcher) AddContainer(cid string) {
c, err := lw.docker.InspectContainer(cid)
if err != nil {
log.WithFields(log.Fields{
"ID": cid,
}).Warn(err)
return
}
var envExpand = func(key string) string {
for _, v := range c.Config.Env {
if strings.HasPrefix(v, key+"=") {
return strings.SplitN(v, "=", 2)[1]
}
}
return ""
}
if os.Expand("$DLE_IGNORE", envExpand) != "" {
log.WithFields(log.Fields{
"ID": cid,
}).Info("Ignoring container")
return
}
token := os.Expand("$DLE_TOKEN", envExpand)
if token == "" {
token = defaultToken
}
lf := log.Fields{
"ID": cid,
"token": token,
}
log.WithFields(lf).Info("Watching container")
logwriter := &LogWriter{
logline: lw.LogLines,
token: token,
}
logopts := docker.LogsOptions{
Container: cid,
OutputStream: logwriter,
ErrorStream: logwriter,
Stdout: true,
Stderr: true,
Follow: true,
Tail: "0",
RawTerminal: true,
}
err = lw.docker.Logs(logopts)
if err != nil {
fmt.Println("error:", err)
}
log.WithFields(lf).Info("Stopped watching container")
}
func TestReplaceEnvironment(t *testing.T) {
r := ReplaceEnvironment([]string{
"TMP_DIR=/home/user/please/src/core",
"PKG=src/core",
"SRCS=core.go build_env.go",
})
assert.Equal(t,
"/home/user/please/src/core src/core core.go build_env.go",
os.Expand("$TMP_DIR ${PKG} ${SRCS}", r))
assert.Equal(t, "", os.Expand("$WIBBLE", r))
}
func (g *goemon) externalCommand(command, file string) bool {
var cmd *exec.Cmd
command = os.Expand(command, func(s string) string {
switch s {
case "GOEMON_TARGET_FILE":
return file
case "GOEMON_TARGET_BASE":
return filepath.Base(file)
case "GOEMON_TARGET_DIR":
return filepath.ToSlash(filepath.Dir(file))
case "GOEMON_TARGET_EXT":
return filepath.Ext(file)
case "GOEMON_TARGET_NAME":
fn := filepath.Base(file)
ext := filepath.Ext(file)
return fn[:len(fn)-len(ext)]
}
return os.Getenv(s)
})
if runtime.GOOS == "windows" {
cmd = exec.Command("cmd", "/c", command)
} else {
cmd = exec.Command("sh", "-c", command)
}
g.Logger.Println("executing", command)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
g.Logger.Println(err)
return false
}
return true
}
func main() {
fmt.Println(os.ExpandEnv("HOME = $HOME; GOROOT = $GOROOT"))
fmt.Println(os.Expand("$GOROOT", func(s string) string {
return s + " = " + os.Getenv(s)
}))
}
// ParseEnv parses io.Reader into an arrays of strings
// representing the environment, in the form "key=value".
func ParseEnv(r io.Reader) ([]string, error) {
sysenv := env2Map(os.Environ())
localenv := make(map[string]string)
mapping := func(key string) string {
value, ok := localenv[key]
if !ok {
value, ok = sysenv[key]
}
return value
}
env := []string{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
entry := strings.TrimSpace(scanner.Text())
if len(entry) == 0 {
continue
}
pair := strings.SplitN(entry, "=", 2)
key := pair[0]
value := os.Expand(pair[1], mapping)
localenv[key] = value
env = append(env, fmt.Sprintf("%s=%s", key, value))
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("procker: parse env error: %s", err)
}
return env, nil
}
func testExpand(name string) {
testStr := fmt.Sprintf("hello, ${%s}", name)
newStr := os.Expand(testStr, func(s string) string {
return "My " + s
})
fmt.Println(newStr)
}
// parseURLPrefixTag expects an input in the form of 'tag-host/path'
// and returns the lower cased host plus the path unaltered if the
// prefix matches the tag.
func parseURLPrefixTag(s, prefix string, env map[string]string) (host, path string, ok bool) {
s = strings.TrimSpace(s)
if !strings.HasPrefix(s, prefix) {
return "", "", false
}
// split host/path
p := strings.SplitN(s[len(prefix):], "/", 2)
if len(p) != 2 {
log.Printf("[WARN] consul: Invalid %s tag %q", prefix, s)
return "", "", false
}
// expand $x or ${x} to env[x] or ""
expand := func(s string) string {
return os.Expand(s, func(x string) string {
if env == nil {
return ""
}
return env[x]
})
}
host = strings.ToLower(expand(strings.TrimSpace(p[0])))
path = "/" + expand(strings.TrimSpace(p[1]))
return host, path, true
}
// makeEnvMap parse the given env string set and returns a new envMap
func makeEnvMap(set []string, inherit bool, ref ...*envMap) *envMap {
env := make(envMap)
for _, kv := range set {
if strings.Contains(kv, PathListSeparator) {
kv = strings.Replace(kv, PathListSeparator, string(os.PathListSeparator), -1)
}
kv = os.Expand(kv, func(key string) string {
for _, r := range append(ref, &env) {
if v, ok := (*r)[key]; ok {
return v
}
}
if v := os.Getenv(key); inherit && v != "" {
return v
} else {
return "" //key
}
})
if pair := strings.Split(kv, "="); len(pair) < 2 {
env[pair[0]] = ""
} else if len(pair) > 2 {
pair = strings.FieldsFunc(kv, getQuoteSplitter('='))
env[pair[0]] = pair[1]
} else {
env[pair[0]] = pair[1]
}
}
return &env
}
// getCmd returns exec.Cmd
// binary names will be evaluated with Env here since this is the last step before Run()
func (a *app) getCmd() (*exec.Cmd, error) {
path, err := exec.LookPath(a.bin)
if err != nil {
if path, err = exec.LookPath(os.Expand(a.bin, func(key string) string {
if v, ok := (*Env)[key]; ok {
return v
} else {
return ""
}
})); err != nil {
return nil, fmt.Errorf("installing %v is in your future...", a.bin)
}
}
cmd := exec.Command(path, a.arg...)
if a.dir != "" {
cmd.Dir = a.dir
}
cmd.Env = Env.combine(append(a.env, os.Environ()...)).list()
cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
if len(Env.list()) > 0 {
logger.Printf("Env: %s\n", Env)
}
logger.Printf("Command loaded: %s\n", a.cmd)
return cmd, nil
}
// parseURLPrefixTag expects an input in the form of 'tag-host/path[ opts]'
// and returns the lower cased host and the unaltered path if the
// prefix matches the tag.
func parseURLPrefixTag(s, prefix string, env map[string]string) (route, opts string, ok bool) {
// expand $x or ${x} to env[x] or ""
expand := func(s string) string {
return os.Expand(s, func(x string) string {
if env == nil {
return ""
}
return env[x]
})
}
s = strings.TrimSpace(s)
if !strings.HasPrefix(s, prefix) {
return "", "", false
}
s = strings.TrimSpace(s[len(prefix):])
p := strings.SplitN(s, " ", 2)
if len(p) == 2 {
opts = p[1]
}
s = p[0]
p = strings.SplitN(s, "/", 2)
if len(p) == 1 {
log.Printf("[WARN] consul: Invalid %s tag %q - You need to have a trailing slash!", prefix, s)
return "", "", false
}
host, path := p[0], p[1]
return strings.ToLower(expand(host)) + "/" + expand(path), opts, true
}
func (e *EnvMap) get(
key string, top *EnvMap, processQueue map[string]*EnvMap,
cache map[string]string,
) (string, bool) {
resolve := func(s string) string {
if value, ok := cache[s]; ok == true {
return value
}
if last, ok := processQueue[s]; ok == true {
// If this is the last element in this environment map
// then we return ""
if last == nil {
return ""
}
processQueue[s] = last.parent
r, _ := last.get(s, top, processQueue, cache)
return r
}
processQueue[s] = top
r, _ := top.get(s, top, processQueue, cache)
return r
}
for e != nil {
if value, ok := e.env[key]; ok == true {
processQueue[key] = e.parent
s := os.Expand(value, resolve)
delete(processQueue, key)
return s, true
}
e = e.parent
}
return "", false
}
func (m *Machine) executePostProvision() ([]string, error) {
fmt.Println("Executing post-provision commands...")
out := []string{}
for _, p := range m.postProvision() {
fmt.Printf(" ... '%s'\n", p)
args, err := shellwords.Parse(p)
if err != nil {
return []string{}, err
}
if args[0] == "scp" {
// it's docker-machine sub-command
args = append([]string{"docker-machine", "-s", dpmHome()}, args...)
}
for i := range args {
args[i] = os.Expand(args[i], func(key string) string { return m.expand(key) })
}
cmd := exec.Command(args[0], args[1:]...)
if args[0] == "docker" {
cmd.Env = m.GetEnv()
}
o, err := cmd.CombinedOutput()
out = append(out, string(o))
}
return out, nil
}
func ExpandFromMap(template string, m map[string]interface{}) string {
return os.Expand(template, func(s string) string {
if v, ok := m[s]; ok {
return fmt.Sprintf("%v", v)
}
return "[?]"
})
}
func main() {
mapping := func(s string) string {
m := map[string]string{"hello": "world", "go": "perfect program language"}
return m[s]
}
str := "Golang is$not a $go in the ${hello}!"
fmt.Printf("%s\n", os.Expand(str, mapping))
}
// split breaks the line into words, evaluating quoted
// strings and evaluating environment variables.
// The initial //go:generate element is present in line.
func (g *Generator) split(line string) []string {
// Parse line, obeying quoted strings.
var words []string
line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
// There may still be a carriage return.
if len(line) > 0 && line[len(line)-1] == '\r' {
line = line[:len(line)-1]
}
// One (possibly quoted) word per iteration.
Words:
for {
line = strings.TrimLeft(line, " \t")
if len(line) == 0 {
break
}
if line[0] == '"' {
for i := 1; i < len(line); i++ {
c := line[i] // Only looking for ASCII so this is OK.
switch c {
case '\\':
if i+1 == len(line) {
g.errorf("bad backslash")
}
i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
case '"':
word, err := strconv.Unquote(line[0 : i+1])
if err != nil {
g.errorf("bad quoted string")
}
words = append(words, word)
line = line[i+1:]
// Check the next character is space or end of line.
if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
g.errorf("expect space after quoted argument")
}
continue Words
}
}
g.errorf("mismatched quoted string")
}
i := strings.IndexAny(line, " \t")
if i < 0 {
i = len(line)
}
words = append(words, line[0:i])
line = line[i:]
}
// Substitute command if required.
if len(words) > 0 && g.commands[words[0]] != nil {
// Replace 0th word by command substitution.
words = append(g.commands[words[0]], words[1:]...)
}
// Substitute environment variables.
for i, word := range words {
words[i] = os.Expand(word, g.expandVar)
}
return words
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
return
}
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *memprofile != "" {
f, err := os.Create(*memprofile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
defer pprof.WriteHeapProfile(f)
}
config.Parse(*config_file)
runtime.GOMAXPROCS(*processes)
var err error
pct, err := timers.NewPercentiles(*percentile_thresholds)
if err != nil {
log.Fatal(err)
}
inst := os.Expand(*instance, expand_cfg_vars)
if inst == "" {
inst = "null"
}
signalchan := make(chan os.Signal, 1)
signal.Notify(signalchan)
if *profile_addr != "" {
go func() {
fmt.Println("Profiling endpoint listening on " + *profile_addr)
log.Println(http.ListenAndServe(*profile_addr, nil))
}()
}
daemon := statsdaemon.New(inst, *listen_addr, *admin_addr, *graphite_addr, *prefix_rates, *prefix_timers, *prefix_gauges, *pct, *flushInterval, MAX_UNPROCESSED_PACKETS, *max_timers_per_s, signalchan, *debug)
if *debug {
consumer := make(chan interface{}, 100)
daemon.Invalid_lines.Register(consumer)
go func() {
for line := range consumer {
log.Printf("invalid line '%s'\n", line)
}
}()
}
daemon.Run()
}
func (rec *checkPerms) ServeHTTP(w http.ResponseWriter, r *http.Request, p httprouter.Params, next httprouter.Handle) {
var (
fields = strings.Split(r.Header.Get("Authorization"), ":")
l = len(fields)
timestamp int64
err error
)
// check hmac and santize fields
if l < 3 {
w.WriteHeader(http.StatusUnauthorized)
return
}
message := []byte(strings.Join(fields[:l-1], ":"))
messageMAC := []byte(fields[l-1])
if !util.CheckMAC(message, messageMAC, rec.secret) {
w.WriteHeader(http.StatusUnauthorized)
return
}
// check timestamp is still valid
if timestamp, err = strconv.ParseInt(fields[l-2], 10, 32); err != nil {
w.WriteHeader(http.StatusUnauthorized)
return
}
if int64(time.Now().Unix())-timestamp >= int64(rec.expire_secs) {
w.WriteHeader(http.StatusUnauthorized)
return
}
// expand params
required_perm := os.Expand(rec.perm, func(str string) string {
return p.ByName(str)
})
// compile regexp
rx, err := regexp.Compile("^" + required_perm + "$")
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
// checks if any of the permissions suffices
//
// NOTE: constant time comparison is not needed because what is permission
// needed will be public anyway and the incoming string has been already
// authenticated with the HMAC. If it was needed, we wouldn't allow regexps,
// as there's currently no easy way to do constant time comparison with
// them.
if !rx.MatchString(fields[0]) {
w.WriteHeader(http.StatusUnauthorized)
return
}
next(w, r, p)
}
func expandHandlerVars(strs []string, values map[string]string) (expanded []string) {
mappingFunc := func(name string) string {
return values[name]
}
for _, str := range strs {
expanded = append(expanded, os.Expand(str, mappingFunc))
}
return
}
func ExpandEnv(s string) string {
return os.Expand(s, func(key string) string {
if key == "$" {
return key
} else {
v, _ := syscall.Getenv(key)
return v
}
})
}
func (s *Spec) ExportEnvsToFile(filename string) error {
envs := []string{}
for k, v := range s.ExportedEnvs {
val := os.Expand(v, func(key string) string { return expand(key) })
envs = append(envs, k+"="+val)
}
content := strings.Join(envs, "\n")
return ioutil.WriteFile(filename, []byte(content), 0644)
}
func (m *Machine) postProvision() []string {
result := []string{}
for _, p := range m.post {
expanded := os.Expand(p, func(key string) string {
return m.expand(key)
})
result = append(result, expanded)
}
return result
}
func printTempDirs(state *core.BuildState, duration float64) {
fmt.Printf("Temp directories prepared, total time %0.2fs:\n", duration)
for _, label := range state.ExpandOriginalTargets() {
target := state.Graph.TargetOrDie(label)
cmd := build.ReplaceSequences(target, target.GetCommand())
env := core.BuildEnvironment(state, target, false)
fmt.Printf(" %s: %s\n", label, target.TmpDir())
fmt.Printf(" Command: %s\n", cmd)
fmt.Printf(" Expanded: %s\n", os.Expand(cmd, core.ReplaceEnvironment(env)))
}
}
// FIXME(ssx): maybe need to return error
func (p *Process) buildCommand() *kexec.KCommand {
cmd := kexec.CommandString(p.Command)
// cmd := kexec.Command(p.Command[0], p.Command[1:]...)
logDir := filepath.Join(defaultConfigDir, "log", sanitize.Name(p.Name))
if !IsDir(logDir) {
os.MkdirAll(logDir, 0755)
}
var fout io.Writer
var err error
p.OutputFile, err = os.OpenFile(filepath.Join(logDir, "output.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.Warn("create stdout log failed:", err)
fout = ioutil.Discard
} else {
fout = p.OutputFile
}
cmd.Stdout = io.MultiWriter(p.Stdout, p.Output, fout)
cmd.Stderr = io.MultiWriter(p.Stderr, p.Output, fout)
// config environ
cmd.Env = os.Environ() // inherit current vars
environ := map[string]string{}
if p.User != "" {
err := cmd.SetUser(p.User)
if err != nil {
log.Warnf("[%s] chusr to %s failed", p.Name, p.User)
} else {
var homeDir string
switch runtime.GOOS {
case "linux":
homeDir = "/home/" + p.User // FIXME(ssx): maybe there is a better way
case "darwin":
homeDir = "/Users/" + p.User
}
cmd.Env = append(cmd.Env, "HOME="+homeDir, "USER="******"HOME"] = homeDir
environ["USER"] = p.User
}
}
cmd.Env = append(cmd.Env, p.Environ...)
mapping := func(key string) string {
val := os.Getenv(key)
if val != "" {
return val
}
return environ[key]
}
cmd.Dir = os.Expand(p.Dir, mapping)
if strings.HasPrefix(cmd.Dir, "~") {
cmd.Dir = mapping("HOME") + cmd.Dir[1:]
}
log.Infof("[%s] use dir: %s\n", p.Name, cmd.Dir)
return cmd
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
return
}
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *memprofile != "" {
f, err := os.Create(*memprofile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
defer pprof.WriteHeapProfile(f)
}
config.Parse(*config_file)
pcts := strings.Split(*percentile_thresholds, ",")
for _, pct := range pcts {
percentThreshold.Set(pct)
}
inst := os.Expand(*instance, expand_cfg_vars)
if inst == "" {
inst = "null"
}
prefix_internal = "service=statsdaemon.instance=" + inst + "."
log.Printf("statsdaemon instance '%s' starting\n", inst)
signalchan = make(chan os.Signal, 1)
signal.Notify(signalchan)
if *debug {
consumer := make(chan interface{}, 100)
invalid_lines.Register(consumer)
go func() {
for line := range consumer {
log.Printf("invalid line '%s'\n", line)
}
}()
}
output := &common.Output{Metrics, metricAmounts, valid_lines, invalid_lines}
go udp.StatsListener(*listen_addr, prefix_internal, output)
go udp.ArchiveStatsListener(*listen_archive_addr, prefix_internal, output)
go adminListener()
go metricStatsMonitor()
metricsMonitor()
}
// Defer takes a string (with optional variables) and an expansion function and returns
// a function that can be called to get the value. This method will optimize the
// expansion away in the event that no expansion is necessary.
func (c *ServiceResolverCache) Defer(env string) (func() (string, bool), error) {
hasExpansion := false
invalid := []string{}
os.Expand(env, func(name string) string {
hasExpansion = true
if _, _, ok := recognizeVariable(name); !ok {
invalid = append(invalid, name)
}
return ""
})
if len(invalid) != 0 {
return nil, fmt.Errorf("invalid variable name(s): %s", strings.Join(invalid, ", "))
}
if !hasExpansion {
return func() (string, bool) { return env, true }, nil
}
// only load the value once
lock := sync.Mutex{}
loaded := false
return func() (string, bool) {
lock.Lock()
defer lock.Unlock()
if loaded {
return env, true
}
resolved := true
expand := os.Expand(env, func(s string) string {
s, ok := c.resolve(s)
resolved = resolved && ok
return s
})
if !resolved {
return "", false
}
loaded = true
env = expand
return env, true
}, nil
}
func buildEnviOS(arch string) (map[string]string, error) {
env := map[string]string{}
m := func(k string) string { return env[k] }
env["GOMOBILE"] = filepath.Clean(filepath.Join(filepath.Dir(goMobileExe), "..", "pkg", "gomobile"))
env["GOOS"] = "darwin"
env["GOARCH"] = arch
env["CC"] = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
env["CXX"] = env["CC"]
env["SDK"] = "/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk"
env["CGO_ENABLED"] = "1"
switch arch {
case "arm":
env["GOARM"] = "7"
env["CGO_CFLAGS"] = os.Expand("-isysroot ${SDK} -arch armv7", m)
env["CGO_LDFLAGS"] = os.Expand("-isysroot ${SDK} -arch armv7", m)
env["PKGDIR"] = os.Expand("${GOMOBILE}/pkg_darwin_arm", m)
case "arm64":
env["CGO_CFLAGS"] = os.Expand("-isysroot ${SDK} -arch arm64", m)
env["CGO_LDFLAGS"] = os.Expand("-isysroot ${SDK} -arch arm64", m)
env["PKGDIR"] = os.Expand("${GOMOBILE}/pkg_darwin_arm64", m)
default:
return nil, fmt.Errorf("unknown arch: %q", arch)
}
return env, nil
}
func (e *EnvMap) Set(key, value string) {
if prev, ok := e.env[key]; ok == true {
resolve := func(s string) string {
if s == key {
return prev
}
return "$" + key
}
e.env[key] = os.Expand(value, resolve)
} else {
e.env[key] = value
}
}
func (instance *Service) expandValue(ai *access.Access, in string) string {
return os.Expand(in, func(key string) string {
if value, ok := (*instance).config.Environment[key]; ok {
return value
} else if key == "CTD_PEM" {
if ai.Type() == access.GenerateToEnvironment {
return string(ai.Pem())
}
return ""
}
return os.Getenv(key)
})
}