func (fs *osFileSystem) Symlink(oldPath, newPath string) error {
fs.logger.Debug(fs.logTag, "Symlinking oldPath %s with newPath %s", oldPath, newPath)
source, target, err := fs.symlinkPaths(oldPath, newPath)
if err != nil {
bosherr.WrapErrorf(err, "Getting absolute paths for target and path links: %s %s", oldPath, newPath)
}
if fi, err := fs.Lstat(target); err == nil {
if fi.Mode()&os.ModeSymlink != 0 {
// Symlink
new, err := fs.Readlink(target)
if err != nil {
return bosherr.WrapErrorf(err, "Reading link for %s", target)
}
if filepath.Clean(source) == filepath.Clean(new) {
return nil
}
}
if err := fs.RemoveAll(target); err != nil {
return bosherr.WrapErrorf(err, "Removing new path at %s", target)
}
}
containingDir := filepath.Dir(target)
if !fs.FileExists(containingDir) {
fs.MkdirAll(containingDir, os.FileMode(0700))
}
return fsWrapper.Symlink(source, target)
}
// vcsForDir inspects dir and its parents to determine the
// version control system and code repository to use.
// On return, root is the import path
// corresponding to the root of the repository
// (thus root is a prefix of importPath).
func vcsForDir(p *Package) (vcs *vcsCmd, root string, err error) {
// Clean and double-check that dir is in (a subdirectory of) srcRoot.
dir := filepath.Clean(p.Dir)
srcRoot := filepath.Clean(p.build.SrcRoot)
if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator {
return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
}
origDir := dir
for len(dir) > len(srcRoot) {
for _, vcs := range vcsList {
if fi, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil && fi.IsDir() {
return vcs, dir[len(srcRoot)+1:], nil
}
}
// Move to parent.
ndir := filepath.Dir(dir)
if len(ndir) >= len(dir) {
// Shouldn't happen, but just in case, stop.
break
}
dir = ndir
}
return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir)
}
func AbsPathify(inPath string) string {
if filepath.IsAbs(inPath) {
return filepath.Clean(inPath)
}
return filepath.Clean(filepath.Join(viper.GetString("WorkingDir"), inPath))
}
func MoveAllCmd(args []string) (err error) {
if len(args) != 2 {
return MakeErr("Usage: gorf [flags] moveall <old path> <new path>")
}
oldPath, newPath := filepath.Clean(args[0]), filepath.Clean(args[1])
err = ScanAllForImports(LocalRoot)
if err != nil {
return
}
prefix := fmt.Sprintf("%s%c", oldPath, filepath.Separator)
fmt.Println("prefix:", prefix)
for opath := range PackageTops {
opath = fmt.Sprintf("%s%c", filepath.Clean(opath), filepath.Separator)
if strings.HasPrefix(opath, prefix) {
tail := opath[len(prefix):]
npath := filepath.Join(newPath, tail)
err = MoveCmd([]string{opath, npath})
if err != nil {
return
}
}
}
return
}
func TestClean(t *testing.T) {
tests := cleantests
if runtime.GOOS == "windows" {
for i := range tests {
tests[i].result = filepath.FromSlash(tests[i].result)
}
tests = append(tests, wincleantests...)
}
for _, test := range tests {
if s := filepath.Clean(test.path); s != test.result {
t.Errorf("Clean(%q) = %q, want %q", test.path, s, test.result)
}
if s := filepath.Clean(test.result); s != test.result {
t.Errorf("Clean(%q) = %q, want %q", test.result, s, test.result)
}
}
if testing.Short() {
t.Skip("skipping malloc count in short mode")
}
if runtime.GOMAXPROCS(0) > 1 {
t.Log("skipping AllocsPerRun checks; GOMAXPROCS>1")
return
}
for _, test := range tests {
allocs := testing.AllocsPerRun(100, func() { filepath.Clean(test.result) })
if allocs > 0 {
t.Errorf("Clean(%q): %v allocs, want zero", test.result, allocs)
}
}
}
func parseBindMountSpec(spec string) (*volumeMount, error) {
arr := strings.Split(spec, ":")
mnt := &volumeMount{}
switch len(arr) {
case 2:
mnt.hostPath = arr[0]
mnt.containerPath = arr[1]
mnt.writable = true
case 3:
mnt.hostPath = arr[0]
mnt.containerPath = arr[1]
mnt.writable = validMountMode(arr[2]) && arr[2] == "rw"
default:
return nil, fmt.Errorf("Invalid volume specification: %s", spec)
}
if !filepath.IsAbs(mnt.hostPath) {
return nil, fmt.Errorf("cannot bind mount volume: %s volume paths must be absolute.", mnt.hostPath)
}
mnt.hostPath = filepath.Clean(mnt.hostPath)
mnt.containerPath = filepath.Clean(mnt.containerPath)
return mnt, nil
}
// CopyResource performs an archive copy from the given source path to the
// given destination path. The source path MUST exist and the destination
// path's parent directory must exist.
func CopyResource(srcPath, dstPath string) error {
var (
srcInfo CopyInfo
err error
)
// Ensure in platform semantics
srcPath = normalizePath(srcPath)
dstPath = normalizePath(dstPath)
// Clean the source and destination paths.
srcPath = PreserveTrailingDotOrSeparator(filepath.Clean(srcPath), srcPath)
dstPath = PreserveTrailingDotOrSeparator(filepath.Clean(dstPath), dstPath)
if srcInfo, err = CopyInfoSourcePath(srcPath); err != nil {
return err
}
content, err := TarResource(srcInfo)
if err != nil {
return err
}
defer content.Close()
return CopyTo(content, srcInfo, dstPath)
}
func handleFileChange(event fsnotify.Event) {
validC, _ := regexp.MatchString(validCString, event.Name)
validH, _ := regexp.MatchString(validHString, event.Name)
db := <-writer
defer func() { writer <- db }()
switch {
case validC:
switch {
case event.Op&(fsnotify.Create|fsnotify.Write) != 0:
files <- event.Name
case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
db.RemoveFileReferences(event.Name)
}
case validH:
exist, uptodate, _, err := db.UptodateFile(event.Name)
switch {
case err != nil:
return
case event.Op&(fsnotify.Write) != 0:
if exist && !uptodate {
removeFileAndReparseDepends(filepath.Clean(event.Name), db)
}
case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
if exist {
removeFileAndReparseDepends(filepath.Clean(event.Name), db)
}
}
}
}
func MakeParser(src, projectPath, scriptPath, fileName string, isRootFile bool, compilingMixer bool, layers []string, appliedLayers string) *Parser {
fullpath := filepath.Join(projectPath, scriptPath, fileName)
fullpath, _ = filepath.Abs(fullpath)
scriptPath = filepath.Clean(scriptPath)
projectPath = filepath.Clean(projectPath)
p := &Parser{
Tokenizer: MakeTokenizer([]byte(src)),
ProjectPath: projectPath, // the project path (probably absolute)
ScriptPath: scriptPath, // the folder containing the script file being parsed (relative to the project path)
FileName: fileName, // the base-name of the script file being parsed
FullPath: fullpath,
Lookahead: nil,
counter: 0,
RootFile: isRootFile,
Namespaces: make([]string, 1),
Defspace: "tritium",
inFunc: false,
CompilingMixer: compilingMixer,
Layers: layers,
AppliedLayers: appliedLayers,
}
p.Namespaces[0] = "tritium"
p.pop()
return p
}
func main() {
flag.Parse()
args := flag.Args()
if len(args) < 2 {
logrus.Fatal("Expecting source and target directory")
}
targetDir, err := filepath.Abs(args[len(args)-1])
if err != nil {
logrus.Fatal("Error resolving target directory %s: %v", args[len(args)-1], err)
}
targetDir = filepath.Clean(targetDir)
for _, bundleDir := range args[:len(args)-1] {
absDir, err := filepath.Abs(bundleDir)
if err != nil {
logrus.Fatalf("Error resolving directory %s: %v", bundleDir, err)
}
bundleDir = filepath.Clean(absDir)
logrus.Infof("Copying %s to %s", bundleDir, targetDir)
if err := buildutil.CopyBundleBinaries(bundleDir, targetDir); err != nil {
logrus.Fatalf("Error copying binaries: %v", err)
}
}
}
func LoadConfig() {
var cp = os.Getenv("ICECONFIGPATH")
if cp == "" {
dir, err := os.Getwd()
if err != nil {
panic(err)
}
cp = findEnv(dir)
} else {
cp = path.Join(path.Clean(cp), ".env")
}
fs, err := os.Open(cp)
if err != nil {
panic("Error opening config file " + cp + err.Error())
}
defer fs.Close()
err = ParseJSON(fs, &Config)
if err != nil {
panic("Error parsing config file " + err.Error())
}
Config.ConfigRoot = path.Dir(cp)
if !path.IsAbs(Config.MigrationPath) {
Config.MigrationPath = path.Clean(path.Join(Config.ConfigRoot, Config.MigrationPath))
}
if Config.Settings == nil {
Config.Settings = make(map[string]string)
}
}
func sqlclusterPath() string {
dir := scriptDir()
// If you're using our build script, sqlcluster will be a
// directory down from octopus.
p := filepath.Clean(filepath.Join(dir, "../sqlcluster"))
if isExecutable(p) {
return p
}
// See whether you built it somewhere in your GOPATH
search := strings.Split(os.Getenv("GOPATH"), ":")
for _, d := range search {
p := filepath.Clean(filepath.Join(d, "src/stripe-ctf.com/sqlcluster/sqlcluster"))
if isExecutable(p) {
return p
}
p = filepath.Clean(filepath.Join(d, "bin/sqlcluster"))
if isExecutable(p) {
return p
}
}
// As a last ditch, see whether it's in our path
p, err := exec.LookPath("sqlcluster")
if err == nil {
return p
}
return ""
}
func (r *Repository) LoadPackages(funcMap GoTemplate.FuncMap) {
log.Trace.Printf("Loading packages from %s ", filepath.Clean(r.configDirectory+"/packages.json"))
ok := r.loadPackagesFromFile(filepath.Clean(r.configDirectory+"/packages.json"), funcMap)
if !ok {
log.Warning.Printf("Could not load packages from packages.json")
}
_, err := os.Stat(filepath.Clean(r.configDirectory + "/conf.d/"))
if err == nil {
log.Trace.Printf("Loading packages from %s ", filepath.Clean(r.configDirectory+"/conf.d"))
files, _ := filepath.Glob(filepath.Clean(r.configDirectory + "/conf.d/*.json"))
for _, f := range files {
ok := r.loadPackagesFromFile(f, funcMap)
if !ok {
log.Info.Printf("Could not load packages from file: %s", f)
}
}
}
if r.packages == nil || len(r.packages) <= 0 {
log.Warning.Printf("No package definitions were found")
} else {
log.Info.Printf("%d packages have been loaded", len(r.packages))
}
}
// PostProcess post-processes the flags to fix any compatibility issue.
func (a *ArchiveOptions) PostProcess(cwd string) {
// Set default blacklist only if none is set.
if len(a.Blacklist) == 0 {
// This cannot be generalized as ".*" as there is known use that require
// a ".pki" directory to be mapped.
a.Blacklist = common.Strings{
".git",
".hg",
".svn",
}
}
if !filepath.IsAbs(a.Isolate) {
a.Isolate = filepath.Join(cwd, a.Isolate)
}
a.Isolate = filepath.Clean(a.Isolate)
if !filepath.IsAbs(a.Isolated) {
a.Isolated = filepath.Join(cwd, a.Isolated)
}
a.Isolated = filepath.Clean(a.Isolated)
for k, v := range a.PathVariables {
// This is due to a Windows + GYP specific issue, where double-quoted paths
// would get mangled in a way that cannot be resolved unless a space is
// injected.
a.PathVariables[k] = strings.TrimSpace(v)
}
}
func WalkDirectory(walkDirectories []string) (directoryList []string, fileList []string, templateList []string, err error) {
for _, value := range walkDirectories {
log.Debugf("+ Going to Walk %s", value)
if DirectoryExists(value) {
err = filepath.Walk(value, func(path string, f os.FileInfo, err error) error {
log.Debugf(" + %s", f.Name())
if f.IsDir() {
directoryList = append(directoryList, filepath.Clean(path))
} else {
if strings.HasPrefix(f.Name(), ".") {
return nil
}
if !strings.Contains(f.Name(), ".mustache") {
fileList = append(fileList, filepath.Clean(path))
} else {
if !strings.Contains(f.Name(), ".after") || !strings.Contains(f.Name(), ".before") {
templateList = append(templateList, filepath.Clean(path))
}
}
}
return nil
})
}
if err != nil {
return []string{}, []string{}, []string{}, err
}
}
return directoryList, fileList, templateList, nil
}
func find_bzl_project_root(libpath, path string) (string, error) {
if libpath == "" {
return "", fmt.Errorf("could not find project root, libpath is empty")
}
pathMap := map[string]struct{}{}
for _, lp := range strings.Split(libpath, ":") {
lp := strings.TrimSpace(lp)
pathMap[filepath.Clean(lp)] = struct{}{}
}
path = filepath.Dir(path)
if path == "" {
return "", fmt.Errorf("project root is blank")
}
start := path
for path != "/" {
if _, ok := pathMap[filepath.Clean(path)]; ok {
return path, nil
}
path = filepath.Dir(path)
}
return "", fmt.Errorf("could not find project root in %q or its parents", start)
}
// AbsPath is a utility function to get the absolute path from a path
// It will first check for an absolute path then GOPATH relative then a pwd relative
// will return an error for a path that does not exist
func AbsPath(path string) (ap string, err error) {
// Check for absolute path
if filepath.IsAbs(path) {
ap = filepath.Clean(path)
if _, err := os.Stat(ap); err == nil {
return ap, nil
}
}
// Check for GOPATH relative
for _, gp := range GoPaths() {
ap = filepath.Clean(filepath.Join(gp, path))
_, err = os.Stat(ap)
if err == nil {
return
}
}
ap, err = filepath.Abs(path)
if err == nil {
if _, err = os.Stat(ap); err == nil {
return
}
}
return "", fmt.Errorf("%q: no such file or directory", path)
}
func clean(dir string, patterns []string) {
if *all {
files, _ := ioutil.ReadDir(dir)
for _, f := range files {
if f.IsDir() {
clean(filepath.Clean(dir+"/"+f.Name()), patterns)
}
}
}
for _, pattern := range patterns {
globFiles, err := filepath.Glob(dir + "/" + pattern)
if err != nil {
log.Println(err)
} else {
for _, globFile := range globFiles {
err2 := os.Remove(filepath.Clean(globFile))
if err2 != nil {
log.Print(err2)
} else {
log.Println("Remove", filepath.Clean(globFile))
}
}
}
}
}
func parseArgsForOSSUpload(args []string) (locals []string, remote string) {
l := len(args)
if l == 0 {
die("Error: Please specify remote file location.")
}
defer func() {
suffix := false
if strings.HasSuffix(remote, "/") {
suffix = true
}
remote = filepath.Clean(remote)
if !strings.HasPrefix(remote, "/") {
remote = "/" + remote
}
if !strings.HasSuffix(remote, "/") && suffix {
remote += "/"
}
}()
if l == 1 {
remote = args[0]
return
}
for _, arg := range args[0 : l-1] {
locals = append(locals, filepath.Clean(arg))
}
remote = args[l-1]
return
}
// parseDriverDSN cuts off DB name from dsn. It returns error if the dsn is not
// valid.
func parseDriverDSN(dsn string) (params *driverParams, err error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, errors.Trace(err)
}
path := filepath.Join(u.Host, u.Path)
dbName := filepath.Clean(filepath.Base(path))
if dbName == "" || dbName == "." || dbName == string(filepath.Separator) {
return nil, errors.Errorf("invalid DB name %q", dbName)
}
// cut off dbName
path = filepath.Clean(filepath.Dir(path))
if path == "" || path == "." || path == string(filepath.Separator) {
return nil, errors.Errorf("invalid dsn %q", dsn)
}
u.Path, u.Host = path, ""
params = &driverParams{
storePath: u.String(),
dbName: dbName,
}
// parse additional driver params
query := u.Query()
if parseTime := query.Get("parseTime"); parseTime == "true" {
params.parseTime = true
}
return params, nil
}
// Walk implements filepath.Walk function for memory file system.
func (mfs MemoryFileSystem) Walk(root string, walkFn filepath.WalkFunc) error {
cleaned := filepath.Clean(root)
paths := []string{}
for path, _ := range mfs.files {
if strings.HasPrefix(path, cleaned) {
paths = append(paths, path)
}
}
sort.Strings(paths)
for _, path := range paths {
// TODO: Provide the right information.
mode := os.FileMode(0777)
if strings.HasSuffix(path, "/") {
mode = mode | os.ModeDir
}
mf := MemoryFile{
path: path,
content: []byte{},
mode: mode,
}
err := walkFn(filepath.Clean(path), mf, nil)
if err != nil {
return err
}
}
return nil
}
func getAllConfiguredRepos(configRoot string) []repo {
var repos []repo
repoPattern := filepath.Join(configRoot, "*/*")
matches, err := filepath.Glob(repoPattern)
if err != nil {
return repos
}
for _, match := range matches {
fi, fiErr := os.Stat(match)
if fiErr != nil || !fi.Mode().IsDir() {
continue
}
rel, relErr := filepath.Rel(configRoot, match)
if relErr != nil {
continue
}
owner, name := filepath.Split(rel)
if owner != "" && name != "" {
repos = append(repos, repo{filepath.Clean(owner), filepath.Clean(name)})
}
}
return repos
}
func (k *KLog) init() (err error) {
if len(k.name) == 0 {
return errors.New("not init.")
}
day := time.Now().Format("20060102")
if k.day == day {
return nil
}
k.Close()
file := filepath.Clean(k.path + "/" + k.name + day + ".log")
if !filepath.IsAbs(file) {
file = filepath.Clean(ProcessPath() + "/" + file)
}
path := filepath.Dir(file)
if err = os.MkdirAll(path, 0777); err != nil {
return err
}
k.logFile, err = os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0660)
if err != nil {
return err
}
if os.Stdout == nil || k.bstdout == false {
k.log = log.New(k.logFile, k.prefix, log.Ldate|log.Ltime)
} else {
k.wr = io.MultiWriter(os.Stdout, k.logFile)
k.log = log.New(k.wr, k.prefix, log.Ldate|log.Ltime)
}
k.day = day
return nil
}
func main() {
wd, err := os.Getwd()
if err != nil {
panic(err)
}
pkgDir := filepath.Clean(filepath.Join(wd, "src", "pkg"))
pkgWalker := &srcWalker{srcDir: pkgDir, pkg: true}
filepath.Walk(pkgDir, pkgWalker, nil)
pkgTargets := pkgWalker.finish()
for pkg, target := range pkgTargets {
makePkg(pkg, target, pkgTargets)
}
makeDirs(pkgDir, pkgTargets)
makeDeps(pkgDir, pkgTargets)
cmdDir := filepath.Clean(filepath.Join(wd, "src", "cmd"))
cmdWalker := &srcWalker{srcDir: cmdDir, pkg: false}
filepath.Walk(cmdDir, cmdWalker, nil)
cmdTargets := cmdWalker.finish()
for cmd, target := range cmdTargets {
makeCmd(cmd, target, pkgTargets)
}
makeDirs(cmdDir, cmdTargets)
makeGitIgnore(cmdDir, cmdTargets)
}
func absPathify(inPath string) string {
jww.INFO.Println("Trying to resolve absolute path to", inPath)
if strings.HasPrefix(inPath, "$HOME") {
inPath = userHomeDir() + inPath[5:]
}
if strings.HasPrefix(inPath, "$") {
end := strings.Index(inPath, string(os.PathSeparator))
inPath = os.Getenv(inPath[1:end]) + inPath[end:]
}
if filepath.IsAbs(inPath) {
return filepath.Clean(inPath)
}
p, err := filepath.Abs(inPath)
if err == nil {
return filepath.Clean(p)
} else {
jww.ERROR.Println("Couldn't discover absolute path")
jww.ERROR.Println(err)
}
return ""
}
func (fs *osFileSystem) Symlink(oldPath, newPath string) error {
fs.logger.Debug(fs.logTag, "Symlinking oldPath %s with newPath %s", oldPath, newPath)
if fi, err := os.Lstat(newPath); err == nil {
if fi.Mode()&os.ModeSymlink != 0 {
// Symlink
new, err := os.Readlink(newPath)
if err != nil {
return bosherr.WrapErrorf(err, "Reading link for %s", newPath)
}
if filepath.Clean(oldPath) == filepath.Clean(new) {
return nil
}
}
if err := os.Remove(newPath); err != nil {
return bosherr.WrapErrorf(err, "Removing new path at %s", newPath)
}
}
containingDir := filepath.Dir(newPath)
if !fs.FileExists(containingDir) {
fs.MkdirAll(containingDir, os.FileMode(0700))
}
return symlink(oldPath, newPath)
}
// extractFileFromTar extracts a regular file from the given tar, returning its
// contents as a byte slice
func extractFileFromTar(tr *tar.Reader, file string) ([]byte, error) {
for {
hdr, err := tr.Next()
switch err {
case io.EOF:
return nil, fmt.Errorf("file not found")
case nil:
if filepath.Clean(hdr.Name) != filepath.Clean(file) {
continue
}
switch hdr.Typeflag {
case tar.TypeReg:
case tar.TypeRegA:
default:
return nil, fmt.Errorf("requested file not a regular file")
}
buf, err := ioutil.ReadAll(tr)
if err != nil {
return nil, fmt.Errorf("error extracting tarball: %v", err)
}
return buf, nil
default:
return nil, fmt.Errorf("error extracting tarball: %v", err)
}
}
}
func expandPaths(paths []string) []string {
if len(paths) == 0 {
paths = []string{"."}
}
dirs := map[string]bool{}
for _, path := range paths {
if strings.HasSuffix(path, "/...") {
root := filepath.Dir(path)
_ = filepath.Walk(root, func(p string, i os.FileInfo, err error) error {
kingpin.FatalIfError(err, "invalid path '"+p+"'")
base := filepath.Base(p)
skip := strings.ContainsAny(base[0:1], "_.") && base != "." && base != ".."
if i.IsDir() {
if skip {
return filepath.SkipDir
}
} else if !skip && strings.HasSuffix(p, ".go") {
dirs[filepath.Clean(filepath.Dir(p))] = true
}
return nil
})
} else {
dirs[filepath.Clean(path)] = true
}
}
out := make([]string, 0, len(dirs))
for d := range dirs {
out = append(out, d)
}
return out
}
func parseBindMountSpec(spec string) (string, string, bool, error) {
var (
path, mountToPath string
writable bool
arr = strings.Split(spec, ":")
)
switch len(arr) {
case 2:
path = arr[0]
mountToPath = arr[1]
writable = true
case 3:
path = arr[0]
mountToPath = arr[1]
writable = validMountMode(arr[2]) && arr[2] == "rw"
default:
return "", "", false, fmt.Errorf("Invalid volume specification: %s", spec)
}
if !filepath.IsAbs(path) {
return "", "", false, fmt.Errorf("cannot bind mount volume: %s volume paths must be absolute.", path)
}
path = filepath.Clean(path)
mountToPath = filepath.Clean(mountToPath)
return path, mountToPath, writable, nil
}
// CopyDir copies recursively src directory to dst directory.
func CopyDir(dst, src string) error {
dst, src = filepath.Clean(dst), filepath.Clean(src)
ok, err := IsTheSame(dst, src)
switch {
case err != nil:
return err
case ok:
return nil
}
for _, p := range []string{src, dst} {
if err = isdir(p); err != nil {
return err
}
}
var dest string
return filepath.Walk(src,
func(path string, fi os.FileInfo, err error) error {
dest = strings.Replace(path, src, dst, 1)
if fi.IsDir() {
if err = os.MkdirAll(dest, fi.Mode()); err != nil {
return err
}
return nil
}
return CopyFile(dest, path)
})
}