// expandBundlePath calls filepath.Glob first, then recursively adds
// descendents of any directories specified.
func expandBundlePath(pathGlob string) ([]string, error) {
paths, err := filepath.Glob(pathGlob)
if err != nil {
return nil, err
}
for _, path := range paths {
fi, err := os.Stat(path)
if err != nil {
return nil, err
}
if fi.Mode().IsDir() {
w := fs.Walk(path)
for w.Step() {
if err := w.Err(); err != nil {
return nil, err
}
paths = append(paths, w.Path())
}
}
}
return paths, nil
}
func main() {
if len(os.Args) > 1 {
starting_path = os.Args[1]
}
w := new(tabwriter.Writer)
w.Init(os.Stdout, 2, 0, 1, ' ', tabwriter.AlignRight)
walker := fs.Walk(starting_path)
for walker.Step() {
if err := walker.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
continue
}
path := walker.Path()
if REGEX_EXCLUDE_PATH.FindStringSubmatch(path) != nil {
continue
}
info := walker.Stat()
file_type := "f"
if info.IsDir() {
file_type = "d"
}
fmt.Fprintln(w, file_type+" \t"+(info.ModTime().Format(DATE_LAYOUT)+" \t"+strconv.FormatInt(info.Size(), 10)+"\t\t"+path))
}
w.Flush()
}
func (w *Walker) Walk() fusefs.Tree {
wg := sync.WaitGroup{}
paths := make(chan string, runtime.NumCPU())
for i := 0; i < runtime.NumCPU(); i++ {
go worker(w, paths, &wg)
}
walker := fs.Walk(w.Path)
for walker.Step() {
if err := walker.Err(); err != nil {
continue
}
if walker.Stat().IsDir() {
continue
}
wg.Add(1)
paths <- walker.Path()
}
close(paths)
wg.Wait()
return w.tree
}
func RewriteImports(path string, rw func(string) string, filter func(string) bool) error {
w := fs.Walk(path)
for w.Step() {
rel := w.Path()[len(path):]
if len(rel) == 0 {
continue
}
rel = rel[1:]
if strings.HasPrefix(rel, ".git") || strings.HasPrefix(rel, "vendor") {
w.SkipDir()
continue
}
if !strings.HasSuffix(w.Path(), ".go") {
continue
}
if !filter(rel) {
continue
}
err := rewriteImportsInFile(w.Path(), rw)
if err != nil {
fmt.Println("rewrite error: ", err)
}
}
return nil
}
func copySrc(dir string, g *Godeps) error {
ok := true
for _, dep := range g.Deps {
w := fs.Walk(dep.dir)
for w.Step() {
if w.Err() != nil {
log.Println(w.Err())
ok = false
continue
}
if c := w.Stat().Name()[0]; c == '.' || c == '_' {
// Skip directories using a rule similar to how
// the go tool enumerates packages.
// See $GOROOT/src/cmd/go/main.go:/matchPackagesInFs
w.SkipDir()
}
if w.Stat().IsDir() {
continue
}
dst := filepath.Join(dir, w.Path()[len(dep.ws)+1:])
if err := copyFile(dst, w.Path()); err != nil {
log.Println(err)
ok = false
}
}
}
if !ok {
return errors.New("error copying source code")
}
return nil
}
func TestBug3486(t *testing.T) { // http://code.google.com/p/go/issues/detail?id=3486
root, err := filepath.EvalSymlinks(runtime.GOROOT())
if err != nil {
t.Fatal(err)
}
lib := filepath.Join(root, "lib")
src := filepath.Join(root, "src")
seenSrc := false
walker := fs.Walk(root)
for walker.Step() {
if walker.Err() != nil {
t.Fatal(walker.Err())
}
switch walker.Path() {
case lib:
walker.SkipDir()
case src:
seenSrc = true
}
}
if !seenSrc {
t.Fatalf("%q not seen", src)
}
}
func copySrc(dir string, deps []Dependency) error {
ok := true
for _, dep := range deps {
srcdir := filepath.Join(dep.ws, "src")
rel, err := filepath.Rel(srcdir, dep.dir)
if err != nil { // this should never happen
return err
}
dstpkgroot := filepath.Join(dir, rel)
err = os.RemoveAll(dstpkgroot)
if err != nil {
log.Println(err)
ok = false
}
w := fs.Walk(dep.dir)
for w.Step() {
err = copyPkgFile(dir, srcdir, w)
if err != nil {
log.Println(err)
ok = false
}
}
}
if !ok {
return errors.New("error copying source code")
}
return nil
}
// Walk through root and hash each file, return a map with
// filepath for key and hash for value, check only file not dir
func marcher(dir string, method string) map[string]string {
fmt.Println("")
fmt.Println("Scanner Report")
fmt.Println("==============")
fmt.Println("")
res := make(map[string]string)
walker := fs.Walk(dir)
for walker.Step() {
// Start walking
if err := walker.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
continue
}
// Check if it is a file
finfo, err := os.Stat(walker.Path())
if err != nil {
fmt.Println(err)
continue
}
if finfo.IsDir() {
// it's a dir so pass and continue
continue
} else {
// it's a file so add couple to data map
path := walker.Path()
hash := hasher(walker.Path(), method)
res[path] = hash
fmt.Println(hash, "\t", path)
}
}
fmt.Println("")
return res
}
func readArticles() ([]*Article, []string, error) {
timeStart := time.Now()
walker := fs.Walk("blog_posts")
var res []*Article
var dirs []string
for walker.Step() {
if walker.Err() != nil {
fmt.Printf("readArticles: walker.Step() failed with %s\n", walker.Err())
return nil, nil, walker.Err()
}
st := walker.Stat()
path := walker.Path()
if st.IsDir() {
dirs = append(dirs, path)
continue
}
a, err := readArticle(path)
if err != nil {
fmt.Printf("readArticle() of %s failed with %s\n", path, err)
return nil, nil, err
}
if a != nil {
res = append(res, a)
}
}
fmt.Printf("read %d articles in %s\n", len(res), time.Since(timeStart))
return res, dirs, nil
}
func RewriteImports(path string, rw func(string) string, filter func(string) bool) error {
w := fs.Walk(path)
for w.Step() {
rel := w.Path()[len(path):]
if len(rel) == 0 {
continue
}
rel = rel[1:]
if strings.HasPrefix(rel, ".git") || strings.HasPrefix(rel, "vendor") {
w.SkipDir()
continue
}
if !filter(rel) {
continue
}
dir, fi := filepath.Split(w.Path())
good, err := build.Default.MatchFile(dir, fi)
if err != nil {
return err
}
if !good {
continue
}
err = rewriteImportsInFile(w.Path(), rw)
if err != nil {
fmt.Println("rewrite error: ", err)
return err
}
}
return nil
}
func (r *Radio) StdPopulate() error {
// Add a dummy manual playlist
r.NewPlaylist("manual")
// Add local directory
playlistsDirs := []string{"/playlists"}
playlistsDirs = append(playlistsDirs, path.Join(os.Getenv("HOME"), "playlists"))
playlistsDirs = append(playlistsDirs, path.Join("/home", "playlists"))
dir, err := os.Getwd()
if err == nil && os.Getenv("NO_LOCAL_PLAYLISTS") != "1" {
r.NewDirectoryPlaylist("local directory", dir)
playlistsDirs = append(playlistsDirs, path.Join(dir, "playlists"))
}
// Add each folders in '/playlists' and './playlists'
for _, playlistsDir := range playlistsDirs {
walker := fs.Walk(playlistsDir)
for walker.Step() {
if walker.Path() == playlistsDir {
continue
}
if err := walker.Err(); err != nil {
logrus.Warnf("walker error: %v", err)
continue
}
var realpath string
if walker.Stat().IsDir() {
realpath = walker.Path()
walker.SkipDir()
} else {
realpath, err = filepath.EvalSymlinks(walker.Path())
if err != nil {
logrus.Warnf("filepath.EvalSymlinks error for %q: %v", walker.Path(), err)
continue
}
}
stat, err := os.Stat(realpath)
if err != nil {
logrus.Warnf("os.Stat error: %v", err)
continue
}
if stat.IsDir() {
r.NewDirectoryPlaylist(fmt.Sprintf("playlist: %s", walker.Stat().Name()), realpath)
}
}
}
// Add 'standard' music paths
r.NewDirectoryPlaylist("iTunes Music", "~/Music/iTunes/iTunes Media/Music/")
r.NewDirectoryPlaylist("iTunes Podcasts", "~/Music/iTunes/iTunes Media/Podcasts/")
r.NewDirectoryPlaylist("iTunes Music", "/home/Music/iTunes/iTunes Media/Music/")
r.NewDirectoryPlaylist("iTunes Podcasts", "/home/Music/iTunes/iTunes Media/Podcasts/")
return nil
}
func ExampleWalker() {
walker := fs.Walk("/usr/lib")
for walker.Step() {
if err := walker.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
continue
}
fmt.Println(walker.Path())
}
}
// Get all python source files under dir
func pythonSourceFiles(dir string, discoveredScripts map[string]bool) (files []string) {
walker := fs.Walk(dir)
for walker.Step() {
if err := walker.Err(); err == nil && !walker.Stat().IsDir() && filepath.Ext(walker.Path()) == ".py" {
file := walker.Path()
_, found := discoveredScripts[file]
if !found {
files = append(files, filepath.ToSlash(file))
discoveredScripts[file] = true
}
}
}
return
}
// rewriteTree recursively visits the go files in path, rewriting
// import statments according to the rules for func qualify.
func rewriteTree(path, qual string, paths []string) error {
w := fs.Walk(path)
for w.Step() {
if w.Err() != nil {
log.Println("rewrite:", w.Err())
continue
}
if !w.Stat().IsDir() && strings.HasSuffix(w.Path(), ".go") {
err := rewriteGoFile(w.Path(), qual, paths)
if err != nil {
return err
}
}
}
return nil
}
func (p *Playlist) AutoUpdate() error {
if p.Path == "" {
logrus.Debugf("Playlist %q is not dynamic, skipping update", p.Name)
return nil
}
// if we are here, the playlist is based on local file system
logrus.Infof("Updating playlist %q", p.Name)
p.Status = "updating"
walker := fs.Walk(p.Path)
for walker.Step() {
if err := walker.Err(); err != nil {
logrus.Warnf("walker error: %v", err)
continue
}
stat := walker.Stat()
if stat.IsDir() {
switch stat.Name() {
case ".git", "bower_components":
walker.SkipDir()
}
} else {
switch stat.Name() {
case ".DS_Store":
continue
}
p.NewLocalTrack(walker.Path())
}
}
logrus.Infof("Playlist %q updated, %d tracks", p.Name, len(p.Tracks))
if p.Stats.Tracks > 0 {
p.Status = "ready"
} else {
p.Status = "empty"
}
p.ModificationDate = time.Now()
return nil
}
func (f *Finder) List(ignorePatterns []string) []string {
fileList := []string{}
for _, location := range f.locations {
walker := fs.Walk(location)
for walker.Step() {
err := walker.Err()
if err != nil || filenameMatchPatterns(walker.Path(), ignorePatterns) {
continue
}
fileList = append(fileList, walker.Path())
}
}
return fileList
}
func main() {
var g regexp.Grep
g.AddFlags()
g.Stdout = os.Stdout
g.Stderr = os.Stderr
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) == 0 {
flag.Usage()
}
pat := "(?m)" + strings.Join(args, ".*")
if *iflag {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
g.Regexp = re
walker := fs.Walk(".")
for walker.Step() {
if walker.Stat().IsDir() {
if strings.Contains(walker.Path(), ".git") {
walker.SkipDir()
}
continue
}
g.File(walker.Path())
}
if err := walker.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if !g.Match {
os.Exit(1)
}
}
func visit(filename string) error {
walker := fs.Walk(filename)
for walker.Step() {
if err := walker.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
continue
}
path := walker.Path()
info := walker.Stat()
if info.IsDir() {
if isHidden(path) {
walker.SkipDir()
}
}
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
walker.SkipDir()
}
if info.Mode().IsRegular() {
wg.Add(1)
go parseFile(path)
}
}
return nil
}
func copySrc(dir string, deps []Dependency) error {
// mapping to see if we visited a parent directory already
visited := make(map[string]bool)
ok := true
for _, dep := range deps {
debugln("copySrc for", dep.ImportPath)
srcdir := filepath.Join(dep.ws, "src")
rel, err := filepath.Rel(srcdir, dep.dir)
debugln("srcdir", srcdir)
debugln("rel", rel)
debugln("err", err)
if err != nil { // this should never happen
return err
}
dstpkgroot := filepath.Join(dir, rel)
err = os.RemoveAll(dstpkgroot)
if err != nil {
log.Println(err)
ok = false
}
// copy actual dependency
vf := dep.vcs.listFiles(dep.dir)
debugln("vf", vf)
w := fs.Walk(dep.dir)
for w.Step() {
err = copyPkgFile(vf, dir, srcdir, w)
if err != nil {
log.Println(err)
ok = false
}
}
// Look for legal files in root
// some packages are imports as a sub-package but license info
// is at root: exampleorg/common has license file in exampleorg
//
if dep.ImportPath == dep.root {
// we are already at root
continue
}
// prevent copying twice This could happen if we have
// two subpackages listed someorg/common and
// someorg/anotherpack which has their license in
// the parent dir of someorg
rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root))
if visited[rootdir] {
continue
}
visited[rootdir] = true
vf = dep.vcs.listFiles(rootdir)
w = fs.Walk(rootdir)
for w.Step() {
fname := filepath.Base(w.Path())
if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) {
err = copyPkgFile(vf, dir, srcdir, w)
if err != nil {
log.Println(err)
ok = false
}
}
}
}
if !ok {
return errorCopyingSourceCode
}
return nil
}
// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
if !dirExists(xmlOpts.RomDir) {
log.Printf("ERR %s: does not exists", xmlOpts.RomDir)
return nil
}
extraMap := make(map[string]struct{})
if *extraExt != "" {
extraSlice := strings.Split(*extraExt, ",")
for _, e := range extraSlice {
if e[0] != '.' {
extraMap["."+e] = struct{}{}
} else {
extraMap[e] = struct{}{}
}
}
}
for _, x := range gl.GameList {
switch {
case *appendOut:
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
case *refreshOut:
existing[x.Path] = struct{}{}
}
}
var wg sync.WaitGroup
results := make(chan *rom.GameXML, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
if _, ok := existing[r.Path]; ok && *refreshOut {
for i, g := range gl.GameList {
if g.Path != r.Path {
continue
}
copy(gl.GameList[i:], gl.GameList[i+1:])
gl.GameList = gl.GameList[:len(gl.GameList)-1]
}
}
gl.Append(r)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
_, isExtra := extraMap[r.Ext]
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" || isExtra {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}
// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var missingCSV *csv.Writer
var gdbDS *ds.GDB
if *missing != "" {
f, err := os.Create(*missing)
if err != nil {
return err
}
missingCSV = csv.NewWriter(f)
defer func() {
missingCSV.Flush()
if err := missingCSV.Error(); err != nil {
log.Fatal(err)
}
f.Close()
}()
if err := missingCSV.Write([]string{"Game", "Error", "Hash", "Extra"}); err != nil {
return err
}
for _, d := range sources {
switch d := d.(type) {
case *ds.GDB:
gdbDS = d
}
}
}
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
if !dirExists(xmlOpts.RomDir) {
log.Printf("ERR %s: does not exists", xmlOpts.RomDir)
return nil
}
extraMap := make(map[string]struct{})
if *extraExt != "" {
extraSlice := strings.Split(*extraExt, ",")
for _, e := range extraSlice {
if e[0] != '.' {
extraMap["."+e] = struct{}{}
} else {
extraMap[e] = struct{}{}
}
}
}
for _, x := range gl.GameList {
switch {
case *appendOut:
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
case *refreshOut:
existing[x.Path] = struct{}{}
}
}
var wg sync.WaitGroup
results := make(chan Result, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
if r.XML == nil {
if *missing == "" {
continue
}
files := []string{r.ROM.Path}
if r.ROM.Cue {
files = append(files, r.ROM.Bins...)
}
for _, file := range files {
var hash, extra string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(file)
if err != nil {
log.Printf("ERR: Can't hash file %s", file)
}
name := gdbDS.GetName(file)
if name != "" && r.Err == ds.NotFoundErr {
extra = "hash found but no GDB ID"
}
}
if err := missingCSV.Write([]string{file, r.Err.Error(), hash, extra}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
continue
}
if r.XML.Image == "" && *missing != "" {
var hash string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(r.ROM.Path)
if err != nil {
log.Printf("ERR: Can't hash file %s", r.ROM.Path)
}
}
if err := missingCSV.Write([]string{r.ROM.FileName, "", hash, "missing image"}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
if _, ok := existing[r.XML.Path]; ok && *refreshOut {
for i, g := range gl.GameList {
if g.Path != r.XML.Path {
continue
}
copy(gl.GameList[i:], gl.GameList[i+1:])
gl.GameList = gl.GameList[:len(gl.GameList)-1]
}
}
gl.Append(r.XML)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
_, isExtra := extraMap[r.Ext]
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" || isExtra {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}
func Scan(dir string) ([]string, error) {
fileInfo, err := os.Stat(dir)
if err != nil {
return nil, err
}
if !fileInfo.IsDir() {
return nil, fmt.Errorf("'%s' is a file, directories contain packages", fileInfo.Name())
}
imports := map[string]bool{}
w := fs.Walk(dir)
w.Step()
for w.Step() {
fstat := w.Stat()
if fstat.IsDir() {
w.SkipDir()
continue
}
// check for errors
if w.Err() != nil {
return nil, w.Err()
}
fpath := w.Path()
// check the file is a .go file
if strings.HasSuffix(fpath, ".go") {
fset := token.NewFileSet()
// parse only the import declarations in the .go file
f, err := parser.ParseFile(fset, w.Path(), nil, parser.ImportsOnly)
if err != nil {
e := err.Error()
if strings.Contains(e, eofError) {
continue
}
if strings.Contains(e, importPathError) {
for _, i := range f.Imports {
if Valid(i.Path.Value) {
path, err := strconv.Unquote(i.Path.Value)
if err != nil {
return nil, err
}
imports[path] = true
continue
}
}
continue
}
return nil, err
}
// unquote the import path value
for _, i := range f.Imports {
path, err := strconv.Unquote(i.Path.Value)
if err != nil {
return nil, err
}
imports[path] = true
}
}
}
for path, _ := range imports {
for _, exception := range Exceptions {
if Match(path, exception) {
imports[path] = false
}
}
}
paths := []string{}
for path, ok := range imports {
if ok {
paths = append(paths, path)
}
}
return paths, nil
}
func compareDirectoriesRecursive(t *testing.T, aroot, broot string) {
walker := fs.Walk(aroot)
for walker.Step() {
if err := walker.Err(); err != nil {
t.Fatal(err)
}
// find paths
aPath := walker.Path()
aRel, err := filepath.Rel(aroot, aPath)
if err != nil {
t.Fatalf("could not find relative path for %v: %v", aPath, err)
}
bPath := path.Join(broot, aRel)
if aRel == "." {
continue
}
//t.Logf("comparing: %v a: %v b %v", aRel, aPath, bPath)
// if a is a link, the sftp recursive copy won't have copied it. ignore
aLink, err := os.Lstat(aPath)
if err != nil {
t.Fatalf("could not lstat %v: %v", aPath, err)
}
if aLink.Mode()&os.ModeSymlink != 0 {
continue
}
// stat the files
aFile, err := os.Stat(aPath)
if err != nil {
t.Fatalf("could not stat %v: %v", aPath, err)
}
bFile, err := os.Stat(bPath)
if err != nil {
t.Fatalf("could not stat %v: %v", bPath, err)
}
// compare stats, with some leniency for the timestamp
if aFile.Mode() != bFile.Mode() {
t.Fatalf("modes different for %v: %v vs %v", aRel, aFile.Mode(), bFile.Mode())
}
if !aFile.IsDir() {
if aFile.Size() != bFile.Size() {
t.Fatalf("sizes different for %v: %v vs %v", aRel, aFile.Size(), bFile.Size())
}
}
timeDiff := aFile.ModTime().Sub(bFile.ModTime())
if timeDiff > time.Second || timeDiff < -time.Second {
t.Fatalf("mtimes different for %v: %v vs %v", aRel, aFile.ModTime(), bFile.ModTime())
}
// compare contents
if !aFile.IsDir() {
if aContents, err := ioutil.ReadFile(aPath); err != nil {
t.Fatal(err)
} else if bContents, err := ioutil.ReadFile(bPath); err != nil {
t.Fatal(err)
} else if string(aContents) != string(bContents) {
t.Fatalf("contents different for %v", aRel)
}
}
}
}
// List finds all toolchains in the SRCLIBPATH.
//
// List does not find nested toolchains; i.e., if DIR is a toolchain
// dir (with a DIR/Srclibtoolchain file), then none of DIR's
// subdirectories are searched for toolchains.
func List() ([]*Info, error) {
if noToolchains {
return nil, nil
}
var found []*Info
seen := map[string]string{}
dirs := filepath.SplitList(srclib.Path)
// maps symlinked trees to their original path
origDirs := map[string]string{}
for i := 0; i < len(dirs); i++ {
dir := dirs[i]
if dir == "" {
dir = "."
}
w := fs.Walk(dir)
for w.Step() {
if w.Err() != nil {
return nil, w.Err()
}
fi := w.Stat()
name := fi.Name()
path := w.Path()
if path != dir && (name[0] == '.' || name[0] == '_') {
w.SkipDir()
} else if fi.Mode()&os.ModeSymlink != 0 {
// Check if symlink points to a directory.
if sfi, err := os.Stat(path); err == nil {
if !sfi.IsDir() {
continue
}
} else if os.IsNotExist(err) {
continue
} else {
return nil, err
}
// traverse symlinks but refer to symlinked trees' toolchains using
// the path to them through the original entry in SRCLIBPATH
dirs = append(dirs, path+string(filepath.Separator))
origDirs[path+string(filepath.Separator)] = dir
} else if fi.Mode().IsDir() {
// Check for Srclibtoolchain file in this dir.
if _, err := os.Stat(filepath.Join(path, ConfigFilename)); os.IsNotExist(err) {
continue
} else if err != nil {
return nil, err
}
// Found a Srclibtoolchain file.
path = filepath.Clean(path)
var base string
if orig, present := origDirs[dir]; present {
base = orig
} else {
base = dir
}
toolchainPath, _ := filepath.Rel(base, path)
if otherDir, seen := seen[toolchainPath]; seen {
return nil, fmt.Errorf("saw 2 toolchains at path %s in dirs %s and %s", toolchainPath, otherDir, path)
}
seen[toolchainPath] = path
info, err := newInfo(toolchainPath, path, ConfigFilename)
if err != nil {
return nil, err
}
found = append(found, info)
// Disallow nested toolchains to speed up List. This
// means that if DIR/Srclibtoolchain exists, no other
// Srclibtoolchain files underneath DIR will be read.
w.SkipDir()
}
}
}
return found, nil
}
func CompareTree(basepath string, remoteMap map[string]file, skipFiles, skipDirs []*regexp.Regexp, appendFile bool) (rmdirs []file, rms []file, mkdirs []file, puts []file) {
walker := fs.Walk(basepath)
for walker.Step() {
if err := walker.Err(); err != nil {
log.Warn("walker error", "err", err)
continue
}
rel, err := filepath.Rel(basepath, walker.Path())
if err != nil {
log.Warn("rel error", "err", err)
continue
}
if rel == "." {
continue
}
name := filepath.Base(walker.Path())
if walker.Stat().IsDir() {
matched := false
for _, skipDir := range skipDirs {
if skipDir.MatchString(name) {
walker.SkipDir()
matched = true
break
}
}
if matched {
continue
}
} else {
matched := false
for _, skipFile := range skipFiles {
if skipFile.MatchString(name) {
matched = true
break
}
}
if matched {
continue
}
}
stat := walker.Stat()
mine := file{
mode: stat.Mode(),
size: stat.Size(),
mod: stat.ModTime(),
path: walker.Path(),
relPath: rel,
}
if remote, ok := remoteMap[mine.relPath]; ok {
if !mine.mode.IsDir() && !remote.mode.IsDir() {
if mine.mod.After(remote.mod) {
puts = append(puts, mine)
} else if mine.size != remote.size {
if remote.size < mine.size && appendFile {
mine.offset = remote.size
}
puts = append(puts, mine)
}
} else if !mine.mode.IsDir() && remote.mode.IsDir() {
rmdirs = append(rmdirs, remote)
puts = append(puts, mine)
} else if mine.mode.IsDir() && !remote.mode.IsDir() {
rms = append(rms, remote)
}
delete(remoteMap, remote.relPath)
} else {
if mine.mode.IsDir() {
mkdirs = append(mkdirs, mine)
} else {
puts = append(puts, mine)
}
}
}
for _, remote := range remoteMap {
if remote.mode.IsDir() {
rmdirs = append(rmdirs, remote)
} else {
rms = append(rms, remote)
}
}
sort.Sort(fileByPath(mkdirs))
sort.Sort(fileByPath(puts))
sort.Reverse(fileByPath(rmdirs))
sort.Reverse(fileByPath(rms))
return rmdirs, rms, mkdirs, puts
}
// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
for _, x := range gl.GameList {
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
}
var wg sync.WaitGroup
results := make(chan *rom.GameXML, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
gl.Append(r)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if _, ok := existing[f]; ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && rh.KnownExt(r.Ext) {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}
func TestWalk(t *testing.T) {
makeTree(t)
errors := make([]error, 0, 10)
clear := true
markFn := func(walker *fs.Walker) (err error) {
for walker.Step() {
err = mark(walker.Path(), walker.Stat(), walker.Err(), &errors, clear)
if err != nil {
break
}
}
return err
}
// Expect no errors.
err := markFn(fs.Walk(tree.name))
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if len(errors) != 0 {
t.Fatalf("unexpected errors: %s", errors)
}
checkMarks(t, true)
errors = errors[0:0]
// Test permission errors. Only possible if we're not root
// and only on some file systems (AFS, FAT). To avoid errors during
// all.bash on those file systems, skip during go test -short.
if os.Getuid() > 0 && !testing.Short() {
// introduce 2 errors: chmod top-level directories to 0
os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0)
os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0)
// 3) capture errors, expect two.
// mark respective subtrees manually
markTree(tree.entries[1])
markTree(tree.entries[3])
// correct double-marking of directory itself
tree.entries[1].mark--
tree.entries[3].mark--
err := markFn(fs.Walk(tree.name))
if err != nil {
t.Fatalf("expected no error return from Walk, got %s", err)
}
if len(errors) != 2 {
t.Errorf("expected 2 errors, got %d: %s", len(errors), errors)
}
// the inaccessible subtrees were marked manually
checkMarks(t, true)
errors = errors[0:0]
// 4) capture errors, stop after first error.
// mark respective subtrees manually
markTree(tree.entries[1])
markTree(tree.entries[3])
// correct double-marking of directory itself
tree.entries[1].mark--
tree.entries[3].mark--
clear = false // error will stop processing
err = markFn(fs.Walk(tree.name))
if err == nil {
t.Fatalf("expected error return from Walk")
}
if len(errors) != 1 {
t.Errorf("expected 1 error, got %d: %s", len(errors), errors)
}
// the inaccessible subtrees were marked manually
checkMarks(t, false)
errors = errors[0:0]
// restore permissions
os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770)
os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770)
}
// cleanup
if err := os.RemoveAll(tree.name); err != nil {
t.Errorf("removeTree: %v", err)
}
}
func main() {
fn := "CDNUploader v0.1"
cloudAPI, err := googles.Login()
if err != nil {
panic(err)
}
fileIndex := map[string]string{}
indexBytes, err := ioutil.ReadFile(CDN_PROGRESS)
if err == nil {
if !theta.UnmarshalJSON(fn, indexBytes, &fileIndex) {
fmt.Println("FAILED TO LOAD EXISTING PROGRESS FILE:", CDN_PROGRESS)
fileIndex = map[string]string{}
}
} else {
fmt.Println("CANNOT FIND EXISTING PROGRESS FILE:", CDN_PROGRESS)
}
walker := fs.Walk(".")
for walker.Step() {
if err := walker.Err(); err != nil {
fmt.Println(err)
continue
}
if walker.Stat().IsDir() {
continue
}
objectName := walker.Path()
if objectName == "cdn.go" || objectName == "cdn.progress" || string(objectName[0]) == "." {
continue
}
// write to google cloud
b, err := ioutil.ReadFile(objectName)
if err != nil {
panic(err)
}
objectHash := theta.SHA1b(b)
if fileIndex[objectName] == objectHash {
fmt.Println(objectHash, "OK")
continue
}
n := strings.LastIndex(objectName, ".") + 1
var ct string
if n > -1 {
ct = objectName[n:]
}
var contentType string
switch ct {
case "css":
contentType = "text/css"
case "js":
contentType = "application/javascript"
default:
contentType = mime.TypeByExtension(objectName)
if len(contentType) == 0 || contentType == "application/octet-stream" {
contentType = http.DetectContentType(b)
}
}
err = cloudAPI.StorageWriter(true, contentType, "libs.leadinglocally.com", objectName, b)
if err != nil {
panic(err)
}
fileIndex[objectName] = objectHash
fmt.Println(contentType+" UPLOADED:", objectName)
}
ok, b := theta.MarshalJSON(fn, fileIndex)
for ok {
err := ioutil.WriteFile(CDN_PROGRESS, b, 0777)
if err != nil {
panic(err)
}
fmt.Println("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
fmt.Println("UPLOAD QUEUE COMPLETED...")
fmt.Println("______________________________________________")
return
}
fmt.Println("ERROR COMPLETING UPLOADS")
}
func (c *LintCmd) Execute(args []string) error {
if len(c.Args.Paths) == 0 {
c.Args.Paths = []string{"."}
}
issuec := make(chan string)
quitc := make(chan struct{})
go func() {
for {
select {
case issue := <-issuec:
fmt.Println(issue)
case <-quitc:
return
}
}
}()
lrepo, lrepoErr := openLocalRepo()
if lrepoErr != nil {
log.Printf("warning: while opening current dir's repo: %s", lrepoErr)
}
var wg sync.WaitGroup
for _, path := range c.Args.Paths {
w := fs.Walk(path)
for w.Step() {
if err := w.Err(); err != nil {
return err
}
if fi := w.Stat(); fi.Mode().IsRegular() {
suffix, typ := buildstore.DataType(fi.Name())
if suffix != "" {
absPath, err := filepath.Abs(w.Path())
if err != nil {
return err
}
var unitType, unitName string
if !c.NoCheckResolve {
if !strings.Contains(absPath, buildstore.BuildDataDirName) {
return fmt.Errorf("couldn't infer which source unit %s corresponds to, because its absolute path is not under any %s dir; either run with --no-check-resolve to skip checking that internal refs resolve to valid defs (which requires knowing what source unit each output file is from), or run 'src lint' against .srclib-cache or subdirectories of it", w.Path(), buildstore.BuildDataDirName)
}
unitType = strings.TrimSuffix(fi.Name(), "."+suffix+".json")
// Infer source unit name from file path (the
// path components after .srclib-cache until
// the basename).
pcs := strings.Split(absPath, string(filepath.Separator))
for i, pc := range pcs {
if pc == buildstore.BuildDataDirName && len(pcs) > i+2 {
unitName = filepath.Join(pcs[i+2 : len(pcs)-1]...)
break
}
}
}
var commitID string
if !c.NoCheckFiles {
// Infer commit ID from file path (the path component after .srclib-cache).
pcs := strings.Split(absPath, string(filepath.Separator))
for i, pc := range pcs {
if pc == buildstore.BuildDataDirName && len(pcs) > i+1 {
commitID = pcs[i+1]
break
}
}
}
if commitID == "" && !c.NoCheckFiles {
return fmt.Errorf("couldn't infer which commit ID %s was built from, which is necessary to check that file/dir fields refer to actual files; either run with --no-check-files to skip the file/dir check or pass paths that contain '.../.srclib-cache/COMMITID/...' (which allows this command to infer the commit ID)", w.Path())
}
// Ensure that commitID matches the local repo's commit ID.
if commitID != "" && !c.NoCheckFiles {
if lrepo != nil && lrepo.CommitID != commitID {
return fmt.Errorf("%s was built from commit %s, but the current repo working tree HEAD is commit %s; these must be the same to check that file/dir fields refer to actual files in the repo that their specific commit, so you must either (1) only run lint against build data files for commit %s; (2) run with --no-check-files to skip the file/dir check; or (3) check out commit %s in this repo", w.Path(), commitID, lrepo.CommitID, lrepo.CommitID, commitID)
}
}
checkFilesExist := !c.NoCheckFiles
wg.Add(1)
go func(path string) {
defer wg.Done()
var issues []string
var err error
switch typ.(type) {
case unit.SourceUnit:
issues, err = lintSourceUnit(lrepo.RootDir, path, checkFilesExist)
case *graph.Output:
issues, err = lintGraphOutput(lrepo.RootDir, c.Repo, unitType, unitName, path, checkFilesExist)
case []*dep.ResolvedDep:
issues, err = lintDepresolveOutput(lrepo.RootDir, path, checkFilesExist)
}
for _, issue := range prependLabelToStrings(path, issues) {
issuec <- issue
}
if err != nil {
log.Fatalf(redbg("ERR")+" %s: %s", path, err)
}
}(w.Path())
}
}
}
}
wg.Wait()
close(quitc)
return nil
}
// ScanProject
func ScanProject(dir string) ([]string, error) {
imports := map[string]bool{}
w := fs.Walk(dir)
for w.Step() {
fstat := w.Stat()
if fstat.IsDir() {
// check if that directory is "_vendor"
n := fstat.Name()
if n == "vendor" || n == "testdata" || []rune(n)[0] == '_' {
w.SkipDir()
continue
}
}
// check for errors
if w.Err() != nil {
return nil, w.Err()
}
fpath := w.Path()
// check the file is a .go file
if strings.HasSuffix(fpath, ".go") {
fset := token.NewFileSet()
// parse only the import declarations in the .go file
f, err := parser.ParseFile(fset, w.Path(), nil, parser.ImportsOnly)
if err != nil {
e := err.Error()
if strings.Contains(e, eofError) {
continue
}
if strings.Contains(e, importPathError) {
for _, i := range f.Imports {
if Valid(i.Path.Value) {
path, err := strconv.Unquote(i.Path.Value)
if err != nil {
return nil, err
}
imports[path] = true
continue
}
}
continue
}
return nil, err
}
// unquote the import path value
for _, i := range f.Imports {
path, err := strconv.Unquote(i.Path.Value)
if err != nil {
return nil, err
}
imports[path] = true
}
}
}
for path := range imports {
for _, exception := range Exceptions {
if Match(path, exception) {
imports[path] = false
}
}
}
paths := []string{}
for path, ok := range imports {
if ok {
paths = append(paths, path)
}
}
paths = FilterStdPkgs(paths)
projectpath, err := ImportPath(".")
if err != nil {
return nil, err
}
// filter out packages internal to the project
paths = strutil.RemovePrefixInStringSlice(projectpath, paths)
return paths, nil
}