func ParallelCompile(pkgs []*dag.Package) { var localDeps *stringset.StringSet var compiledDeps *stringset.StringSet var y, z, count int var parallel []*dag.Package var oldPkgFound bool = false var zeroFirst []*dag.Package localDeps = stringset.New() compiledDeps = stringset.New() for y = 0; y < len(pkgs); y++ { localDeps.Add(pkgs[y].Name) pkgs[y].ResetIndegree() } zeroFirst = make([]*dag.Package, len(pkgs)) for y = 0; y < len(pkgs); y++ { if pkgs[y].Indegree == 0 { zeroFirst[count] = pkgs[y] count++ } } for y = 0; y < len(pkgs); y++ { if pkgs[y].Indegree > 0 { zeroFirst[count] = pkgs[y] count++ } } parallel = make([]*dag.Package, 0) for y = 0; y < len(zeroFirst); { if !zeroFirst[y].Ready(localDeps, compiledDeps) { oldPkgFound = compileMultipe(parallel, oldPkgFound) for z = 0; z < len(parallel); z++ { compiledDeps.Add(parallel[z].Name) } parallel = make([]*dag.Package, 0) } else { parallel = append(parallel, zeroFirst[y]) y++ } } if len(parallel) > 0 { _ = compileMultipe(parallel, oldPkgFound) } }
func TestStringSet(t *testing.T) { ss := stringset.New() ss.Add("en") if ss.Len() != 1 { t.Fatal("stringset.Len() != 1\n") } ss.Add("to") if ss.Len() != 2 { t.Fatal("stringset.Len() != 2\n") } if !ss.Contains("en") { t.Fatal("! stringset.Contains('en')\n") } if !ss.Contains("to") { t.Fatal("! stringset.Contains('to')\n") } if ss.Contains("not here") { t.Fatal(" stringset.Contains('not here')\n") } }
func hasModifiedImports(fname string) (string, bool) { fileset := token.NewFileSet() mode := parser.ImportsOnly absSynTree, err := parser.ParseFile(fileset, fname, nil, mode) if err != nil { log.Fatalf("%s\n", err) } c := &collector{make([]string, 0)} ast.Walk(c, absSynTree) set := stringset.New() set.Add(`"os"`) set.Add(`"fmt"`) set.Add(`"io/ioutil"`) set.Add(`"regexp"`) set.Add(`"exec"`) set.Add(`"log"`) set.Add(`"flag"`) set.Add(`"path/filepath"`) for i := 0; i < len(c.deps); i++ { if !set.Contains(c.deps[i]) { return c.String(), true } } return "", false }
func newPackage() *Package { p := new(Package) p.Indegree = 0 p.Files = make([]string, 0) p.dependencies = stringset.New() p.children = make([]*Package, 0) return p }
func (d Dag) External() { var err os.Error var argv []string var tmp string var set *stringset.StringSet var i int = 0 set = stringset.New() for _, v := range d { for dep := range v.dependencies.Iter() { if !d.localDependency(dep) { set.Add(dep) } } } for u := range set.Iter() { if !seemsExternal(u) { set.Remove(u) } } argv = make([]string, 0) tmp, err = exec.LookPath("goinstall") if err != nil { log.Fatalf("[ERROR] %s\n", err) } argv = append(argv, tmp) if global.GetBool("-verbose") { argv = append(argv, "-v=true") } argv = append(argv, "-u=true") argv = append(argv, "-clean=true") i = len(argv) argv = append(argv, "dummy") for u := range set.Iter() { argv[i] = u if global.GetBool("-dryrun") { fmt.Printf("%s || exit 1\n", strings.Join(argv, " ")) } else { say.Printf("goinstall: %s\n", u) handy.StdExecve(argv, true) } } }
func newPackage() *Package { p := new(Package) p.Indegree = 0 p.Files = make([]string, 0) p.dependencies = stringset.New() p.children = make([]*Package, 0) p.waiter = nil p.needsCompile = false // yeah yeah.. p.lock = new(sync.Mutex) return p }
// SRCROOT variable is set during testing func TestWalker(t *testing.T) { walker.IncludeDir = func(s string) bool { _, dirname := filepath.Split(s) return dirname[0] != '.' } walker.IncludeFile = func(s string) bool { return strings.HasSuffix(s, ".go") && !strings.HasPrefix(s, "_") } srcroot := os.Getenv("SRCROOT") if srcroot == "" { t.Fatalf("$SRCROOT variable not set\n") } ss := stringset.New() // this is a bit static, will cause problems if // stuff is added or removed == not ideal.. ss.Add(filepath.Join(srcroot, "cmplr", "compiler.go")) ss.Add(filepath.Join(srcroot, "cmplr", "dag.go")) ss.Add(filepath.Join(srcroot, "cmplr", "gdmake.go")) ss.Add(filepath.Join(srcroot, "parse", "gopt.go")) ss.Add(filepath.Join(srcroot, "parse", "gopt_test.go")) ss.Add(filepath.Join(srcroot, "parse", "option.go")) ss.Add(filepath.Join(srcroot, "start", "main.go")) ss.Add(filepath.Join(srcroot, "utilz", "handy.go")) ss.Add(filepath.Join(srcroot, "utilz", "stringbuffer.go")) ss.Add(filepath.Join(srcroot, "utilz", "stringset.go")) ss.Add(filepath.Join(srcroot, "utilz", "utilz_test.go")) ss.Add(filepath.Join(srcroot, "utilz", "walker.go")) ss.Add(filepath.Join(srcroot, "utilz", "global.go")) ss.Add(filepath.Join(srcroot, "utilz", "timer.go")) ss.Add(filepath.Join(srcroot, "utilz", "say.go")) files := walker.PathWalk(filepath.Clean(srcroot)) // make sure stringset == files if len(files) != ss.Len() { t.Fatalf("walker.Len() != files.Len()\n") } for i := 0; i < len(files); i++ { if !ss.Contains(files[i]) { t.Fatalf("walker picked up files not in SRCROOT\n") } ss.Remove(files[i]) } }
func CreateLibArgv(pkgs []*dag.Package) { ss := stringset.New() for i := range pkgs { if len(pkgs[i].Name) > len(pkgs[i].ShortName) { ss.Add(pkgs[i].Name[:(len(pkgs[i].Name) - len(pkgs[i].ShortName))]) } } slice := ss.Slice() for i := 0; i < len(slice); i++ { slice[i] = filepath.Join(libroot, slice[i]) handy.DirOrMkdir(slice[i]) } CreateArgv(pkgs) }
func (d Dag) Alien() (set *stringset.StringSet) { set = stringset.New() for _, v := range d { for dep := range v.dependencies.Iter() { if !d.localDependency(dep) { set.Add(dep) } } } for u := range set.Iter() { if !seemsExternal(u) { set.Remove(u) } } return set }
func ForkLink(output string, pkgs []*dag.Package, extra []*dag.Package, up2date bool) { var mainPKG *dag.Package gotMain := make([]*dag.Package, 0) for i := 0; i < len(pkgs); i++ { if pkgs[i].ShortName == "main" { gotMain = append(gotMain, pkgs[i]) } } if len(gotMain) == 0 { log.Fatal("[ERROR] (linking) no main package found\n") } if len(gotMain) > 1 { choice := mainChoice(gotMain) mainPKG = gotMain[choice] } else { mainPKG = gotMain[0] } compiled := filepath.Join(libroot, mainPKG.Name) + suffix if up2date && !global.GetBool("-dryrun") && handy.IsFile(output) { if handy.ModifyTimestamp(compiled) < handy.ModifyTimestamp(output) { say.Printf("up 2 date: %s\n", output) return } } argv := make([]string, 0) argv = append(argv, pathLinker) switch global.GetString("-backend") { case "gc", "express": argv = append(argv, "-L") argv = append(argv, libroot) } argv = append(argv, "-o") argv = append(argv, output) // gcc get's this no matter what... if global.GetString("-backend") == "gcc" || global.GetString("-backend") == "gccgo" { argv = append(argv, "-static") } else if global.GetBool("-static") { argv = append(argv, "-d") } switch global.GetString("-backend") { case "gccgo", "gcc": walker.IncludeFile = func(s string) bool { return strings.HasSuffix(s, ".o") } walker.IncludeDir = func(s string) bool { return true } for y := 0; y < len(includes); y++ { argv = append(argv, walker.PathWalk(includes[y])...) } case "gc", "express": for y := 0; y < len(includes); y++ { argv = append(argv, "-L") argv = append(argv, includes[y]) } } argv = append(argv, compiled) if global.GetString("-backend") == "gcc" || global.GetString("-backend") == "gccgo" { ss := stringset.New() if len(extra) > 0 { for j := 0; j < len(extra); j++ { // main package untestable using GCC if extra[j].ShortName != "main" { ss.Add(filepath.Join(libroot, extra[j].Name) + suffix) } } } else { for k := 0; k < len(pkgs); k++ { ss.Add(filepath.Join(libroot, pkgs[k].Name) + suffix) } ss.Remove(compiled) } if ss.Len() > 0 { argv = append(argv, ss.Slice()...) } } if global.GetBool("-dryrun") { linker := filepath.Base(pathLinker) fmt.Printf("%s %s || exit 1\n", linker, strings.Join(argv[1:], " ")) } else { say.Println("linking :", output) handy.StdExecve(argv, true) } }
func (d Dag) MakeMainTest(root string) ([]*Package, string, string) { var max, i int var isTest bool var lname, sname, tmplib, tmpdir, tmpstub, tmpfile string sbImports := stringbuffer.NewSize(300) imprtSet := stringset.New() sbTests := stringbuffer.NewSize(1000) sbBench := stringbuffer.NewSize(1000) sbImports.Add("\n// autogenerated code\n\n") sbImports.Add("package main\n\n") imprtSet.Add("import \"regexp\"\n") imprtSet.Add("import \"testing\"\n") sbTests.Add("\n\nvar tests = []testing.InternalTest{\n") sbBench.Add("\n\nvar benchmarks = []testing.InternalBenchmark{\n") for _, v := range d { isTest = false sname = v.ShortName lname = v.ShortName max = len(v.ShortName) if max > 5 && sname[max-5:] == "_test" { collector := newTestCollector() for i = 0; i < len(v.Files); i++ { tree := getSyntaxTreeOrDie(v.Files[i], 0) ast.Walk(collector, tree) } if len(collector.Names) > 0 { isTest = true if hasSlash(v.Name) { lname = removeSlashAndDot(v.Name) imprtSet.Add(fmt.Sprintf("import %s \"%s\"\n", lname, v.Name)) } else { imprtSet.Add(fmt.Sprintf("import \"%s\"\n", v.Name)) } for i = 0; i < len(collector.Names); i++ { testFunc := collector.Names[i] if len(testFunc) >= 4 && testFunc[0:4] == "Test" { sbTests.Add(fmt.Sprintf("testing.InternalTest{\"%s.%s\", %s.%s },\n", sname, testFunc, lname, testFunc)) } else if len(testFunc) >= 9 && testFunc[0:9] == "Benchmark" { sbBench.Add(fmt.Sprintf("testing.InternalBenchmark{\"%s.%s\", %s.%s },\n", sname, testFunc, lname, testFunc)) } } } } if !isTest { collector := newTestCollector() for i = 0; i < len(v.Files); i++ { fname := v.Files[i] if len(fname) > 8 && fname[len(fname)-8:] == "_test.go" { tree := getSyntaxTreeOrDie(fname, 0) ast.Walk(collector, tree) } } if len(collector.Names) > 0 { if hasSlash(v.Name) { lname = removeSlashAndDot(v.Name) imprtSet.Add(fmt.Sprintf("import %s \"%s\"\n", lname, v.Name)) } else { imprtSet.Add(fmt.Sprintf("import \"%s\"\n", v.Name)) } for i = 0; i < len(collector.Names); i++ { testFunc := collector.Names[i] if len(testFunc) >= 4 && testFunc[0:4] == "Test" { sbTests.Add(fmt.Sprintf("testing.InternalTest{\"%s.%s\", %s.%s },\n", sname, testFunc, lname, testFunc)) } else if len(testFunc) >= 9 && testFunc[0:9] == "Benchmark" { sbBench.Add(fmt.Sprintf("testing.InternalBenchmark{\"%s.%s\", %s.%s },\n", sname, testFunc, lname, testFunc)) } } } } } sbTests.Add("};\n") sbBench.Add("};\n\n") for im := range imprtSet.Iter() { sbImports.Add(im) } sbTotal := stringbuffer.NewSize(sbImports.Len() + sbTests.Len() + sbBench.Len() + 100) sbTotal.Add(sbImports.String()) sbTotal.Add(sbTests.String()) sbTotal.Add(sbBench.String()) sbTotal.Add("func main(){\n") sbTotal.Add("testing.Main(regexp.MatchString, tests, benchmarks);\n}\n\n") tmpstub = fmt.Sprintf("tmp%d", time.Seconds()) tmpdir = filepath.Join(root, tmpstub) if global.GetString("-lib") != "" { tmplib = filepath.Join(global.GetString("-lib"), tmpstub) } dir, e1 := os.Stat(tmpdir) if e1 == nil && dir.IsDirectory() { log.Printf("[ERROR] directory: %s already exists\n", tmpdir) } else { e_mk := os.Mkdir(tmpdir, 0777) if e_mk != nil { log.Fatal("[ERROR] failed to create directory for testing") } } tmpfile = filepath.Join(tmpdir, "_main.go") fil, e2 := os.OpenFile(tmpfile, os.O_WRONLY|os.O_CREATE, 0777) if e2 != nil { log.Fatalf("[ERROR] %s\n", e2) } n, e3 := fil.WriteString(sbTotal.String()) if e3 != nil { log.Fatalf("[ERROR] %s\n", e3) } else if n != sbTotal.Len() { log.Fatal("[ERROR] failed to write test") } fil.Close() p := newPackage() p.Name = filepath.Join(tmpstub, "main") p.ShortName = "main" p.Files = append(p.Files, tmpfile) vec := make([]*Package, 1) vec[0] = p return vec, tmpdir, tmplib }
func (d Dag) MakeMainTest(root string) ([]*Package, string, string) { var ( i int lname string sname string tmplib string tmpdir string tmpstub string tmpfile string collector *TestCollector ) sbImports := stringbuffer.NewSize(300) imprtSet := stringset.New() sbTests := stringbuffer.NewSize(1000) sbBench := stringbuffer.NewSize(1000) sbExample := stringbuffer.NewSize(1000) sbImports.Add("\n// autogenerated code\n\n") sbImports.Add("package main\n\n") imprtSet.Add("import \"regexp\"\n") imprtSet.Add("import \"testing\"\n") sbTests.Add("\n\nvar tests = []testing.InternalTest{\n") sbBench.Add("\n\nvar benchmarks = []testing.InternalBenchmark{\n") sbExample.Add("\n\nvar examples = []testing.InternalExample{\n") for _, v := range d { sname = v.ShortName lname = v.ShortName collector = newTestCollector() if strings.HasSuffix(v.ShortName, "_test") { for i = 0; i < len(v.Files); i++ { tree := getSyntaxTreeOrDie(v.Files[i], parser.ParseComments) ast.Walk(collector, tree) } } else { for i = 0; i < len(v.Files); i++ { if strings.HasSuffix(v.Files[i], "_test.go") { tree := getSyntaxTreeOrDie(v.Files[i], parser.ParseComments) ast.Walk(collector, tree) } } } if collector.FoundAnything() { if hasSlash(v.Name) { lname = removeSlashAndDot(v.Name) imprtSet.Add(fmt.Sprintf("import %s \"%s\"\n", lname, v.Name)) } else { imprtSet.Add(fmt.Sprintf("import \"%s\"\n", v.Name)) } // add tests for i := 0; i < len(collector.TestFuncs); i++ { fn := collector.TestFuncs[i] sbTests.Add(fmt.Sprintf( "testing.InternalTest{\"%s.%s\", %s.%s },\n", sname, fn, lname, fn)) } // add benchmarks for i := 0; i < len(collector.BenchFuncs); i++ { fn := collector.BenchFuncs[i] sbBench.Add(fmt.Sprintf( "testing.InternalBenchmark{\"%s.%s\", %s.%s },\n", sname, fn, lname, fn)) } // add examples ( not really ) for i := 0; i < len(collector.ExampleFuncs); i++ { // fn := collector.ExampleFuncs[i] //TODO add comment which seems to be what we compare against.. // sbExample.Add(fmt.Sprintf("testing.InternalExample{\"%s.%s\", %s.%s },\n", sname, fn, lname, fn)) } } } sbTests.Add("};\n") sbBench.Add("};\n") sbExample.Add("};\n") for im := range imprtSet.Iter() { sbImports.Add(im) } sbTotal := stringbuffer.NewSize(sbImports.Len() + sbTests.Len() + sbBench.Len() + 100) sbTotal.Add(sbImports.String()) sbTotal.Add(sbTests.String()) sbTotal.Add(sbBench.String()) sbTotal.Add(sbExample.String()) sbTotal.Add("func main(){\n") sbTotal.Add("testing.Main(regexp.MatchString, tests, benchmarks, examples);\n}\n\n") tmpstub = fmt.Sprintf("tmp%d", time.Now().Unix()) tmpdir = filepath.Join(root, tmpstub) if global.GetString("-lib") != "" { tmplib = filepath.Join(global.GetString("-lib"), tmpstub) } dir, e1 := os.Stat(tmpdir) if e1 == nil && dir.IsDir() { log.Printf("[ERROR] directory: %s already exists\n", tmpdir) } else { e_mk := os.Mkdir(tmpdir, 0777) if e_mk != nil { log.Fatal("[ERROR] failed to create directory for testing") } } tmpfile = filepath.Join(tmpdir, "_main.go") fil, e2 := os.OpenFile(tmpfile, os.O_WRONLY|os.O_CREATE, 0777) if e2 != nil { log.Fatalf("[ERROR] %s\n", e2) } n, e3 := fil.WriteString(sbTotal.String()) if e3 != nil { log.Fatalf("[ERROR] %s\n", e3) } else if n != sbTotal.Len() { log.Fatal("[ERROR] failed to write test") } fil.Close() p := newPackage() p.Name = filepath.Join(tmpstub, "main") p.ShortName = "main" p.Files = append(p.Files, tmpfile) vec := make([]*Package, 1) vec[0] = p return vec, tmpdir, tmplib }