func TestSingleParse(t *testing.T) { // Test Paragraph {{{ reader := bufio.NewReader(strings.NewReader(`Foo: bar Bar-Baz: fnord:and:this:here and:not second:here line here Hello: world But-not: me `)) // }}} deb822, err := control.ParseParagraph(reader) isok(t, err) assert(t, deb822 != nil) assert(t, len(deb822.Order) == 3) assert(t, len(deb822.Order) == len(deb822.Values)) assert(t, deb822.Values["Foo"] == "bar") if _, ok := deb822.Values["But-not"]; ok { /* In the case where we *have* this key; let's abort this guy * quickly. */ t.FailNow() } deb822, err = control.ParseParagraph(reader) isok(t, err) assert(t, deb822 != nil) assert(t, len(deb822.Order) == 1) assert(t, len(deb822.Order) == len(deb822.Values)) assert(t, deb822.Values["But-not"] == "me") }
func TestBasicControlParse(t *testing.T) { reader := bufio.NewReader(strings.NewReader(`Foo: bar `)) deb822, err := control.ParseParagraph(reader) isok(t, err) assert(t, deb822 != nil) assert(t, len(deb822.Order) == 1) assert(t, len(deb822.Order) == len(deb822.Values)) assert(t, deb822.Values["Foo"] == "bar") reader = bufio.NewReader(strings.NewReader(`Foo: bar`)) deb822, err = control.ParseParagraph(reader) assert(t, deb822 == nil) assert(t, err == io.EOF) }
func LoadConfig(basedir string) (*NMRConfig, error) { ret := NMRConfig{ Basedir: basedir, } file, err := os.Open(basedir + "/conf/nmr") if err != nil { return nil, err } reader := bufio.NewReader(file) global, err := control.ParseParagraph(reader) if err != nil { return nil, err } ret.Global = GlobalConfig{ PublicArchiveRoot: global.Values["PublicArchiveRoot"], ArchIndepBuildArch: global.Values["ArchIndepBuildArch"], } for { para, err := control.ParseParagraph(reader) if err != nil { return nil, err } if para == nil { break } upstream := strings.Split(para.Values["Upstream"], " ") ret.Blocks = append(ret.Blocks, DistConfig{ Names: strings.Split(para.Values["Name"], " "), UpstreamArches: strings.Split(para.Values["UpstreamArches"], " "), Arches: strings.Split(para.Values["Arches"], " "), Incoming: para.Values["Incoming"], Upstream: UpstreamLocation{ Root: upstream[0], Dist: upstream[1], // FIXME }, Schroot: para.Values["Schroot"], }) } return &ret, nil }
func TestFunControlParse(t *testing.T) { reader := bufio.NewReader(strings.NewReader(`Foo: bar Bar-Baz: fnord:and:this:here and:not second:here line here `)) deb822, err := control.ParseParagraph(reader) isok(t, err) assert(t, deb822 != nil) assert(t, len(deb822.Order) == 2) assert(t, len(deb822.Order) == len(deb822.Values)) assert(t, deb822.Values["Foo"] == "bar") }
func LoadDistributions(basedir string) (*Distributions, error) { ret := Distributions{} file, err := os.Open(basedir + "/conf/distributions") if err != nil { return nil, err } reader := bufio.NewReader(file) for { para, err := control.ParseParagraph(reader) if err != nil { return nil, err } if para == nil { break } arches := []string{} source := false for _, arch := range strings.Split(para.Values["Architectures"], " ") { if arch == "source" { source = true continue } arches = append(arches, arch) } ret.Blocks = append(ret.Blocks, Distribution{ Codename: para.Values["Codename"], Suite: para.Values["Suite"], Components: strings.Split(para.Values["Components"], " "), Architectures: arches, Source: source, Tracking: strings.Split(para.Values["Tracking"], " "), SignWith: para.Values["SignWith"], }) } return &ret, nil }
func main() { flag.Parse() if flag.NArg() != 1 { log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0]) } changesPath := flag.Arg(0) log.Printf("Loading changes file %q\n", changesPath) c, err := os.Open(changesPath) if err != nil { log.Fatal(err) } defer c.Close() changes, err := control.ParseChanges(bufio.NewReader(c), changesPath) if err != nil && err != io.EOF { log.Fatal(err) } binaries := make(map[string]bool) var debs []string for _, file := range changes.Files { if filepath.Ext(file.Filename) == ".deb" { debs = append(debs, file.Filename) } } for _, binary := range changes.Binaries { binaries[binary] = true } log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " ")) log.Printf(" - corresponding .debs (will be injected when building):\n") for _, deb := range debs { log.Printf(" %s\n", deb) } var sourcesPaths []string var packagesPaths []string indexTargets := exec.Command("apt-get", "indextargets", "--format", "$(FILENAME)", "Codename: sid", "ShortDesc: Sources") if lines, err := indexTargets.Output(); err == nil { for _, line := range strings.Split(string(lines), "\n") { trimmed := strings.TrimSpace(line) if trimmed != "" { sourcesPaths = append(sourcesPaths, line) } } binaryIndexTargets := exec.Command( "apt-get", "indextargets", "--format", "$(FILENAME)", "Codename: sid", "ShortDesc: Packages") lines, err = binaryIndexTargets.Output() if err != nil { log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err) } for _, line := range strings.Split(string(lines), "\n") { trimmed := strings.TrimSpace(line) if trimmed != "" { packagesPaths = append(packagesPaths, line) } } } else { // Fallback for older versions of apt-get. See // https://bugs.debian.org/801594 for context. releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease") if err != nil { log.Fatal(err) } for _, releasepath := range releaseMatches { r, err := os.Open(releasepath) if err != nil { log.Fatal(err) } defer r.Close() release, err := control.ParseParagraph(bufio.NewReader(r)) if err != nil && err != io.EOF { log.Fatal(err) } if release.Values["Suite"] != "unstable" { continue } listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath) if len(listsPrefix) != 2 { log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe) } sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1])) if err != nil { log.Fatal(err) } sourcesPaths = append(sourcesPaths, sourceMatches...) packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1])) if err != nil { log.Fatal(err) } packagesPaths = append(packagesPaths, packagesMatches...) } } if len(sourcesPaths) == 0 { log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?") } rebuild := make(map[string][]version.Version) archCmd := exec.Command( "dpkg-architecture", "--query=DEB_BUILD_ARCH") archOut, err := archCmd.Output() if err != nil { log.Fatal(err) } arch := strings.TrimSpace(string(archOut)) // TODO: Cache this output based on the .changes file. dose-ceve takes quite a while. ceve := exec.Command( "dose-ceve", "--deb-native-arch="+arch, "-T", "debsrc", "-r", strings.Join(changes.Binaries, ","), "-G", "pkg") for _, packagesPath := range packagesPaths { ceve.Args = append(ceve.Args, "deb://"+packagesPath) } for _, sourcesPath := range sourcesPaths { ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath) } log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while\n") if out, err := ceve.Output(); err == nil { r := bufio.NewReader(strings.NewReader(string(out))) for { paragraph, err := control.ParseParagraph(r) if paragraph == nil || err == io.EOF { break } pkg := paragraph.Values["Package"] ver, err := version.Parse(paragraph.Values["Version"]) if err != nil { log.Fatalf("Cannot parse version number %q in dose-ceve(1) output: %v", paragraph.Values["Version"], err) } rebuild[pkg] = append(rebuild[pkg], ver) } } else { log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly\n", err) for _, sourcesPath := range sourcesPaths { if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil { log.Fatal(err) } } } // TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time). // TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable. if strings.TrimSpace(*sbuildDist) == "" { *sbuildDist = changes.Distribution log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist) } buildresults := make(map[string]bool) for src, versions := range rebuild { sort.Sort(sort.Reverse(version.Slice(versions))) newest := versions[0] target := fmt.Sprintf("%s_%s", src, newest) // TODO: discard resulting package immediately? args := []string{ "--arch-all", "--dist=" + *sbuildDist, "--nolog", target, } for _, filename := range debs { args = append(args, fmt.Sprintf("--extra-package=%s", filename)) } cmd := exec.Command("sbuild", args...) if err := os.MkdirAll(*logDir, 0755); err != nil { log.Fatal(err) } log.Printf("Building %s (commandline: %v)\n", target, cmd.Args) if *dryRun { continue } buildlog, err := os.Create(filepath.Join(*logDir, target)) if err != nil { log.Fatal(err) } defer buildlog.Close() cmd.Stdout = buildlog cmd.Stderr = buildlog if err := cmd.Run(); err != nil { log.Printf("building %s failed: %v\n", target, err) buildresults[target] = false } else { buildresults[target] = true } } log.Printf("Build results:\n") // Print all successful builds first (not as interesting), then failed ones. for target, result := range buildresults { if !result { continue } log.Printf("PASSED: %s\n", target) } for target, result := range buildresults { if result { continue } log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target)) } }
func main() { flag.Parse() if flag.NArg() != 1 { log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0]) } changesPath := flag.Arg(0) log.Printf("Loading changes file %q\n", changesPath) c, err := os.Open(changesPath) if err != nil { log.Fatal(err) } defer c.Close() changes, err := control.ParseChanges(bufio.NewReader(c), changesPath) if err != nil { log.Fatal(err) } binaries := make(map[string]bool) var debs []string for _, file := range changes.Files { if filepath.Ext(file.Filename) == ".deb" { debs = append(debs, file.Filename) } } for _, binary := range changes.Binaries { binaries[binary] = true } log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " ")) log.Printf(" - corresponding .debs (will be injected when building):\n") for _, deb := range debs { log.Printf(" %s\n", deb) } var sourcesPaths []string releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease") if err != nil { log.Fatal(err) } for _, releasepath := range releaseMatches { r, err := os.Open(releasepath) if err != nil { log.Fatal(err) } defer r.Close() release, err := control.ParseParagraph(bufio.NewReader(r)) if err != nil { log.Fatal(err) } if release.Values["Suite"] != "unstable" { continue } listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath) if len(listsPrefix) != 2 { log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe) } sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1])) if err != nil { log.Fatal(err) } sourcesPaths = append(sourcesPaths, sourceMatches...) } if len(sourcesPaths) == 0 { log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?") } rebuild := make(map[string][]version.Version) for _, sourcesPath := range sourcesPaths { if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil { log.Fatal(err) } } // TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time). // TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable. buildresults := make(map[string]bool) for src, versions := range rebuild { sort.Sort(sort.Reverse(version.Slice(versions))) newest := versions[0] target := fmt.Sprintf("%s_%s", src, newest) // TODO: discard resulting package immediately? args := []string{ "--arch-all", "--dist=sid", "--nolog", target, } for _, filename := range debs { args = append(args, fmt.Sprintf("--extra-package=%s", filename)) } cmd := exec.Command("sbuild", args...) if err := os.MkdirAll(*logDir, 0755); err != nil { log.Fatal(err) } log.Printf("Building %s (commandline: %v)\n", target, cmd.Args) if *dryRun { continue } buildlog, err := os.Create(filepath.Join(*logDir, target)) if err != nil { log.Fatal(err) } defer buildlog.Close() cmd.Stdout = buildlog cmd.Stderr = buildlog if err := cmd.Run(); err != nil { log.Printf("building %s failed: %v\n", target, err) buildresults[target] = false } else { buildresults[target] = true } } log.Printf("Build results:\n") // Print all successful builds first (not as interesting), then failed ones. for target, result := range buildresults { if !result { continue } log.Printf("PASSED: %s\n", target) } for target, result := range buildresults { if result { continue } log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target)) } }