Exemple #1
0
func TestSingleParse(t *testing.T) {
	// Test Paragraph {{{
	reader := bufio.NewReader(strings.NewReader(`Foo: bar
Bar-Baz: fnord:and:this:here
 and:not
 second:here
 line
 here
Hello: world

But-not: me
`))
	// }}}
	deb822, err := control.ParseParagraph(reader)
	isok(t, err)
	assert(t, deb822 != nil)

	assert(t, len(deb822.Order) == 3)
	assert(t, len(deb822.Order) == len(deb822.Values))
	assert(t, deb822.Values["Foo"] == "bar")

	if _, ok := deb822.Values["But-not"]; ok {
		/* In the case where we *have* this key; let's abort this guy
		 * quickly. */
		t.FailNow()
	}

	deb822, err = control.ParseParagraph(reader)
	isok(t, err)
	assert(t, deb822 != nil)

	assert(t, len(deb822.Order) == 1)
	assert(t, len(deb822.Order) == len(deb822.Values))
	assert(t, deb822.Values["But-not"] == "me")
}
Exemple #2
0
func TestBasicControlParse(t *testing.T) {
	reader := bufio.NewReader(strings.NewReader(`Foo: bar
`))
	deb822, err := control.ParseParagraph(reader)
	isok(t, err)
	assert(t, deb822 != nil)

	assert(t, len(deb822.Order) == 1)
	assert(t, len(deb822.Order) == len(deb822.Values))
	assert(t, deb822.Values["Foo"] == "bar")

	reader = bufio.NewReader(strings.NewReader(`Foo: bar`))
	deb822, err = control.ParseParagraph(reader)
	assert(t, deb822 == nil)
	assert(t, err == io.EOF)
}
Exemple #3
0
func LoadConfig(basedir string) (*NMRConfig, error) {
	ret := NMRConfig{
		Basedir: basedir,
	}

	file, err := os.Open(basedir + "/conf/nmr")
	if err != nil {
		return nil, err
	}
	reader := bufio.NewReader(file)
	global, err := control.ParseParagraph(reader)
	if err != nil {
		return nil, err
	}

	ret.Global = GlobalConfig{
		PublicArchiveRoot:  global.Values["PublicArchiveRoot"],
		ArchIndepBuildArch: global.Values["ArchIndepBuildArch"],
	}

	for {
		para, err := control.ParseParagraph(reader)
		if err != nil {
			return nil, err
		}
		if para == nil {
			break
		}
		upstream := strings.Split(para.Values["Upstream"], " ")

		ret.Blocks = append(ret.Blocks, DistConfig{
			Names:          strings.Split(para.Values["Name"], " "),
			UpstreamArches: strings.Split(para.Values["UpstreamArches"], " "),
			Arches:         strings.Split(para.Values["Arches"], " "),
			Incoming:       para.Values["Incoming"],
			Upstream: UpstreamLocation{
				Root: upstream[0],
				Dist: upstream[1], // FIXME
			},
			Schroot: para.Values["Schroot"],
		})
	}

	return &ret, nil
}
Exemple #4
0
func TestFunControlParse(t *testing.T) {
	reader := bufio.NewReader(strings.NewReader(`Foo: bar
Bar-Baz: fnord:and:this:here
 and:not
 second:here
 line
 here
`))
	deb822, err := control.ParseParagraph(reader)
	isok(t, err)
	assert(t, deb822 != nil)

	assert(t, len(deb822.Order) == 2)
	assert(t, len(deb822.Order) == len(deb822.Values))
	assert(t, deb822.Values["Foo"] == "bar")
}
Exemple #5
0
func LoadDistributions(basedir string) (*Distributions, error) {
	ret := Distributions{}

	file, err := os.Open(basedir + "/conf/distributions")
	if err != nil {
		return nil, err
	}
	reader := bufio.NewReader(file)

	for {
		para, err := control.ParseParagraph(reader)
		if err != nil {
			return nil, err
		}
		if para == nil {
			break
		}

		arches := []string{}
		source := false

		for _, arch := range strings.Split(para.Values["Architectures"], " ") {
			if arch == "source" {
				source = true
				continue
			}
			arches = append(arches, arch)
		}

		ret.Blocks = append(ret.Blocks, Distribution{
			Codename:      para.Values["Codename"],
			Suite:         para.Values["Suite"],
			Components:    strings.Split(para.Values["Components"], " "),
			Architectures: arches,
			Source:        source,
			Tracking:      strings.Split(para.Values["Tracking"], " "),
			SignWith:      para.Values["SignWith"],
		})
	}

	return &ret, nil
}
Exemple #6
0
func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil && err != io.EOF {
		log.Fatal(err)
	}

	binaries := make(map[string]bool)
	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}
	for _, binary := range changes.Binaries {
		binaries[binary] = true
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	var sourcesPaths []string
	var packagesPaths []string
	indexTargets := exec.Command("apt-get",
		"indextargets",
		"--format",
		"$(FILENAME)",
		"Codename: sid",
		"ShortDesc: Sources")
	if lines, err := indexTargets.Output(); err == nil {
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				sourcesPaths = append(sourcesPaths, line)
			}
		}
		binaryIndexTargets := exec.Command(
			"apt-get",
			"indextargets",
			"--format",
			"$(FILENAME)",
			"Codename: sid",
			"ShortDesc: Packages")
		lines, err = binaryIndexTargets.Output()
		if err != nil {
			log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err)
		}
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				packagesPaths = append(packagesPaths, line)
			}
		}
	} else {
		// Fallback for older versions of apt-get. See
		// https://bugs.debian.org/801594 for context.
		releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
		if err != nil {
			log.Fatal(err)
		}
		for _, releasepath := range releaseMatches {
			r, err := os.Open(releasepath)
			if err != nil {
				log.Fatal(err)
			}
			defer r.Close()
			release, err := control.ParseParagraph(bufio.NewReader(r))
			if err != nil && err != io.EOF {
				log.Fatal(err)
			}
			if release.Values["Suite"] != "unstable" {
				continue
			}

			listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
			if len(listsPrefix) != 2 {
				log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
			}
			sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			sourcesPaths = append(sourcesPaths, sourceMatches...)
			packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			packagesPaths = append(packagesPaths, packagesMatches...)
		}
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?")
	}

	rebuild := make(map[string][]version.Version)

	archCmd := exec.Command(
		"dpkg-architecture",
		"--query=DEB_BUILD_ARCH")
	archOut, err := archCmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	arch := strings.TrimSpace(string(archOut))

	// TODO: Cache this output based on the .changes file. dose-ceve takes quite a while.
	ceve := exec.Command(
		"dose-ceve",
		"--deb-native-arch="+arch,
		"-T", "debsrc",
		"-r", strings.Join(changes.Binaries, ","),
		"-G", "pkg")
	for _, packagesPath := range packagesPaths {
		ceve.Args = append(ceve.Args, "deb://"+packagesPath)
	}
	for _, sourcesPath := range sourcesPaths {
		ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath)
	}

	log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while\n")
	if out, err := ceve.Output(); err == nil {
		r := bufio.NewReader(strings.NewReader(string(out)))
		for {
			paragraph, err := control.ParseParagraph(r)
			if paragraph == nil || err == io.EOF {
				break
			}
			pkg := paragraph.Values["Package"]
			ver, err := version.Parse(paragraph.Values["Version"])
			if err != nil {
				log.Fatalf("Cannot parse version number %q in dose-ceve(1) output: %v", paragraph.Values["Version"], err)
			}
			rebuild[pkg] = append(rebuild[pkg], ver)
		}
	} else {
		log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly\n", err)
		for _, sourcesPath := range sourcesPaths {
			if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil {
				log.Fatal(err)
			}
		}
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	if strings.TrimSpace(*sbuildDist) == "" {
		*sbuildDist = changes.Distribution
		log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist)
	}

	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=" + *sbuildDist,
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building %s (commandline: %v)\n", target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}
Exemple #7
0
func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil {
		log.Fatal(err)
	}

	binaries := make(map[string]bool)
	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}
	for _, binary := range changes.Binaries {
		binaries[binary] = true
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	var sourcesPaths []string
	releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
	if err != nil {
		log.Fatal(err)
	}
	for _, releasepath := range releaseMatches {
		r, err := os.Open(releasepath)
		if err != nil {
			log.Fatal(err)
		}
		defer r.Close()
		release, err := control.ParseParagraph(bufio.NewReader(r))
		if err != nil {
			log.Fatal(err)
		}
		if release.Values["Suite"] != "unstable" {
			continue
		}

		listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
		if len(listsPrefix) != 2 {
			log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
		}
		sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
		if err != nil {
			log.Fatal(err)
		}
		sourcesPaths = append(sourcesPaths, sourceMatches...)
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?")
	}

	rebuild := make(map[string][]version.Version)

	for _, sourcesPath := range sourcesPaths {
		if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil {
			log.Fatal(err)
		}
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=sid",
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building %s (commandline: %v)\n", target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}