func TestArchBasics(t *testing.T) {
arch, err := dependency.ParseArch("amd64")
isok(t, err)
assert(t, arch.CPU == "amd64")
assert(t, arch.ABI == "gnu")
assert(t, arch.OS == "linux")
}
func ComputeBuildStatus(
repo reprepro.Repo,
index resolver.Candidates,
packages []reprepro.BuildNeedingPackage,
) []BuildStatus {
ret := []BuildStatus{}
for _, pkg := range packages {
dsc, err := control.ParseDscFile(repo.Basedir + "/" + pkg.Location)
if err != nil {
continue
}
arch, err := dependency.ParseArch(pkg.Arch)
if err != nil {
/// XXX: ERROR OUT
continue
}
buildable, why := index.ExplainSatisfiesBuildDepends(*arch, dsc.BuildDepends)
ret = append(ret, BuildStatus{
Package: pkg,
Buildable: buildable,
Why: why,
})
}
return ret
}
func TestResolverVersion(t *testing.T) {
arch, err := dependency.ParseArch("amd64")
isok(t, err)
candidates, err := resolver.ReadFromBinaryIndex(
strings.NewReader(testBinaryIndex),
)
isok(t, err)
assert(t, len(*candidates) == 3)
dep, err := dependency.Parse("android-tools-fsutils (>= 1.0)")
isok(t, err)
possi := dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
dep, err = dependency.Parse("android-tools-fsutils (>= 1:1.0)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils (<= 1:1.0)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
dep, err = dependency.Parse("android-tools-fsutils (<= 0:0)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils (= 4.2.2+git20130529-5.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
dep, err = dependency.Parse("android-tools-fsutils (= 2.2.2+git20130529-5.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils (<< 4.2.2+git20130529-5.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils (<< 4.2.2+git20130529-6.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
dep, err = dependency.Parse("android-tools-fsutils (>> 4.2.2+git20130529-5.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils (>> 4.2.2+git20130529-4.1)")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
}
func TestArchSetCompare(t *testing.T) {
dep, err := dependency.Parse("foo [amd64], bar [!sparc]")
isok(t, err)
iAm, err := dependency.ParseArch("amd64")
isok(t, err)
fooArch := dep.Relations[0].Possibilities[0].Architectures
barArch := dep.Relations[1].Possibilities[0].Architectures
assert(t, fooArch.Matches(iAm))
assert(t, barArch.Matches(iAm))
iAmNot, err := dependency.ParseArch("armhf")
isok(t, err)
assert(t, !fooArch.Matches(iAmNot))
assert(t, barArch.Matches(iAmNot))
}
func TestArchSliceParse(t *testing.T) {
dep, err := dependency.Parse("foo, bar [sparc] | baz")
isok(t, err)
arch, err := dependency.ParseArch("amd64")
isok(t, err)
els := dep.GetPossibilities(*arch)
assert(t, len(els) == 2)
assert(t, els[0].Name == "foo")
assert(t, els[1].Name == "baz")
}
func TestArchCompareBasics(t *testing.T) {
arch, err := dependency.ParseArch("amd64")
isok(t, err)
equivs := []string{
"gnu-linux-amd64",
"linux-amd64",
"linux-any",
"any",
"gnu-linux-any",
}
for _, el := range equivs {
other, err := dependency.ParseArch(el)
isok(t, err)
assert(t, arch.Is(other))
assert(t, other.Is(arch))
}
unequivs := []string{
"gnu-linux-all",
"all",
"gnuu-linux-amd64",
"gnu-linuxx-amd64",
"gnu-linux-amd644",
}
for _, el := range unequivs {
other, err := dependency.ParseArch(el)
isok(t, err)
assert(t, !arch.Is(other))
assert(t, !other.Is(arch))
}
}
func TestArchString(t *testing.T) {
equivs := map[string]string{
"all": "all",
"any": "all",
"amd64": "amd64",
"gnu-linux-amd64": "amd64",
"bsd-windows-i386": "bsd-windows-i386",
}
for _, el := range equivs {
arch, err := dependency.ParseArch(el)
isok(t, err)
assert(t, arch.String() == equivs[el])
}
}
func TestResolverDependsVersion(t *testing.T) {
candidates, err := resolver.ReadFromBinaryIndex(
strings.NewReader(testBinaryIndex),
)
isok(t, err)
assert(t, len(*candidates) == 3)
arch, err := dependency.ParseArch("amd64")
isok(t, err)
dep, err := dependency.Parse("android-tools-fsutils (>= 1.0)")
isok(t, err)
assert(t, candidates.SatisfiesBuildDepends(*arch, *dep) == true)
dep, err = dependency.Parse("android-tools-fsutils (>= 1.0), quix")
isok(t, err)
assert(t, candidates.SatisfiesBuildDepends(*arch, *dep) == false)
}
func TestResolverBasics(t *testing.T) {
arch, err := dependency.ParseArch("amd64")
isok(t, err)
candidates, err := resolver.ReadFromBinaryIndex(
strings.NewReader(testBinaryIndex),
)
isok(t, err)
assert(t, len(*candidates) == 3)
dep, err := dependency.Parse("baz")
isok(t, err)
possi := dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == false)
dep, err = dependency.Parse("android-tools-fsutils")
isok(t, err)
possi = dep.GetAllPossibilities()[0]
assert(t, candidates.Satisfies(*arch, possi) == true)
}
func main() {
log.SetFlags(log.Lshortfile)
// TODO configurable path? perhaps allow for an optional *.dsc instead?
con, err := control.ParseControlFile("debian/control")
if err != nil {
log.Fatalf("error: %v\n", err)
}
chg, err := changelog.ParseFileOne("debian/changelog")
if err != nil {
log.Fatalf("error: %v\n", err)
}
// TODO configurable or something to avoid guesswork
targetSuite := chg.Target
if targetSuite == "UNRELEASED" {
// check for "Upload to XYZ." or "Rebuild for XYZ." in changelog
re := regexp.MustCompile(`^\s*\*?\s*(Upload\s+to|Rebuild\s+for)\s+(\S+?)\.?(\s+|$)`)
matches := re.FindStringSubmatch(chg.Changelog)
if matches != nil {
targetSuite = matches[2]
} else {
targetSuite = "unstable"
}
}
// TODO configurable (or auto-sensed from the mirror and/or package source)
arches := []string{"amd64", "i386"}
components := []string{"main", "contrib", "non-free"}
fmt.Printf("Target: %s (%s)\n", targetSuite, chg.Target)
fmt.Printf("Architectures: %s\n", strings.Join(arches, " "))
fmt.Printf("Components: %s\n", strings.Join(components, " "))
fmt.Printf("Source: %s\n", con.Source.Source)
fmt.Printf("Version: %s\n", chg.Version)
fmt.Printf("\n")
indexSources := aptsources.DebianSources(targetSuite, components...)
index, err := indexSources.FetchCandidates(arches...)
if err != nil {
log.Fatalf("error: %v\n", err)
}
incoming := NewTarget(
"http://incoming.debian.org/debian-buildd",
[]string{"buildd-" + targetSuite},
components,
arches,
)
if err = incoming.Fetch(); err != nil {
log.Fatalf("error: %v\n", err)
}
newQueue, err := dnew.ParseNewUrl(dnew.New822)
if err != nil {
log.Fatalf("error: %v\n", err)
}
newBinaries := map[string]dnew.NewEntry{}
for _, newPkg := range newQueue {
for _, newBin := range newPkg.Binary {
newBinaries[newBin] = newPkg
}
}
allDeps := dependency.Dependency{}
binRelation := dependency.Relation{}
for _, bin := range con.Binaries {
binRelation.Possibilities = append(binRelation.Possibilities, dependency.Possibility{
Name: bin.Package,
Version: &dependency.VersionRelation{
Operator: "=",
Number: chg.Version.String(),
},
})
}
allDeps.Relations = append(allDeps.Relations, binRelation)
allDeps.Relations = append(allDeps.Relations, con.Source.BuildDepends.Relations...)
allDeps.Relations = append(allDeps.Relations, con.Source.BuildDependsIndep.Relations...)
for _, bin := range con.Binaries {
allDeps.Relations = append(allDeps.Relations, bin.Depends.Relations...)
allDeps.Relations = append(allDeps.Relations, bin.Recommends.Relations...)
allDeps.Relations = append(allDeps.Relations, bin.Suggests.Relations...)
allDeps.Relations = append(allDeps.Relations, bin.Enhances.Relations...)
allDeps.Relations = append(allDeps.Relations, bin.PreDepends.Relations...)
}
depArch, err := dependency.ParseArch("any")
if err != nil {
log.Fatalf("error: %v\n", err)
}
seenRelations := map[string]bool{}
for _, relation := range allDeps.Relations {
relationString := relation.String()
if seenRelations[relationString] {
continue
}
seenRelations[relationString] = true
oneCan := false
notes := []string{}
for _, possi := range relation.Possibilities {
if possi.Substvar {
//fmt.Printf("ignoring substvar %s\n", possi)
continue
}
can, why, _ := index.ExplainSatisfies(*depArch, possi)
if !can {
inCan, _, incomingBins := incoming.ExplainSatisfies(*depArch, possi)
if !inCan {
if newPkg, ok := newBinaries[possi.Name]; ok {
newUrl := fmt.Sprintf("https://ftp-master.debian.org/new/%s_%s.html", newPkg.Source, newPkg.Version[0])
notes = append(notes, fmt.Sprintf("NEW (%s): %s", possi.Name, newUrl))
} else {
notes = append(notes, why)
}
} else {
notes = append(notes, fmt.Sprintf("incoming (%s): %s", possi.Name, incoming.UrlTo(incomingBins[0])))
}
} else {
oneCan = true
// TODO figure out how we can incorporate this ("Section: oldlibs" from debian/control doesn't propagate to the Packages file on the mirror, so we'd have to parse the .deb itself to get this info, which is fairly untenable)
/*
// NOTE "bins" is the last return value in the call to "index.ExplainSatisfies" above
for _, bin := range bins {
if bin.Section == "oldlibs" {
oneCan = false
notes = append(notes, fmt.Sprintf(`%s (%s) is "Section: oldlibs", which suggests it is likely transitional`, bin.Package, bin.Version.String()))
}
}
*/
}
}
if ignoreRelationSecondaryFails && oneCan {
continue
}
if len(notes) > 0 {
fmt.Printf("Relation: %s\n", relation)
if len(notes) > 1 {
fmt.Printf("Notes:\n %s\n", strings.Join(notes, "\n "))
} else {
fmt.Printf("Notes: %s\n", notes[0])
}
fmt.Printf("\n")
}
}
}
func buildBin(dscFile string) (control.DSC, string) {
dscDir := filepath.Dir(dscFile)
dsc, err := control.ParseDscFile(dscFile)
if err != nil {
log.Fatalf("error: %v\n", err)
}
if err := dsc.Validate(); err != nil {
log.Fatalf("error, validation failed: %v\n", err)
}
hasArch := false
hasIndep := false
for _, arch := range dsc.Architectures {
if arch.CPU == "all" {
hasIndep = true
} else {
hasArch = true
}
}
img := fmt.Sprintf("gdbuild/bin:%s_%s", dsc.Source, scrubForDockerTag(dsc.Version.String()))
// TODO parse this information from an image? optional commandline parameters?
suite := "unstable"
sources := aptsources.DebianSources(suite, "main")
arch := "amd64"
// prepend incoming so we get the latest and greatest
sources = sources.Prepend(aptsources.Source{
Types: []string{"deb", "deb-src"},
URIs: []string{"http://incoming.debian.org/debian-buildd"},
Suites: []string{"buildd-" + suite},
Components: []string{"main"},
})
index, err := sources.FetchCandidates(arch)
if err != nil {
log.Fatalf("error: %v\n", err)
}
depArch, err := dependency.ParseArch(arch)
if err != nil {
log.Fatalf("error: %v\n", err)
}
buildEssential, err := dependency.Parse("build-essential, dpkg-dev, fakeroot")
if err != nil {
log.Fatalf("error: %v\n", err)
}
allCan := true
bins := map[string]control.BinaryIndex{}
binsSlice := []string{} // ugh Go
RelLoop:
for _, rel := range append(buildEssential.Relations, dsc.BuildDepends.Relations...) {
canRel := false
for _, possi := range rel.Possibilities {
if possi.Substvar {
continue
}
if bin, ok := bins[possi.Name]; ok && binSatPossi(depArch, bin, possi) {
continue RelLoop
}
}
PossiLoop:
for _, possi := range rel.Possibilities {
if possi.Substvar {
continue
}
entries, ok := map[string][]control.BinaryIndex(*index)[possi.Name]
if !ok {
continue
}
for _, bin := range entries {
if binSatPossi(depArch, bin, possi) {
if existBin, ok := bins[bin.Package]; ok {
log.Printf("uh oh, already chose %s=%s but want %s=%s for %q\n", existBin.Package, existBin.Version, bin.Package, bin.Version, possi)
continue PossiLoop
}
bins[bin.Package] = bin
binsSlice = append(binsSlice, bin.Package)
canRel = true
break PossiLoop
}
}
}
if !canRel {
log.Printf("warning: unable to satisfy %q\n", rel)
allCan = false
}
}
sort.Strings(binsSlice)
if !allCan {
//log.Fatalf("Unsatisfied possi; exiting.\n")
log.Println()
log.Println("WARNING: Unsatisfied possi!")
log.Println()
}
dockerfile := fmt.Sprintf("FROM debian:%s\n", suite)
// TODO allow this to instead be "FROM scratch\nADD some-chroot-tarball.tar.* /\n"
// see https://sources.debian.net/src/pbuilder/jessie/pbuilder-modules/#L306
// and https://sources.debian.net/src/pbuilder/jessie/pbuilder-modules/#L408
dockerfile += `
# setup environment configuration
RUN { echo '#!/bin/sh'; echo 'exit 101'; } > /usr/sbin/policy-rc.d \
&& chmod +x /usr/sbin/policy-rc.d
RUN echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/15gdbuild
# put /tmp in a volume so it's always ephemeral (and performant)
VOLUME /tmp
`
// setup sources.list explicitly -- don't trust the tarball/base image
dockerfile += fmt.Sprintf(`
# setup sources.list
RUN find /etc/apt/sources.list.d -type f -exec rm -v '{}' + \
&& echo %q | tee /etc/apt/sources.list >&2
`, sources.ListString())
eatMyDataPrefix := ""
if doEatMyData {
eatMyDataPrefix = "eatmydata "
dockerfile += `
RUN apt-get update && apt-get install -y \
eatmydata \
&& rm -rf /var/lib/apt/lists/*
`
}
dockerfile += fmt.Sprintf(`
RUN %sapt-get update && %sapt-get install -y \
`, eatMyDataPrefix, eatMyDataPrefix)
for _, pkg := range binsSlice {
bin := bins[pkg]
dockerfile += fmt.Sprintf("\t\t%s=%s \\\n", bin.Package, bin.Version)
}
dockerfile += "\t&& rm -rf /var/lib/apt/lists/*\n"
files := []string{dsc.Filename}
for _, f := range dsc.Files {
files = append(files, filepath.Join(dscDir, f.Filename))
}
dockerfile += "COPY"
for _, f := range files {
dockerfile += " " + filepath.Base(f)
}
dockerfile += " /usr/src/.in/\n"
if debBuildOptions := os.Getenv("DEB_BUILD_OPTIONS"); debBuildOptions != "" {
dockerfile += fmt.Sprintf("\nENV DEB_BUILD_OPTIONS %s\n", debBuildOptions)
}
buildCommand := fmt.Sprintf("%sdpkg-buildpackage -uc -us -d -sa", eatMyDataPrefix)
if doSASBS {
buildCommandParts := []string{}
if hasIndep {
buildCommandParts = append(
buildCommandParts,
buildCommand+" -S",
buildCommand+" -A",
)
}
if hasArch {
buildCommandParts = append(
buildCommandParts,
buildCommand+" -S",
buildCommand+" -B",
)
}
buildCommandParts = append(
buildCommandParts,
buildCommand+" -S",
// end with a full build of "ALL THE THINGS" so we have good, consistent .changes and .dsc
"rm ../*.changes ../*.dsc ../*.deb",
buildCommand,
)
buildCommand = strings.Join(buildCommandParts, " && ")
}
dockerfile += fmt.Sprintf(`
WORKDIR /usr/src
RUN chown -R nobody:nogroup .
USER nobody:nogroup
RUN dpkg-source -x %q pkg
# work around overlayfs bugs (data inconsistency issues; see https://github.com/docker/docker/issues/10180)
VOLUME /usr/src/pkg
# rm: cannot remove 'pkg/.pc/xyz.patch': Directory not empty
RUN (cd pkg && set -x && %s) \
&& mkdir .out \
&& { \
echo *.changes; \
awk '$1 == "Files:" { files = 1; next } /^ / && files { print $5 } /^[^ ]/ { files = 0 }' *.changes; \
echo .out/; \
} | xargs ln -v
`, ".in/"+filepath.Base(dsc.Filename), buildCommand)
err = dockerBuild(img, dockerfile, files...)
if err != nil {
log.Fatalf("error: %v\n", err)
}
return *dsc, img
}
