func TestParseSignedDsc(t *testing.T) {
contents := bytes.NewBufferString(`-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
Format: 3.0 (quilt)
Source: aria2
Binary: aria2
Architecture: any
Version: 1.18.5-1
Maintainer: Patrick Ruckstuhl <*****@*****.**>
Uploaders: Kartik Mistry <*****@*****.**>
Homepage: http://aria2.sourceforge.net/
Standards-Version: 3.9.5
Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/aria2.git;a=summary
Vcs-Git: git://anonscm.debian.org/collab-maint/aria2.git
Testsuite: autopkgtest
Build-Depends: autotools-dev, debhelper (>= 7), dpkg-dev (>= 1.16.1~), libc-ares-dev, libgcrypt11-dev (>= 1.5.0-3) | libgcrypt-dev, libgnutls28-dev, libsqlite3-dev, libxml2-dev, pkg-config, zlib1g-dev | libz-dev
Package-List:
aria2 deb net optional
Checksums-Sha1:
91639bf99a2e84873675f470fd36cee47f466770 2102797 aria2_1.18.5.orig.tar.bz2
c031efb88a477986dac82477433ee0865643bf27 5428 aria2_1.18.5-1.debian.tar.xz
Checksums-Sha256:
25e21f94bb278a8624e0e4e131e740d105f7d5570290053beb8ae6c33fb9ce3f 2102797 aria2_1.18.5.orig.tar.bz2
112aa6973779e9ebaf51d8ab445534fffad4562d4e2de3afd3352f3f3b2f6df3 5428 aria2_1.18.5-1.debian.tar.xz
Files:
79ddd76decadba7176b27c653f5c5aa2 2102797 aria2_1.18.5.orig.tar.bz2
3f2a5585139c649765c6fc5db95bb32a 5428 aria2_1.18.5-1.debian.tar.xz
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1
iQIcBAEBCAAGBQJTRZTXAAoJEALB0/J4OqTeVlUQAJ0hkUIuf84ixkANGC51nGyW
weWeVg2l1ozkDTgSx4NpDaVGzWzmVVTMHByMLfGToDiuWOxHc6qCwtLLlGg7Qdg8
jbDfR21wUA//b+/Pt8SPUP3uAffQ4Rq7D65Cdr23Fkd9LJcOmgf8NkwRKcfXzsx6
ZWj9zK2RVNAwOjTDQGs7OEx2LZsFmL0mbO67ifCsuhWU9JJltf0VgRz5BwkXPnPw
V7Ouq0zE98w2B/Ssq+eRjw/25e7C+DV58lBWeCy+qH4yKigjz3tm9Y7WS9XVPHUa
EjC8mUzT6RhFLWCgtP0NDhgxX0lcm2MNp7iYV7IVdVq99cKsOBZvNXl+TS7v+tjr
JNEKVT4wMHzC0pdGjR2ly0AkF091u2ewrRfefO56q2LOjrRkzKi9smn7mqTfIx53
WpmQL+3ls27LQ6bwl+KeHuRRyj77TIKGyG/9ywyy3IIR4y7NM3wo9T3DQWHDhF6x
8mKG848AqSwFRNROT0gnW/hRIM6umZnhJT7xYhz3LgTnq+0UG2DldDiAcUzOD+S3
Jf6iv6b+hwO3+exs4sjJ1tzcIu2R7LroTjBn8zqZno5YeVzUcN9kRMHls13F0gtb
HwXGSPZ8O8m3ASS7XPpo+vmT5T/W0h75NvAAm7ju9V7EgpGJbE5RwVskYvIqoeif
U6LiZnj6CDeY9Xtjsi2l
=7fkT
-----END PGP SIGNATURE-----`)
paragraphs, err := godebiancontrol.Parse(godebiancontrol.PGPSignatureStripper(contents))
if err != nil {
t.Fatal(err)
}
if len(paragraphs) != 1 {
t.Fatal("Expected exactly one paragraphs")
}
if paragraphs[0]["Format"] != "3.0 (quilt)" {
t.Fatal(`"Format" (simple) was not parsed correctly`)
}
if paragraphs[0]["Testsuite"] != "autopkgtest" {
t.Fatal(`"Testsuite" was not parsed correctly`)
}
}
// NewList reads a package list from r
func NewList(r io.Reader) (*List, error) {
pp, err := godebiancontrol.Parse(r)
if err != nil {
return nil, err
}
l := &List{
Version: map[string]string{},
Package: map[string]string{},
Location: map[string]string{},
}
for _, e := range pp {
p := e["Package"]
v := e["Version"]
loc := e["Filename"]
if p == "" || v == "" || loc == "" {
continue
}
s, ok := e["Source"]
if !ok {
s = e["Package"]
}
l.Package[p] = s
l.Version[p] = v
l.Location[p] = loc
}
return l, nil
}
// Reads the Sources.gz file and returns a slice of queuedFile entries
// containing all files that should be downloaded.
func buildQueue(sourcesPath string, mirrorDir string) (Queue, error) {
queue := Queue{}
file, err := os.Open(sourcesPath)
if err != nil {
return queue, err
}
defer file.Close()
gzipReader, err := gzip.NewReader(file)
if err != nil {
return queue, err
}
defer gzipReader.Close()
sourcePackages, err := godebiancontrol.Parse(gzipReader)
if err != nil {
return queue, err
}
// On average, there are 3 files per source package, so allocate a slice
// with that capacity to avoid re-allocating memory all the time.
queue = make(Queue, 0, len(sourcePackages)*3)
seen := make(map[string]bool, len(sourcePackages)*3)
for _, pkg := range sourcePackages {
for _, line := range strings.Split(pkg["Files"], "\n") {
parts := strings.Split(strings.TrimSpace(line), " ")
// pkg["Files"] has a newline at the end, so we get one empty line.
if len(parts) < 3 {
continue
}
size, err := strconv.Atoi(parts[1])
if err != nil {
return queue, err
}
path := pkg["Directory"] + "/" + parts[2]
// Skip duplicates.
if seen[path] {
continue
}
// Skip the file if it is already present and has the same size.
// Files with the same name (in /pool/) cannot change, so we don’t
// need to consider the checksum at all.
fi, err := os.Stat(filepath.Join(mirrorDir, path))
if err == nil && fi.Size() == int64(size) {
continue
}
queue = append(queue, queuedFile{
Path: path,
Size: size,
})
seen[path] = true
}
}
return queue, nil
}
func main() {
flag.Parse()
oldShards = strings.Split(*oldShardsStr, ",")
newShards = strings.Split(*newShardsStr, ",")
sourcesSuffix := "/dists/sid/main/source/Sources.gz"
resp, err := http.Get(*mirrorUrl + sourcesSuffix)
if err != nil {
log.Printf("Could not get Sources.gz: %v\n", err)
return
}
defer resp.Body.Close()
reader, err := gzip.NewReader(resp.Body)
if err != nil {
log.Printf("Could not initialize gzip reader: %v\n", err)
return
}
defer reader.Close()
sourcePackages, err := godebiancontrol.Parse(reader)
if err != nil {
log.Printf("Could not parse Sources.gz: %v\n", err)
return
}
scripts := make([]*os.File, len(oldShards))
for idx, _ := range oldShards {
scripts[idx], err = os.Create(fmt.Sprintf("/tmp/dcs-instant-%d.rackspace.zekjur.net", idx) + ".sh")
if err != nil {
log.Fatal(err)
}
defer scripts[idx].Close()
}
// for every package, calculate who’d be responsible and see if it’s present on that shard.
for _, pkg := range sourcePackages {
p := pkg["Package"] + "_" + pkg["Version"]
oldIdx := taskIdxForPackage(p, len(oldShards))
newIdx := taskIdxForPackage(p, len(newShards))
log.Printf("oldidx = %d, newidx = %d\n", oldIdx, newIdx)
if oldIdx == newIdx {
continue
}
fmt.Fprintf(scripts[oldIdx], "scp -o StrictHostKeyChecking=no -i ~/.ssh/dcs-auto-rs -r /dcs-ssd/unpacked/%s /dcs-ssd/unpacked/%s.idx root@%s:/dcs-ssd/unpacked/ && rm -rf /dcs-ssd/unpacked/%s /dcs-ssd/unpacked/%s.idx\n",
p, p, strings.TrimSuffix(newShards[newIdx], ":21010"), p, p)
}
}
func ExampleParse() {
file, err := os.Open("debian-mirror/dists/sid/main/source/Sources")
if err != nil {
log.Fatal(err)
}
defer file.Close()
paragraphs, err := godebiancontrol.Parse(file)
if err != nil {
log.Fatal(err)
}
// Print a list of which source package uses which package format.
for _, pkg := range paragraphs {
fmt.Printf("%s uses %s\n", pkg["Package"], pkg["Format"])
}
}
func mustLoadMirroredControlFile(name string) []godebiancontrol.Paragraph {
url := fmt.Sprintf("%s/dists/sid/main/%s", *mirrorUrl, name)
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
if resp.StatusCode != 200 {
log.Fatalf("URL %q resulted in %v\n", url, resp.Status)
}
defer resp.Body.Close()
reader, err := gzip.NewReader(resp.Body)
if err != nil {
log.Fatal(err)
}
contents, err := godebiancontrol.Parse(reader)
if err != nil {
log.Fatal(err)
}
return contents
}
func (p *Package) updateFieldCache() error {
cmd := exec.Command("dpkg-deb", "--field", p.file)
o, err := cmd.Output()
if err != nil {
return err
}
b := bytes.NewBuffer(o)
pp, err := godebiancontrol.Parse(b)
if err != nil {
return err
}
if len(pp) != 1 {
return os.ErrInvalid
}
p.fieldcache = pp[0]
return nil
}
func checkSources() {
log.Printf("checking sources\n")
lastSanityCheckStarted.Set(float64(time.Now().Unix()))
// Store packages by shard.
type pkgStatus int
const (
NotPresent pkgStatus = iota
Present
Confirmed
)
packages := make(map[string]map[string]pkgStatus)
for _, shard := range shards {
url := fmt.Sprintf("http://%s/listpkgs", shard)
resp, err := http.Get(url)
if err != nil {
log.Printf("Could not get list of packages from %q: %v\n", url, err)
continue
}
defer resp.Body.Close()
type ListPackageReply struct {
Packages []string
}
var reply ListPackageReply
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(&reply); err != nil {
log.Printf("Invalid json from %q: %v\n", url, err)
continue
}
packages[shard] = make(map[string]pkgStatus)
for _, foundpkg := range reply.Packages {
packages[shard][foundpkg] = Present
}
log.Printf("shard %q has %d packages currently\n", shard, len(reply.Packages))
}
sourcesSuffix := "/dists/sid/main/source/Sources.gz"
resp, err := http.Get(*mirrorUrl + sourcesSuffix)
if err != nil {
log.Printf("Could not get Sources.gz: %v\n", err)
return
}
defer resp.Body.Close()
reader, err := gzip.NewReader(resp.Body)
if err != nil {
log.Printf("Could not initialize gzip reader: %v\n", err)
return
}
defer reader.Close()
sourcePackages, err := godebiancontrol.Parse(reader)
if err != nil {
log.Printf("Could not parse Sources.gz: %v\n", err)
return
}
// for every package, calculate who’d be responsible and see if it’s present on that shard.
for _, pkg := range sourcePackages {
if strings.HasSuffix(pkg["Package"], "-data") {
continue
}
p := pkg["Package"] + "_" + pkg["Version"]
shardIdx := shardmapping.TaskIdxForPackage(p, len(shards))
shard := shards[shardIdx]
// Skip shards that are offline (= for which we have no package list).
if _, online := packages[shard]; !online {
continue
}
status := packages[shard][p]
//log.Printf("package %s: shard %d (%s), status %v\n", p, shardIdx, shard, status)
if status == Present {
packages[shard][p] = Confirmed
} else if status == NotPresent {
log.Printf("Feeding package %s to shard %d (%s)\n", p, shardIdx, shard)
var pkgfiles []string
for _, line := range strings.Split(pkg["Files"], "\n") {
parts := strings.Split(strings.TrimSpace(line), " ")
// pkg["Files"] has a newline at the end, so we get one empty line.
if len(parts) < 3 {
continue
}
url := *mirrorUrl + "/" + pkg["Directory"] + "/" + parts[2]
// Append the .dsc to the end, prepend the other files.
if strings.HasSuffix(url, ".dsc") {
pkgfiles = append(pkgfiles, url)
} else {
pkgfiles = append([]string{url}, pkgfiles...)
}
}
feedfiles(p, pkgfiles)
successfulSanityFeed.Inc()
}
}
// Garbage-collect all packages that have not been confirmed.
for _, shard := range shards {
for p, status := range packages[shard] {
if status != Present {
continue
}
log.Printf("garbage-collecting %q on shard %s\n", p, shard)
shard := shards[shardmapping.TaskIdxForPackage(p, len(shards))]
url := fmt.Sprintf("http://%s/garbagecollect", shard)
if _, err := http.PostForm(url, net_url.Values{"package": {p}}); err != nil {
log.Printf("Could not garbage-collect package %q on shard %s: %v\n", p, shard, err)
continue
}
successfulGarbageCollect.Inc()
}
}
}
// Tries to download a package directly from http://incoming.debian.org
// (typically called from dcs-tail-fedmsg).
// See also https://lists.debian.org/debian-devel-announce/2014/08/msg00008.html
func lookfor(dscName string) {
log.Printf("Looking for %q\n", dscName)
startedLooking := time.Now()
attempt := 0
for {
if attempt > 0 {
// Exponential backoff starting with 8s.
backoff := time.Duration(math.Pow(2, float64(attempt)+2)) * time.Second
log.Printf("Starting attempt %d. Waiting %v\n", attempt+1, backoff)
time.Sleep(backoff)
}
attempt++
// We only try to get this file for 25 minutes. Something is probably
// wrong if it does not succeed within that time, and we want to keep
// goroutines from piling up. The periodic sanity check will find the
// package a bit later then.
if time.Since(startedLooking) > 25*time.Minute {
failedLookfor.Inc()
log.Printf("Not looking for %q anymore. Sanity check will catch it.\n", dscName)
return
}
url := "http://incoming.debian.org/debian-buildd/" + poolPath(dscName)
resp, err := http.Get(url)
if err != nil {
log.Printf("Could not HTTP GET %q: %v\n", url, err)
continue
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Printf("HTTP status for %q: %s\n", url, resp.Status)
continue
}
log.Printf("Downloading %q from incoming.debian.org\n", dscName)
var dscContents bytes.Buffer
// Store a copy of the content in dscContents.
reader := io.TeeReader(resp.Body, &dscContents)
// Strip the PGP signature. The worst thing that can happen is that an
// attacker gives us bad source code to index and serve. Verifying PGP
// signatures is harder since we need an up-to-date debian-keyring.
reader = godebiancontrol.PGPSignatureStripper(reader)
paragraphs, err := godebiancontrol.Parse(reader)
if err != nil {
log.Printf("Invalid dsc file: %v\n", err)
return
}
if len(paragraphs) != 1 {
log.Printf("Expected parsing exactly one paragraph, got %d. Skipping.\n", len(paragraphs))
}
pkg := paragraphs[0]
for _, line := range strings.Split(pkg["Files"], "\n") {
parts := strings.Split(strings.TrimSpace(line), " ")
// pkg["Files"] has a newline at the end, so we get one empty line.
if len(parts) < 3 {
continue
}
fileUrl := "http://incoming.debian.org/debian-buildd/" + poolPath(parts[2])
resp, err := http.Get(fileUrl)
if err != nil {
log.Printf("Could not HTTP GET %q: %v\n", url, err)
return
}
defer resp.Body.Close()
if err := feed(strings.TrimSuffix(dscName, ".dsc"), parts[2], resp.Body); err != nil {
log.Printf("Could not feed %q: %v\n", url, err)
}
}
dscReader := bytes.NewReader(dscContents.Bytes())
if err := feed(strings.TrimSuffix(dscName, ".dsc"), dscName, dscReader); err != nil {
log.Printf("Could not feed %q: %v\n", dscName, err)
}
log.Printf("Fed %q.\n", dscName)
successfulLookfor.Inc()
return
}
}
func TestParse(t *testing.T) {
contents := bytes.NewBufferString(`Package: bti
Binary: bti
Version: 032-1
Maintainer: gregor herrmann <*****@*****.**>
Uploaders: tony mancill <*****@*****.**>
Build-Depends: debhelper (>= 8),
bash-completion (>= 1:1.1-3),
libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf
Architecture: any
Standards-Version: 3.9.2
Format: 3.0 (quilt)
Files:
3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc
1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz
ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz
Vcs-Browser: http://svn.toastfreeware.priv.at/wsvn/ToastfreewareDebian/bti/trunk/
Vcs-Svn: http://svn.toastfreeware.priv.at/debian/bti/trunk/
Checksums-Sha1:
3da2c5a42138c884a7d9524b9706dc56c0d6d46e 1980 bti_032-1.dsc
22061e3f56074703be415d65abc9ca27ef775c6a 59773 bti_032.orig.tar.gz
66ae7f56a3c1f0ebe0638d0ec0599a819d72baea 5065 bti_032-1.debian.tar.gz
Checksums-Sha256:
ed6015b79693f270d0a826c695b40e4d8eb4307942cac81a98f1fda479f74215 1980 bti_032-1.dsc
feeabec98a89040a53283d798f7d55eb4311a854f17312a177dc45919883746a 59773 bti_032.orig.tar.gz
f025da42efaf57db5e71a14cb8be27eb802ad23e7ab02b7ce2252454a86ac1d9 5065 bti_032-1.debian.tar.gz
Homepage: http://gregkh.github.com/bti/
Package-List:
bti deb net extra
Directory: pool/main/b/bti
Priority: source
Section: net
Package: i3-wm
Version: 4.2-1
Installed-Size: 1573
Maintainer: Michael Stapelberg <*****@*****.**>
Architecture: amd64
Provides: x-window-manager
Depends: libc6 (>= 2.8), libev4 (>= 1:4.04), libpcre3 (>= 8.10), libstartup-notification0 (>= 0.10), libx11-6, libxcb-icccm4 (>= 0.3.8), libxcb-keysyms1 (>= 0.3.8), libxcb-randr0 (>= 1.3), libxcb-util0 (>= 0.3.8), libxcb-xinerama0, libxcb1, libxcursor1 (>> 1.1.2), libyajl2 (>= 2.0.4), perl, x11-utils
Recommends: xfonts-base
Suggests: rxvt-unicode | x-terminal-emulator
Description-en: improved dynamic tiling window manager
Key features of i3 are good documentation, reasonable defaults (changeable in
a simple configuration file) and good multi-monitor support. The user
interface is designed for power users and emphasizes keyboard usage. i3 uses
XCB for asynchronous communication with X11 and aims to be fast and
light-weight.
.
Please be aware i3 is primarily targeted at advanced users and developers.
Homepage: http://i3wm.org/
Description-md5: 2be7e62f455351435b1e055745d3e81c
Tag: implemented-in::c, interface::x11, role::program, uitoolkit::TODO,
works-with::unicode, x11::window-manager
Section: x11
Priority: extra
Filename: pool/main/i/i3-wm/i3-wm_4.2-1_amd64.deb
Size: 798186
MD5sum: 3c7dbecd76d5c271401860967563fa8c
SHA1: 2e94f3faa5d4d617061f94076b2537d15fbff73f
SHA256: 2894bc999b3982c4e57f100fa31e21b52e14c5f3bc7ad5345f46842fcdab0db7`)
paragraphs, err := godebiancontrol.Parse(contents)
if err != nil {
t.Fatal(err)
}
if len(paragraphs) != 2 {
t.Fatal("Expected exactly two paragraphs")
}
if paragraphs[0]["Format"] != "3.0 (quilt)" {
t.Fatal(`"Format" (simple) was not parsed correctly`)
}
if paragraphs[0]["Build-Depends"] != "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf" {
t.Fatal(`"Build-Depends" (folder) was not parsed correctly`)
}
expectedFiles := ` 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc
1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz
ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz
`
if paragraphs[0]["Files"] != expectedFiles {
t.Fatal(`"Files" (multiline) was not parsed correctly`)
}
}