func TestParseSignedDsc(t *testing.T) {
	contents := bytes.NewBufferString(`-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

Format: 3.0 (quilt)
Source: aria2
Binary: aria2
Architecture: any
Version: 1.18.5-1
Maintainer: Patrick Ruckstuhl <*****@*****.**>
Uploaders: Kartik Mistry <*****@*****.**>
Homepage: http://aria2.sourceforge.net/
Standards-Version: 3.9.5
Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/aria2.git;a=summary
Vcs-Git: git://anonscm.debian.org/collab-maint/aria2.git
Testsuite: autopkgtest
Build-Depends: autotools-dev, debhelper (>= 7), dpkg-dev (>= 1.16.1~), libc-ares-dev, libgcrypt11-dev (>= 1.5.0-3) | libgcrypt-dev, libgnutls28-dev, libsqlite3-dev, libxml2-dev, pkg-config, zlib1g-dev | libz-dev
Package-List: 
 aria2 deb net optional
Checksums-Sha1: 
 91639bf99a2e84873675f470fd36cee47f466770 2102797 aria2_1.18.5.orig.tar.bz2
 c031efb88a477986dac82477433ee0865643bf27 5428 aria2_1.18.5-1.debian.tar.xz
Checksums-Sha256: 
 25e21f94bb278a8624e0e4e131e740d105f7d5570290053beb8ae6c33fb9ce3f 2102797 aria2_1.18.5.orig.tar.bz2
 112aa6973779e9ebaf51d8ab445534fffad4562d4e2de3afd3352f3f3b2f6df3 5428 aria2_1.18.5-1.debian.tar.xz
Files: 
 79ddd76decadba7176b27c653f5c5aa2 2102797 aria2_1.18.5.orig.tar.bz2
 3f2a5585139c649765c6fc5db95bb32a 5428 aria2_1.18.5-1.debian.tar.xz

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1

iQIcBAEBCAAGBQJTRZTXAAoJEALB0/J4OqTeVlUQAJ0hkUIuf84ixkANGC51nGyW
weWeVg2l1ozkDTgSx4NpDaVGzWzmVVTMHByMLfGToDiuWOxHc6qCwtLLlGg7Qdg8
jbDfR21wUA//b+/Pt8SPUP3uAffQ4Rq7D65Cdr23Fkd9LJcOmgf8NkwRKcfXzsx6
ZWj9zK2RVNAwOjTDQGs7OEx2LZsFmL0mbO67ifCsuhWU9JJltf0VgRz5BwkXPnPw
V7Ouq0zE98w2B/Ssq+eRjw/25e7C+DV58lBWeCy+qH4yKigjz3tm9Y7WS9XVPHUa
EjC8mUzT6RhFLWCgtP0NDhgxX0lcm2MNp7iYV7IVdVq99cKsOBZvNXl+TS7v+tjr
JNEKVT4wMHzC0pdGjR2ly0AkF091u2ewrRfefO56q2LOjrRkzKi9smn7mqTfIx53
WpmQL+3ls27LQ6bwl+KeHuRRyj77TIKGyG/9ywyy3IIR4y7NM3wo9T3DQWHDhF6x
8mKG848AqSwFRNROT0gnW/hRIM6umZnhJT7xYhz3LgTnq+0UG2DldDiAcUzOD+S3
Jf6iv6b+hwO3+exs4sjJ1tzcIu2R7LroTjBn8zqZno5YeVzUcN9kRMHls13F0gtb
HwXGSPZ8O8m3ASS7XPpo+vmT5T/W0h75NvAAm7ju9V7EgpGJbE5RwVskYvIqoeif
U6LiZnj6CDeY9Xtjsi2l
=7fkT
-----END PGP SIGNATURE-----`)
	paragraphs, err := godebiancontrol.Parse(godebiancontrol.PGPSignatureStripper(contents))
	if err != nil {
		t.Fatal(err)
	}
	if len(paragraphs) != 1 {
		t.Fatal("Expected exactly one paragraphs")
	}
	if paragraphs[0]["Format"] != "3.0 (quilt)" {
		t.Fatal(`"Format" (simple) was not parsed correctly`)
	}
	if paragraphs[0]["Testsuite"] != "autopkgtest" {
		t.Fatal(`"Testsuite" was not parsed correctly`)
	}
}
Beispiel #2
0
// NewList reads a package list from r
func NewList(r io.Reader) (*List, error) {
	pp, err := godebiancontrol.Parse(r)
	if err != nil {
		return nil, err
	}
	l := &List{
		Version:  map[string]string{},
		Package:  map[string]string{},
		Location: map[string]string{},
	}
	for _, e := range pp {
		p := e["Package"]
		v := e["Version"]
		loc := e["Filename"]
		if p == "" || v == "" || loc == "" {
			continue
		}
		s, ok := e["Source"]
		if !ok {
			s = e["Package"]
		}
		l.Package[p] = s
		l.Version[p] = v
		l.Location[p] = loc
	}
	return l, nil
}
Beispiel #3
0
// Reads the Sources.gz file and returns a slice of queuedFile entries
// containing all files that should be downloaded.
func buildQueue(sourcesPath string, mirrorDir string) (Queue, error) {
	queue := Queue{}
	file, err := os.Open(sourcesPath)
	if err != nil {
		return queue, err
	}
	defer file.Close()

	gzipReader, err := gzip.NewReader(file)
	if err != nil {
		return queue, err
	}
	defer gzipReader.Close()

	sourcePackages, err := godebiancontrol.Parse(gzipReader)
	if err != nil {
		return queue, err
	}

	// On average, there are 3 files per source package, so allocate a slice
	// with that capacity to avoid re-allocating memory all the time.
	queue = make(Queue, 0, len(sourcePackages)*3)
	seen := make(map[string]bool, len(sourcePackages)*3)
	for _, pkg := range sourcePackages {
		for _, line := range strings.Split(pkg["Files"], "\n") {
			parts := strings.Split(strings.TrimSpace(line), " ")
			// pkg["Files"] has a newline at the end, so we get one empty line.
			if len(parts) < 3 {
				continue
			}
			size, err := strconv.Atoi(parts[1])
			if err != nil {
				return queue, err
			}
			path := pkg["Directory"] + "/" + parts[2]

			// Skip duplicates.
			if seen[path] {
				continue
			}

			// Skip the file if it is already present and has the same size.
			// Files with the same name (in /pool/) cannot change, so we don’t
			// need to consider the checksum at all.
			fi, err := os.Stat(filepath.Join(mirrorDir, path))
			if err == nil && fi.Size() == int64(size) {
				continue
			}

			queue = append(queue, queuedFile{
				Path: path,
				Size: size,
			})
			seen[path] = true
		}
	}

	return queue, nil
}
Beispiel #4
0
func main() {
	flag.Parse()

	oldShards = strings.Split(*oldShardsStr, ",")
	newShards = strings.Split(*newShardsStr, ",")

	sourcesSuffix := "/dists/sid/main/source/Sources.gz"
	resp, err := http.Get(*mirrorUrl + sourcesSuffix)
	if err != nil {
		log.Printf("Could not get Sources.gz: %v\n", err)
		return
	}
	defer resp.Body.Close()
	reader, err := gzip.NewReader(resp.Body)
	if err != nil {
		log.Printf("Could not initialize gzip reader: %v\n", err)
		return
	}
	defer reader.Close()

	sourcePackages, err := godebiancontrol.Parse(reader)
	if err != nil {
		log.Printf("Could not parse Sources.gz: %v\n", err)
		return
	}

	scripts := make([]*os.File, len(oldShards))
	for idx, _ := range oldShards {
		scripts[idx], err = os.Create(fmt.Sprintf("/tmp/dcs-instant-%d.rackspace.zekjur.net", idx) + ".sh")
		if err != nil {
			log.Fatal(err)
		}
		defer scripts[idx].Close()
	}

	// for every package, calculate who’d be responsible and see if it’s present on that shard.
	for _, pkg := range sourcePackages {
		p := pkg["Package"] + "_" + pkg["Version"]
		oldIdx := taskIdxForPackage(p, len(oldShards))
		newIdx := taskIdxForPackage(p, len(newShards))
		log.Printf("oldidx = %d, newidx = %d\n", oldIdx, newIdx)
		if oldIdx == newIdx {
			continue
		}
		fmt.Fprintf(scripts[oldIdx], "scp -o StrictHostKeyChecking=no -i ~/.ssh/dcs-auto-rs -r /dcs-ssd/unpacked/%s /dcs-ssd/unpacked/%s.idx root@%s:/dcs-ssd/unpacked/ && rm -rf /dcs-ssd/unpacked/%s /dcs-ssd/unpacked/%s.idx\n",
			p, p, strings.TrimSuffix(newShards[newIdx], ":21010"), p, p)
	}
}
func ExampleParse() {
	file, err := os.Open("debian-mirror/dists/sid/main/source/Sources")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	paragraphs, err := godebiancontrol.Parse(file)
	if err != nil {
		log.Fatal(err)
	}

	// Print a list of which source package uses which package format.
	for _, pkg := range paragraphs {
		fmt.Printf("%s uses %s\n", pkg["Package"], pkg["Format"])
	}
}
Beispiel #6
0
func mustLoadMirroredControlFile(name string) []godebiancontrol.Paragraph {
	url := fmt.Sprintf("%s/dists/sid/main/%s", *mirrorUrl, name)
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	if resp.StatusCode != 200 {
		log.Fatalf("URL %q resulted in %v\n", url, resp.Status)
	}
	defer resp.Body.Close()

	reader, err := gzip.NewReader(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	contents, err := godebiancontrol.Parse(reader)
	if err != nil {
		log.Fatal(err)
	}

	return contents
}
Beispiel #7
0
func (p *Package) updateFieldCache() error {
	cmd := exec.Command("dpkg-deb", "--field", p.file)

	o, err := cmd.Output()
	if err != nil {
		return err
	}

	b := bytes.NewBuffer(o)
	pp, err := godebiancontrol.Parse(b)
	if err != nil {
		return err
	}

	if len(pp) != 1 {
		return os.ErrInvalid

	}

	p.fieldcache = pp[0]
	return nil
}
Beispiel #8
0
func checkSources() {
	log.Printf("checking sources\n")
	lastSanityCheckStarted.Set(float64(time.Now().Unix()))

	// Store packages by shard.
	type pkgStatus int
	const (
		NotPresent pkgStatus = iota
		Present
		Confirmed
	)
	packages := make(map[string]map[string]pkgStatus)

	for _, shard := range shards {
		url := fmt.Sprintf("http://%s/listpkgs", shard)
		resp, err := http.Get(url)
		if err != nil {
			log.Printf("Could not get list of packages from %q: %v\n", url, err)
			continue
		}
		defer resp.Body.Close()

		type ListPackageReply struct {
			Packages []string
		}

		var reply ListPackageReply
		decoder := json.NewDecoder(resp.Body)
		if err := decoder.Decode(&reply); err != nil {
			log.Printf("Invalid json from %q: %v\n", url, err)
			continue
		}
		packages[shard] = make(map[string]pkgStatus)
		for _, foundpkg := range reply.Packages {
			packages[shard][foundpkg] = Present
		}
		log.Printf("shard %q has %d packages currently\n", shard, len(reply.Packages))
	}

	sourcesSuffix := "/dists/sid/main/source/Sources.gz"
	resp, err := http.Get(*mirrorUrl + sourcesSuffix)
	if err != nil {
		log.Printf("Could not get Sources.gz: %v\n", err)
		return
	}
	defer resp.Body.Close()
	reader, err := gzip.NewReader(resp.Body)
	if err != nil {
		log.Printf("Could not initialize gzip reader: %v\n", err)
		return
	}
	defer reader.Close()

	sourcePackages, err := godebiancontrol.Parse(reader)
	if err != nil {
		log.Printf("Could not parse Sources.gz: %v\n", err)
		return
	}

	// for every package, calculate who’d be responsible and see if it’s present on that shard.
	for _, pkg := range sourcePackages {
		if strings.HasSuffix(pkg["Package"], "-data") {
			continue
		}
		p := pkg["Package"] + "_" + pkg["Version"]
		shardIdx := shardmapping.TaskIdxForPackage(p, len(shards))
		shard := shards[shardIdx]
		// Skip shards that are offline (= for which we have no package list).
		if _, online := packages[shard]; !online {
			continue
		}
		status := packages[shard][p]
		//log.Printf("package %s: shard %d (%s), status %v\n", p, shardIdx, shard, status)
		if status == Present {
			packages[shard][p] = Confirmed
		} else if status == NotPresent {
			log.Printf("Feeding package %s to shard %d (%s)\n", p, shardIdx, shard)

			var pkgfiles []string
			for _, line := range strings.Split(pkg["Files"], "\n") {
				parts := strings.Split(strings.TrimSpace(line), " ")
				// pkg["Files"] has a newline at the end, so we get one empty line.
				if len(parts) < 3 {
					continue
				}
				url := *mirrorUrl + "/" + pkg["Directory"] + "/" + parts[2]

				// Append the .dsc to the end, prepend the other files.
				if strings.HasSuffix(url, ".dsc") {
					pkgfiles = append(pkgfiles, url)
				} else {
					pkgfiles = append([]string{url}, pkgfiles...)
				}
			}
			feedfiles(p, pkgfiles)

			successfulSanityFeed.Inc()
		}
	}

	// Garbage-collect all packages that have not been confirmed.
	for _, shard := range shards {
		for p, status := range packages[shard] {
			if status != Present {
				continue
			}

			log.Printf("garbage-collecting %q on shard %s\n", p, shard)

			shard := shards[shardmapping.TaskIdxForPackage(p, len(shards))]
			url := fmt.Sprintf("http://%s/garbagecollect", shard)
			if _, err := http.PostForm(url, net_url.Values{"package": {p}}); err != nil {
				log.Printf("Could not garbage-collect package %q on shard %s: %v\n", p, shard, err)
				continue
			}

			successfulGarbageCollect.Inc()
		}
	}
}
Beispiel #9
0
// Tries to download a package directly from http://incoming.debian.org
// (typically called from dcs-tail-fedmsg).
// See also https://lists.debian.org/debian-devel-announce/2014/08/msg00008.html
func lookfor(dscName string) {
	log.Printf("Looking for %q\n", dscName)
	startedLooking := time.Now()
	attempt := 0
	for {
		if attempt > 0 {
			// Exponential backoff starting with 8s.
			backoff := time.Duration(math.Pow(2, float64(attempt)+2)) * time.Second
			log.Printf("Starting attempt %d. Waiting %v\n", attempt+1, backoff)
			time.Sleep(backoff)
		}
		attempt++

		// We only try to get this file for 25 minutes. Something is probably
		// wrong if it does not succeed within that time, and we want to keep
		// goroutines from piling up. The periodic sanity check will find the
		// package a bit later then.
		if time.Since(startedLooking) > 25*time.Minute {
			failedLookfor.Inc()
			log.Printf("Not looking for %q anymore. Sanity check will catch it.\n", dscName)
			return
		}

		url := "http://incoming.debian.org/debian-buildd/" + poolPath(dscName)
		resp, err := http.Get(url)
		if err != nil {
			log.Printf("Could not HTTP GET %q: %v\n", url, err)
			continue
		}
		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			log.Printf("HTTP status for %q: %s\n", url, resp.Status)
			continue
		}
		log.Printf("Downloading %q from incoming.debian.org\n", dscName)
		var dscContents bytes.Buffer
		// Store a copy of the content in dscContents.
		reader := io.TeeReader(resp.Body, &dscContents)
		// Strip the PGP signature. The worst thing that can happen is that an
		// attacker gives us bad source code to index and serve. Verifying PGP
		// signatures is harder since we need an up-to-date debian-keyring.
		reader = godebiancontrol.PGPSignatureStripper(reader)
		paragraphs, err := godebiancontrol.Parse(reader)
		if err != nil {
			log.Printf("Invalid dsc file: %v\n", err)
			return
		}

		if len(paragraphs) != 1 {
			log.Printf("Expected parsing exactly one paragraph, got %d. Skipping.\n", len(paragraphs))
		}
		pkg := paragraphs[0]

		for _, line := range strings.Split(pkg["Files"], "\n") {
			parts := strings.Split(strings.TrimSpace(line), " ")
			// pkg["Files"] has a newline at the end, so we get one empty line.
			if len(parts) < 3 {
				continue
			}
			fileUrl := "http://incoming.debian.org/debian-buildd/" + poolPath(parts[2])
			resp, err := http.Get(fileUrl)
			if err != nil {
				log.Printf("Could not HTTP GET %q: %v\n", url, err)
				return
			}
			defer resp.Body.Close()
			if err := feed(strings.TrimSuffix(dscName, ".dsc"), parts[2], resp.Body); err != nil {
				log.Printf("Could not feed %q: %v\n", url, err)
			}
		}
		dscReader := bytes.NewReader(dscContents.Bytes())
		if err := feed(strings.TrimSuffix(dscName, ".dsc"), dscName, dscReader); err != nil {
			log.Printf("Could not feed %q: %v\n", dscName, err)
		}
		log.Printf("Fed %q.\n", dscName)
		successfulLookfor.Inc()
		return
	}
}
func TestParse(t *testing.T) {
	contents := bytes.NewBufferString(`Package: bti
Binary: bti
Version: 032-1
Maintainer: gregor herrmann <*****@*****.**>
Uploaders: tony mancill <*****@*****.**>
Build-Depends: debhelper (>= 8),
  bash-completion (>= 1:1.1-3),
  libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf
Architecture: any
Standards-Version: 3.9.2
Format: 3.0 (quilt)
Files:
 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc
 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz
 ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz
Vcs-Browser: http://svn.toastfreeware.priv.at/wsvn/ToastfreewareDebian/bti/trunk/
Vcs-Svn: http://svn.toastfreeware.priv.at/debian/bti/trunk/
Checksums-Sha1:
 3da2c5a42138c884a7d9524b9706dc56c0d6d46e 1980 bti_032-1.dsc
 22061e3f56074703be415d65abc9ca27ef775c6a 59773 bti_032.orig.tar.gz
 66ae7f56a3c1f0ebe0638d0ec0599a819d72baea 5065 bti_032-1.debian.tar.gz
Checksums-Sha256:
 ed6015b79693f270d0a826c695b40e4d8eb4307942cac81a98f1fda479f74215 1980 bti_032-1.dsc
 feeabec98a89040a53283d798f7d55eb4311a854f17312a177dc45919883746a 59773 bti_032.orig.tar.gz
 f025da42efaf57db5e71a14cb8be27eb802ad23e7ab02b7ce2252454a86ac1d9 5065 bti_032-1.debian.tar.gz
Homepage: http://gregkh.github.com/bti/
Package-List:
 bti deb net extra
Directory: pool/main/b/bti
Priority: source
Section: net


Package: i3-wm
Version: 4.2-1
Installed-Size: 1573
Maintainer: Michael Stapelberg <*****@*****.**>
Architecture: amd64
Provides: x-window-manager
Depends: libc6 (>= 2.8), libev4 (>= 1:4.04), libpcre3 (>= 8.10), libstartup-notification0 (>= 0.10), libx11-6, libxcb-icccm4 (>= 0.3.8), libxcb-keysyms1 (>= 0.3.8), libxcb-randr0 (>= 1.3), libxcb-util0 (>= 0.3.8), libxcb-xinerama0, libxcb1, libxcursor1 (>> 1.1.2), libyajl2 (>= 2.0.4), perl, x11-utils
Recommends: xfonts-base
Suggests: rxvt-unicode | x-terminal-emulator
Description-en: improved dynamic tiling window manager
 Key features of i3 are good documentation, reasonable defaults (changeable in
 a simple configuration file) and good multi-monitor support. The user
 interface is designed for power users and emphasizes keyboard usage. i3 uses
 XCB for asynchronous communication with X11 and aims to be fast and
 light-weight.
 .
 Please be aware i3 is primarily targeted at advanced users and developers.
Homepage: http://i3wm.org/
Description-md5: 2be7e62f455351435b1e055745d3e81c
Tag: implemented-in::c, interface::x11, role::program, uitoolkit::TODO,
 works-with::unicode, x11::window-manager
Section: x11
Priority: extra
Filename: pool/main/i/i3-wm/i3-wm_4.2-1_amd64.deb
Size: 798186
MD5sum: 3c7dbecd76d5c271401860967563fa8c
SHA1: 2e94f3faa5d4d617061f94076b2537d15fbff73f
SHA256: 2894bc999b3982c4e57f100fa31e21b52e14c5f3bc7ad5345f46842fcdab0db7`)
	paragraphs, err := godebiancontrol.Parse(contents)
	if err != nil {
		t.Fatal(err)
	}
	if len(paragraphs) != 2 {
		t.Fatal("Expected exactly two paragraphs")
	}
	if paragraphs[0]["Format"] != "3.0 (quilt)" {
		t.Fatal(`"Format" (simple) was not parsed correctly`)
	}
	if paragraphs[0]["Build-Depends"] != "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf" {
		t.Fatal(`"Build-Depends" (folder) was not parsed correctly`)
	}
	expectedFiles := ` 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc
 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz
 ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz
`
	if paragraphs[0]["Files"] != expectedFiles {
		t.Fatal(`"Files" (multiline) was not parsed correctly`)
	}
}