Esempio n. 1
0
// Download downloads all repo files
func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
	list := NewPackageList()

	progress.Printf("Downloading & parsing package files...\n")

	// Download and parse all Packages & Source files
	packagesURLs := [][]string{}

	if repo.IsFlat() {
		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
		if repo.DownloadSources {
			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
		}
	} else {
		for _, component := range repo.Components {
			for _, architecture := range repo.Architectures {
				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
			}
			if repo.DownloadSources {
				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
			}
		}
	}

	for _, info := range packagesURLs {
		url, kind := info[0], info[1]
		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
		if err != nil {
			return err
		}
		defer packagesFile.Close()

		sreader := NewControlFileReader(packagesReader)

		for {
			stanza, err := sreader.ReadStanza()
			if err != nil {
				return err
			}
			if stanza == nil {
				break
			}

			var p *Package

			if kind == "binary" {
				p = NewPackageFromControlFile(stanza)
			} else if kind == "source" {
				p, err = NewSourcePackageFromControlFile(stanza)
				if err != nil {
					return err
				}
			}
			list.Add(p)
		}
	}

	progress.Printf("Saving packages to database...\n")

	progress.InitBar(int64(list.Len()), false)

	packageCollection.db.StartBatch()
	count := 0

	// Save package meta information to DB
	err := list.ForEach(func(p *Package) error {
		progress.AddBar(1)
		count++
		if count > 1000 {
			count = 0
			err := packageCollection.db.FinishBatch()
			if err != nil {
				return err
			}
			packageCollection.db.StartBatch()
		}
		return packageCollection.Update(p)
	})
	if err != nil {
		return fmt.Errorf("unable to save packages to db: %s", err)
	}

	err = packageCollection.db.FinishBatch()
	if err != nil {
		return fmt.Errorf("unable to save packages to db: %s", err)
	}

	progress.ShutdownBar()

	progress.Printf("Building download queue...\n")

	// Build download queue
	queued := make(map[string]PackageDownloadTask, list.Len())
	count = 0
	downloadSize := int64(0)

	err = list.ForEach(func(p *Package) error {
		list, err := p.DownloadList(packagePool)
		if err != nil {
			return err
		}

		for _, task := range list {
			key := task.RepoURI + "-" + task.DestinationPath
			_, found := queued[key]
			if !found {
				count++
				downloadSize += task.Checksums.Size
				queued[key] = task
			}
		}

		return nil
	})
	if err != nil {
		return fmt.Errorf("unable to build download queue: %s", err)
	}

	repo.packageRefs = NewPackageRefListFromPackageList(list)
	// free up package list, we don't need it after this point
	list = nil

	progress.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))

	progress.InitBar(downloadSize, true)

	// Download all package files
	ch := make(chan error, len(queued))

	for _, task := range queued {
		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
	}

	// We don't need queued after this point
	queued = nil

	// Wait for all downloads to finish
	errors := make([]string, 0)

	for count > 0 {
		err = <-ch
		if err != nil {
			errors = append(errors, err.Error())
		}
		count--
	}

	progress.ShutdownBar()

	if len(errors) > 0 {
		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
	}

	repo.LastDownloadDate = time.Now()

	return nil
}
Esempio n. 2
0
// DownloadPackageIndexes downloads & parses package index files
func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
	ignoreMismatch bool) error {
	if repo.packageList != nil {
		panic("packageList != nil")
	}
	repo.packageList = NewPackageList()

	// Download and parse all Packages & Source files
	packagesURLs := [][]string{}

	if repo.IsFlat() {
		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
		if repo.DownloadSources {
			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
		}
	} else {
		for _, component := range repo.Components {
			for _, architecture := range repo.Architectures {
				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
				if repo.DownloadUdebs {
					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
				}
			}
			if repo.DownloadSources {
				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
			}
		}
	}

	for _, info := range packagesURLs {
		url, kind := info[0], info[1]
		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
		if err != nil {
			return err
		}
		defer packagesFile.Close()

		stat, _ := packagesFile.Stat()
		progress.InitBar(stat.Size(), true)

		sreader := NewControlFileReader(packagesReader)

		for {
			stanza, err := sreader.ReadStanza()
			if err != nil {
				return err
			}
			if stanza == nil {
				break
			}

			off, _ := packagesFile.Seek(0, 1)
			progress.SetBar(int(off))

			var p *Package

			if kind == "binary" {
				p = NewPackageFromControlFile(stanza)
			} else if kind == "udeb" {
				p = NewUdebPackageFromControlFile(stanza)
			} else if kind == "source" {
				p, err = NewSourcePackageFromControlFile(stanza)
				if err != nil {
					return err
				}
			}
			err = repo.packageList.Add(p)
			if err != nil {
				if _, ok := err.(*PackageConflictError); ok {
					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
				} else {
					return err
				}
			}

			err = collectionFactory.PackageCollection().Update(p)
			if err != nil {
				return err
			}
		}

		progress.ShutdownBar()
	}

	return nil
}
Esempio n. 3
0
// Download downloads all repo files
func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
	packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
	list := NewPackageList()

	progress.Printf("Downloading & parsing package files...\n")

	// Download and parse all Packages & Source files
	packagesURLs := [][]string{}

	if repo.IsFlat() {
		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
		if repo.DownloadSources {
			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
		}
	} else {
		for _, component := range repo.Components {
			for _, architecture := range repo.Architectures {
				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
			}
			if repo.DownloadSources {
				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
			}
		}
	}

	for _, info := range packagesURLs {
		url, kind := info[0], info[1]
		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
		if err != nil {
			return err
		}
		defer packagesFile.Close()

		stat, _ := packagesFile.Stat()
		progress.InitBar(stat.Size(), true)

		sreader := NewControlFileReader(packagesReader)

		for {
			stanza, err := sreader.ReadStanza()
			if err != nil {
				return err
			}
			if stanza == nil {
				break
			}

			off, _ := packagesFile.Seek(0, 1)
			progress.SetBar(int(off))

			var p *Package

			if kind == "binary" {
				p = NewPackageFromControlFile(stanza)
			} else if kind == "source" {
				p, err = NewSourcePackageFromControlFile(stanza)
				if err != nil {
					return err
				}
			}
			err = list.Add(p)
			if err != nil {
				return err
			}

			err = collectionFactory.PackageCollection().Update(p)
			if err != nil {
				return err
			}
		}

		progress.ShutdownBar()
	}

	var err error

	if repo.Filter != "" {
		progress.Printf("Applying filter...\n")

		list.PrepareIndex()

		emptyList := NewPackageList()
		emptyList.PrepareIndex()

		origPackages := list.Len()
		list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
		if err != nil {
			return err
		}

		progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
	}

	progress.Printf("Building download queue...\n")

	// Build download queue
	queued := make(map[string]PackageDownloadTask, list.Len())
	count := 0
	downloadSize := int64(0)

	err = list.ForEach(func(p *Package) error {
		list, err2 := p.DownloadList(packagePool)
		if err2 != nil {
			return err2
		}
		p.files = nil

		for _, task := range list {
			key := task.RepoURI + "-" + task.DestinationPath
			_, found := queued[key]
			if !found {
				count++
				downloadSize += task.Checksums.Size
				queued[key] = task
			}
		}

		return nil
	})
	if err != nil {
		return fmt.Errorf("unable to build download queue: %s", err)
	}

	repo.packageRefs = NewPackageRefListFromPackageList(list)
	// free up package list, we don't need it after this point
	list = nil

	progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))

	progress.InitBar(downloadSize, true)

	// Download all package files
	ch := make(chan error, len(queued))

	for _, task := range queued {
		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
	}

	// We don't need queued after this point
	queued = nil

	// Wait for all downloads to finish
	errors := make([]string, 0)

	for count > 0 {
		err = <-ch
		if err != nil {
			errors = append(errors, err.Error())
		}
		count--
	}

	progress.ShutdownBar()

	if len(errors) > 0 {
		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
	}

	repo.LastDownloadDate = time.Now()

	return nil
}