// Download downloads all repo files
func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
list := NewPackageList()
progress.Printf("Downloading & parsing package files...\n")
// Download and parse all Packages & Source files
packagesURLs := [][]string{}
if repo.IsFlat() {
packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
}
} else {
for _, component := range repo.Components {
for _, architecture := range repo.Architectures {
packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
}
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
}
}
}
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
if err != nil {
return err
}
defer packagesFile.Close()
sreader := NewControlFileReader(packagesReader)
for {
stanza, err := sreader.ReadStanza()
if err != nil {
return err
}
if stanza == nil {
break
}
var p *Package
if kind == "binary" {
p = NewPackageFromControlFile(stanza)
} else if kind == "source" {
p, err = NewSourcePackageFromControlFile(stanza)
if err != nil {
return err
}
}
list.Add(p)
}
}
progress.Printf("Saving packages to database...\n")
progress.InitBar(int64(list.Len()), false)
packageCollection.db.StartBatch()
count := 0
// Save package meta information to DB
err := list.ForEach(func(p *Package) error {
progress.AddBar(1)
count++
if count > 1000 {
count = 0
err := packageCollection.db.FinishBatch()
if err != nil {
return err
}
packageCollection.db.StartBatch()
}
return packageCollection.Update(p)
})
if err != nil {
return fmt.Errorf("unable to save packages to db: %s", err)
}
err = packageCollection.db.FinishBatch()
if err != nil {
return fmt.Errorf("unable to save packages to db: %s", err)
}
progress.ShutdownBar()
progress.Printf("Building download queue...\n")
// Build download queue
queued := make(map[string]PackageDownloadTask, list.Len())
count = 0
downloadSize := int64(0)
err = list.ForEach(func(p *Package) error {
list, err := p.DownloadList(packagePool)
if err != nil {
return err
}
for _, task := range list {
key := task.RepoURI + "-" + task.DestinationPath
_, found := queued[key]
if !found {
count++
downloadSize += task.Checksums.Size
queued[key] = task
}
}
return nil
})
if err != nil {
return fmt.Errorf("unable to build download queue: %s", err)
}
repo.packageRefs = NewPackageRefListFromPackageList(list)
// free up package list, we don't need it after this point
list = nil
progress.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))
progress.InitBar(downloadSize, true)
// Download all package files
ch := make(chan error, len(queued))
for _, task := range queued {
d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
}
// We don't need queued after this point
queued = nil
// Wait for all downloads to finish
errors := make([]string, 0)
for count > 0 {
err = <-ch
if err != nil {
errors = append(errors, err.Error())
}
count--
}
progress.ShutdownBar()
if len(errors) > 0 {
return fmt.Errorf("download errors:\n %s\n", strings.Join(errors, "\n "))
}
repo.LastDownloadDate = time.Now()
return nil
}
// DownloadPackageIndexes downloads & parses package index files
func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
ignoreMismatch bool) error {
if repo.packageList != nil {
panic("packageList != nil")
}
repo.packageList = NewPackageList()
// Download and parse all Packages & Source files
packagesURLs := [][]string{}
if repo.IsFlat() {
packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
}
} else {
for _, component := range repo.Components {
for _, architecture := range repo.Architectures {
packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
if repo.DownloadUdebs {
packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
}
}
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
}
}
}
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
if err != nil {
return err
}
defer packagesFile.Close()
stat, _ := packagesFile.Stat()
progress.InitBar(stat.Size(), true)
sreader := NewControlFileReader(packagesReader)
for {
stanza, err := sreader.ReadStanza()
if err != nil {
return err
}
if stanza == nil {
break
}
off, _ := packagesFile.Seek(0, 1)
progress.SetBar(int(off))
var p *Package
if kind == "binary" {
p = NewPackageFromControlFile(stanza)
} else if kind == "udeb" {
p = NewUdebPackageFromControlFile(stanza)
} else if kind == "source" {
p, err = NewSourcePackageFromControlFile(stanza)
if err != nil {
return err
}
}
err = repo.packageList.Add(p)
if err != nil {
if _, ok := err.(*PackageConflictError); ok {
progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
} else {
return err
}
}
err = collectionFactory.PackageCollection().Update(p)
if err != nil {
return err
}
}
progress.ShutdownBar()
}
return nil
}
// Download downloads all repo files
func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
list := NewPackageList()
progress.Printf("Downloading & parsing package files...\n")
// Download and parse all Packages & Source files
packagesURLs := [][]string{}
if repo.IsFlat() {
packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
}
} else {
for _, component := range repo.Components {
for _, architecture := range repo.Architectures {
packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
}
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
}
}
}
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
if err != nil {
return err
}
defer packagesFile.Close()
stat, _ := packagesFile.Stat()
progress.InitBar(stat.Size(), true)
sreader := NewControlFileReader(packagesReader)
for {
stanza, err := sreader.ReadStanza()
if err != nil {
return err
}
if stanza == nil {
break
}
off, _ := packagesFile.Seek(0, 1)
progress.SetBar(int(off))
var p *Package
if kind == "binary" {
p = NewPackageFromControlFile(stanza)
} else if kind == "source" {
p, err = NewSourcePackageFromControlFile(stanza)
if err != nil {
return err
}
}
err = list.Add(p)
if err != nil {
return err
}
err = collectionFactory.PackageCollection().Update(p)
if err != nil {
return err
}
}
progress.ShutdownBar()
}
var err error
if repo.Filter != "" {
progress.Printf("Applying filter...\n")
list.PrepareIndex()
emptyList := NewPackageList()
emptyList.PrepareIndex()
origPackages := list.Len()
list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
if err != nil {
return err
}
progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
}
progress.Printf("Building download queue...\n")
// Build download queue
queued := make(map[string]PackageDownloadTask, list.Len())
count := 0
downloadSize := int64(0)
err = list.ForEach(func(p *Package) error {
list, err2 := p.DownloadList(packagePool)
if err2 != nil {
return err2
}
p.files = nil
for _, task := range list {
key := task.RepoURI + "-" + task.DestinationPath
_, found := queued[key]
if !found {
count++
downloadSize += task.Checksums.Size
queued[key] = task
}
}
return nil
})
if err != nil {
return fmt.Errorf("unable to build download queue: %s", err)
}
repo.packageRefs = NewPackageRefListFromPackageList(list)
// free up package list, we don't need it after this point
list = nil
progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
progress.InitBar(downloadSize, true)
// Download all package files
ch := make(chan error, len(queued))
for _, task := range queued {
d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
}
// We don't need queued after this point
queued = nil
// Wait for all downloads to finish
errors := make([]string, 0)
for count > 0 {
err = <-ch
if err != nil {
errors = append(errors, err.Error())
}
count--
}
progress.ShutdownBar()
if len(errors) > 0 {
return fmt.Errorf("download errors:\n %s\n", strings.Join(errors, "\n "))
}
repo.LastDownloadDate = time.Now()
return nil
}