// Download downloads all repo files
func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
list := NewPackageList()
progress.Printf("Downloading & parsing package files...\n")
// Download and parse all Packages & Source files
packagesURLs := [][]string{}
if repo.IsFlat() {
packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
}
} else {
for _, component := range repo.Components {
for _, architecture := range repo.Architectures {
packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
}
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
}
}
}
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
if err != nil {
return err
}
defer packagesFile.Close()
stat, _ := packagesFile.Stat()
progress.InitBar(stat.Size(), true)
sreader := NewControlFileReader(packagesReader)
for {
stanza, err := sreader.ReadStanza()
if err != nil {
return err
}
if stanza == nil {
break
}
off, _ := packagesFile.Seek(0, 1)
progress.SetBar(int(off))
var p *Package
if kind == "binary" {
p = NewPackageFromControlFile(stanza)
} else if kind == "source" {
p, err = NewSourcePackageFromControlFile(stanza)
if err != nil {
return err
}
}
err = list.Add(p)
if err != nil {
return err
}
err = collectionFactory.PackageCollection().Update(p)
if err != nil {
return err
}
}
progress.ShutdownBar()
}
var err error
if repo.Filter != "" {
progress.Printf("Applying filter...\n")
list.PrepareIndex()
emptyList := NewPackageList()
emptyList.PrepareIndex()
origPackages := list.Len()
list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
if err != nil {
return err
}
progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
}
progress.Printf("Building download queue...\n")
// Build download queue
queued := make(map[string]PackageDownloadTask, list.Len())
count := 0
downloadSize := int64(0)
err = list.ForEach(func(p *Package) error {
list, err2 := p.DownloadList(packagePool)
if err2 != nil {
return err2
}
p.files = nil
for _, task := range list {
key := task.RepoURI + "-" + task.DestinationPath
_, found := queued[key]
if !found {
count++
downloadSize += task.Checksums.Size
queued[key] = task
}
}
return nil
})
if err != nil {
return fmt.Errorf("unable to build download queue: %s", err)
}
repo.packageRefs = NewPackageRefListFromPackageList(list)
// free up package list, we don't need it after this point
list = nil
progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
progress.InitBar(downloadSize, true)
// Download all package files
ch := make(chan error, len(queued))
for _, task := range queued {
d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
}
// We don't need queued after this point
queued = nil
// Wait for all downloads to finish
errors := make([]string, 0)
for count > 0 {
err = <-ch
if err != nil {
errors = append(errors, err.Error())
}
count--
}
progress.ShutdownBar()
if len(errors) > 0 {
return fmt.Errorf("download errors:\n %s\n", strings.Join(errors, "\n "))
}
repo.LastDownloadDate = time.Now()
return nil
}
func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
name := args[0]
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
force := context.Flags().Lookup("force").Value.Get().(bool)
if !force {
err = repo.CheckLock()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
if repo.Filter != "" {
context.Progress().Printf("Applying filter...\n")
var filterQuery deb.PackageQuery
filterQuery, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
var oldLen, newLen int
oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
}
var (
downloadSize int64
queue []deb.PackageDownloadTask
)
context.Progress().Printf("Building download queue...\n")
queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
defer func() {
// on any interruption, unlock the mirror
err := context.ReOpenDatabase()
if err == nil {
repo.MarkAsIdle()
context.CollectionFactory().RemoteRepoCollection().Update(repo)
}
}()
repo.MarkAsUpdating()
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CloseDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
// Catch ^C
sigch := make(chan os.Signal)
signal.Notify(sigch, os.Interrupt)
count := len(queue)
context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
// Download from the queue
context.Progress().InitBar(downloadSize, true)
// Download all package files
ch := make(chan error, count)
// In separate goroutine (to avoid blocking main), push queue to downloader
go func() {
for _, task := range queue {
context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
}
// We don't need queue after this point
queue = nil
}()
// Wait for all downloads to finish
errors := make([]string, 0)
for count > 0 {
select {
case <-sigch:
signal.Stop(sigch)
return fmt.Errorf("unable to update: interrupted")
case err = <-ch:
if err != nil {
errors = append(errors, err.Error())
}
count--
}
}
context.Progress().ShutdownBar()
signal.Stop(sigch)
if len(errors) > 0 {
return fmt.Errorf("unable to update: download errors:\n %s\n", strings.Join(errors, "\n "))
}
err = context.ReOpenDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
repo.FinalizeDownload()
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
return err
}
// aptly db cleanup
func aptlyDbCleanup(cmd *commander.Command, args []string) error {
var err error
if len(args) != 0 {
cmd.Usage()
return err
}
// collect information about references packages...
existingPackageRefs := deb.NewPackageRefList()
context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
}
return nil
})
if err != nil {
return err
}
err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
}
return nil
})
if err != nil {
return err
}
err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
if err != nil {
return err
}
existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
return nil
})
if err != nil {
return err
}
// ... and compare it to the list of all packages
context.Progress().Printf("Loading list of all packages...\n")
allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
toDelete := allPackageRefs.Substract(existingPackageRefs)
// delete packages that are no longer referenced
context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
// database can't err as collection factory already constructed
db, _ := context.Database()
db.StartBatch()
err = toDelete.ForEach(func(ref []byte) error {
return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
})
if err != nil {
return err
}
err = db.FinishBatch()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
}
// now, build a list of files that should be present in Repository (package pool)
context.Progress().Printf("Building list of files referenced by packages...\n")
referencedFiles := make([]string, 0, existingPackageRefs.Len())
context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
err = existingPackageRefs.ForEach(func(key []byte) error {
pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
if err2 != nil {
return err2
}
paths, err2 := pkg.FilepathList(context.PackagePool())
if err2 != nil {
return err2
}
referencedFiles = append(referencedFiles, paths...)
context.Progress().AddBar(1)
return nil
})
if err != nil {
return err
}
sort.Strings(referencedFiles)
context.Progress().ShutdownBar()
// build a list of files in the package pool
context.Progress().Printf("Building list of files in package pool...\n")
existingFiles, err := context.PackagePool().FilepathList(context.Progress())
if err != nil {
return fmt.Errorf("unable to collect file paths: %s", err)
}
// find files which are in the pool but not referenced by packages
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
// delete files that are no longer referenced
context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
if len(filesToDelete) > 0 {
context.Progress().InitBar(int64(len(filesToDelete)), false)
var size, totalSize int64
for _, file := range filesToDelete {
size, err = context.PackagePool().Remove(file)
if err != nil {
return err
}
context.Progress().AddBar(1)
totalSize += size
}
context.Progress().ShutdownBar()
context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
}
context.Progress().Printf("Compacting database...\n")
err = db.CompactDB()
return err
}
// aptly db cleanup
func aptlyDbCleanup(cmd *commander.Command, args []string) error {
var err error
if len(args) != 0 {
cmd.Usage()
return commander.ErrCommandError
}
verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
// collect information about references packages...
existingPackageRefs := deb.NewPackageRefList()
// used only in verbose mode to report package use source
packageRefSources := map[string][]string{}
context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
if verbose {
context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
}
err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("mirror %s", repo.Name)
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
}
err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("local repo %s", repo.Name)
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
}
err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
}
err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
if err != nil {
return err
}
existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
if verbose {
description := fmt.Sprintf("snapshot %s", snapshot.Name)
snapshot.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
}
err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
}
if published.SourceKind != "local" {
return nil
}
err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
if err != nil {
return err
}
for _, component := range published.Components() {
existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
if verbose {
description := fmt.Sprintf("published repository %s:%s/%s component %s",
published.Storage, published.Prefix, published.Distribution, component)
published.RefList(component).ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
// ... and compare it to the list of all packages
context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
toDelete := allPackageRefs.Substract(existingPackageRefs)
// delete packages that are no longer referenced
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
// database can't err as collection factory already constructed
db, _ := context.Database()
if toDelete.Len() > 0 {
if verbose {
context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
err = toDelete.ForEach(func(ref []byte) error {
context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
return nil
})
if err != nil {
return err
}
}
if !dryRun {
db.StartBatch()
err = toDelete.ForEach(func(ref []byte) error {
return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
})
if err != nil {
return err
}
err = db.FinishBatch()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
}
} else {
context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
}
}
// now, build a list of files that should be present in Repository (package pool)
context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
referencedFiles := make([]string, 0, existingPackageRefs.Len())
context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
err = existingPackageRefs.ForEach(func(key []byte) error {
pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
if err2 != nil {
tail := ""
if verbose {
tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
}
if dryRun {
context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
string(key), err2, tail)
return nil
}
return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
}
paths, err2 := pkg.FilepathList(context.PackagePool())
if err2 != nil {
return err2
}
referencedFiles = append(referencedFiles, paths...)
context.Progress().AddBar(1)
return nil
})
if err != nil {
return err
}
sort.Strings(referencedFiles)
context.Progress().ShutdownBar()
// build a list of files in the package pool
context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
existingFiles, err := context.PackagePool().FilepathList(context.Progress())
if err != nil {
return fmt.Errorf("unable to collect file paths: %s", err)
}
// find files which are in the pool but not referenced by packages
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
// delete files that are no longer referenced
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
if len(filesToDelete) > 0 {
if verbose {
context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
for _, file := range filesToDelete {
context.Progress().ColoredPrintf(" - @{r}%s@|", file)
}
}
if !dryRun {
context.Progress().InitBar(int64(len(filesToDelete)), false)
var size, totalSize int64
for _, file := range filesToDelete {
size, err = context.PackagePool().Remove(file)
if err != nil {
return err
}
context.Progress().AddBar(1)
totalSize += size
}
context.Progress().ShutdownBar()
context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
} else {
context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
}
}
if !dryRun {
context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
err = db.CompactDB()
} else {
context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
}
return err
}