// Download downloads all repo files func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory, packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error { list := NewPackageList() progress.Printf("Downloading & parsing package files...\n") // Download and parse all Packages & Source files packagesURLs := [][]string{} if repo.IsFlat() { packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"}) if repo.DownloadSources { packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"}) } } else { for _, component := range repo.Components { for _, architecture := range repo.Architectures { packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"}) } if repo.DownloadSources { packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"}) } } } for _, info := range packagesURLs { url, kind := info[0], info[1] packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch) if err != nil { return err } defer packagesFile.Close() stat, _ := packagesFile.Stat() progress.InitBar(stat.Size(), true) sreader := NewControlFileReader(packagesReader) for { stanza, err := sreader.ReadStanza() if err != nil { return err } if stanza == nil { break } off, _ := packagesFile.Seek(0, 1) progress.SetBar(int(off)) var p *Package if kind == "binary" { p = NewPackageFromControlFile(stanza) } else if kind == "source" { p, err = NewSourcePackageFromControlFile(stanza) if err != nil { return err } } err = list.Add(p) if err != nil { return err } err = collectionFactory.PackageCollection().Update(p) if err != nil { return err } } progress.ShutdownBar() } var err error if repo.Filter != "" { progress.Printf("Applying filter...\n") list.PrepareIndex() emptyList := NewPackageList() emptyList.PrepareIndex() origPackages := list.Len() list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures) if err != nil { return err } progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len()) } progress.Printf("Building download queue...\n") // Build download queue queued := make(map[string]PackageDownloadTask, list.Len()) count := 0 downloadSize := int64(0) err = list.ForEach(func(p *Package) error { list, err2 := p.DownloadList(packagePool) if err2 != nil { return err2 } p.files = nil for _, task := range list { key := task.RepoURI + "-" + task.DestinationPath _, found := queued[key] if !found { count++ downloadSize += task.Checksums.Size queued[key] = task } } return nil }) if err != nil { return fmt.Errorf("unable to build download queue: %s", err) } repo.packageRefs = NewPackageRefListFromPackageList(list) // free up package list, we don't need it after this point list = nil progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize)) progress.InitBar(downloadSize, true) // Download all package files ch := make(chan error, len(queued)) for _, task := range queued { d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch) } // We don't need queued after this point queued = nil // Wait for all downloads to finish errors := make([]string, 0) for count > 0 { err = <-ch if err != nil { errors = append(errors, err.Error()) } count-- } progress.ShutdownBar() if len(errors) > 0 { return fmt.Errorf("download errors:\n %s\n", strings.Join(errors, "\n ")) } repo.LastDownloadDate = time.Now() return nil }
func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { var err error if len(args) != 1 { cmd.Usage() return commander.ErrCommandError } name := args[0] repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name) if err != nil { return fmt.Errorf("unable to update: %s", err) } err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) if err != nil { return fmt.Errorf("unable to update: %s", err) } force := context.Flags().Lookup("force").Value.Get().(bool) if !force { err = repo.CheckLock() if err != nil { return fmt.Errorf("unable to update: %s", err) } } ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool) maxTries := context.Flags().Lookup("max-tries").Value.Get().(int) verifier, err := getVerifier(context.Flags()) if err != nil { return fmt.Errorf("unable to initialize GPG verifier: %s", err) } err = repo.Fetch(context.Downloader(), verifier) if err != nil { return fmt.Errorf("unable to update: %s", err) } context.Progress().Printf("Downloading & parsing package files...\n") err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries) if err != nil { return fmt.Errorf("unable to update: %s", err) } if repo.Filter != "" { context.Progress().Printf("Applying filter...\n") var filterQuery deb.PackageQuery filterQuery, err = query.Parse(repo.Filter) if err != nil { return fmt.Errorf("unable to update: %s", err) } var oldLen, newLen int oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery) if err != nil { return fmt.Errorf("unable to update: %s", err) } context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen) } var ( downloadSize int64 queue []deb.PackageDownloadTask ) context.Progress().Printf("Building download queue...\n") queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool()) if err != nil { return fmt.Errorf("unable to update: %s", err) } defer func() { // on any interruption, unlock the mirror err := context.ReOpenDatabase() if err == nil { repo.MarkAsIdle() context.CollectionFactory().RemoteRepoCollection().Update(repo) } }() repo.MarkAsUpdating() err = context.CollectionFactory().RemoteRepoCollection().Update(repo) if err != nil { return fmt.Errorf("unable to update: %s", err) } err = context.CloseDatabase() if err != nil { return fmt.Errorf("unable to update: %s", err) } // Catch ^C sigch := make(chan os.Signal) signal.Notify(sigch, os.Interrupt) count := len(queue) context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize)) // Download from the queue context.Progress().InitBar(downloadSize, true) // Download all package files ch := make(chan error, count) // In separate goroutine (to avoid blocking main), push queue to downloader go func() { for _, task := range queue { context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries) } // We don't need queue after this point queue = nil }() // Wait for all downloads to finish errors := make([]string, 0) for count > 0 { select { case <-sigch: signal.Stop(sigch) return fmt.Errorf("unable to update: interrupted") case err = <-ch: if err != nil { errors = append(errors, err.Error()) } count-- } } context.Progress().ShutdownBar() signal.Stop(sigch) if len(errors) > 0 { return fmt.Errorf("unable to update: download errors:\n %s\n", strings.Join(errors, "\n ")) } err = context.ReOpenDatabase() if err != nil { return fmt.Errorf("unable to update: %s", err) } repo.FinalizeDownload() err = context.CollectionFactory().RemoteRepoCollection().Update(repo) if err != nil { return fmt.Errorf("unable to update: %s", err) } context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name) return err }
// aptly db cleanup func aptlyDbCleanup(cmd *commander.Command, args []string) error { var err error if len(args) != 0 { cmd.Usage() return err } // collect information about references packages... existingPackageRefs := deb.NewPackageRefList() context.Progress().Printf("Loading mirrors, local repos and snapshots...\n") err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) if err != nil { return err } if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) } return nil }) if err != nil { return err } err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) if err != nil { return err } if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) } return nil }) if err != nil { return err } err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) if err != nil { return err } existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false) return nil }) if err != nil { return err } // ... and compare it to the list of all packages context.Progress().Printf("Loading list of all packages...\n") allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs() toDelete := allPackageRefs.Substract(existingPackageRefs) // delete packages that are no longer referenced context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len()) // database can't err as collection factory already constructed db, _ := context.Database() db.StartBatch() err = toDelete.ForEach(func(ref []byte) error { return context.CollectionFactory().PackageCollection().DeleteByKey(ref) }) if err != nil { return err } err = db.FinishBatch() if err != nil { return fmt.Errorf("unable to write to DB: %s", err) } // now, build a list of files that should be present in Repository (package pool) context.Progress().Printf("Building list of files referenced by packages...\n") referencedFiles := make([]string, 0, existingPackageRefs.Len()) context.Progress().InitBar(int64(existingPackageRefs.Len()), false) err = existingPackageRefs.ForEach(func(key []byte) error { pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key) if err2 != nil { return err2 } paths, err2 := pkg.FilepathList(context.PackagePool()) if err2 != nil { return err2 } referencedFiles = append(referencedFiles, paths...) context.Progress().AddBar(1) return nil }) if err != nil { return err } sort.Strings(referencedFiles) context.Progress().ShutdownBar() // build a list of files in the package pool context.Progress().Printf("Building list of files in package pool...\n") existingFiles, err := context.PackagePool().FilepathList(context.Progress()) if err != nil { return fmt.Errorf("unable to collect file paths: %s", err) } // find files which are in the pool but not referenced by packages filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles) // delete files that are no longer referenced context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete)) if len(filesToDelete) > 0 { context.Progress().InitBar(int64(len(filesToDelete)), false) var size, totalSize int64 for _, file := range filesToDelete { size, err = context.PackagePool().Remove(file) if err != nil { return err } context.Progress().AddBar(1) totalSize += size } context.Progress().ShutdownBar() context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize)) } context.Progress().Printf("Compacting database...\n") err = db.CompactDB() return err }
// aptly db cleanup func aptlyDbCleanup(cmd *commander.Command, args []string) error { var err error if len(args) != 0 { cmd.Usage() return commander.ErrCommandError } verbose := context.Flags().Lookup("verbose").Value.Get().(bool) dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool) // collect information about references packages... existingPackageRefs := deb.NewPackageRefList() // used only in verbose mode to report package use source packageRefSources := map[string][]string{} context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|") if verbose { context.Progress().ColoredPrintf("@{y}Loading mirrors:@|") } err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { if verbose { context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name) } err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo) if err != nil { return err } if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true) if verbose { description := fmt.Sprintf("mirror %s", repo.Name) repo.RefList().ForEach(func(key []byte) error { packageRefSources[string(key)] = append(packageRefSources[string(key)], description) return nil }) } } return nil }) if err != nil { return err } if verbose { context.Progress().ColoredPrintf("@{y}Loading local repos:@|") } err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { if verbose { context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name) } err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo) if err != nil { return err } if repo.RefList() != nil { existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true) if verbose { description := fmt.Sprintf("local repo %s", repo.Name) repo.RefList().ForEach(func(key []byte) error { packageRefSources[string(key)] = append(packageRefSources[string(key)], description) return nil }) } } return nil }) if err != nil { return err } if verbose { context.Progress().ColoredPrintf("@{y}Loading snapshots:@|") } err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error { if verbose { context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name) } err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot) if err != nil { return err } existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true) if verbose { description := fmt.Sprintf("snapshot %s", snapshot.Name) snapshot.RefList().ForEach(func(key []byte) error { packageRefSources[string(key)] = append(packageRefSources[string(key)], description) return nil }) } return nil }) if err != nil { return err } if verbose { context.Progress().ColoredPrintf("@{y}Loading published repositories:@|") } err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error { if verbose { context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution) } if published.SourceKind != "local" { return nil } err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory()) if err != nil { return err } for _, component := range published.Components() { existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true) if verbose { description := fmt.Sprintf("published repository %s:%s/%s component %s", published.Storage, published.Prefix, published.Distribution, component) published.RefList(component).ForEach(func(key []byte) error { packageRefSources[string(key)] = append(packageRefSources[string(key)], description) return nil }) } } return nil }) if err != nil { return err } // ... and compare it to the list of all packages context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|") allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs() toDelete := allPackageRefs.Substract(existingPackageRefs) // delete packages that are no longer referenced context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len()) // database can't err as collection factory already constructed db, _ := context.Database() if toDelete.Len() > 0 { if verbose { context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|") err = toDelete.ForEach(func(ref []byte) error { context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref)) return nil }) if err != nil { return err } } if !dryRun { db.StartBatch() err = toDelete.ForEach(func(ref []byte) error { return context.CollectionFactory().PackageCollection().DeleteByKey(ref) }) if err != nil { return err } err = db.FinishBatch() if err != nil { return fmt.Errorf("unable to write to DB: %s", err) } } else { context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|") } } // now, build a list of files that should be present in Repository (package pool) context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|") referencedFiles := make([]string, 0, existingPackageRefs.Len()) context.Progress().InitBar(int64(existingPackageRefs.Len()), false) err = existingPackageRefs.ForEach(func(key []byte) error { pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key) if err2 != nil { tail := "" if verbose { tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", ")) } if dryRun { context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s", string(key), err2, tail) return nil } return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail) } paths, err2 := pkg.FilepathList(context.PackagePool()) if err2 != nil { return err2 } referencedFiles = append(referencedFiles, paths...) context.Progress().AddBar(1) return nil }) if err != nil { return err } sort.Strings(referencedFiles) context.Progress().ShutdownBar() // build a list of files in the package pool context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|") existingFiles, err := context.PackagePool().FilepathList(context.Progress()) if err != nil { return fmt.Errorf("unable to collect file paths: %s", err) } // find files which are in the pool but not referenced by packages filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles) // delete files that are no longer referenced context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete)) if len(filesToDelete) > 0 { if verbose { context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|") for _, file := range filesToDelete { context.Progress().ColoredPrintf(" - @{r}%s@|", file) } } if !dryRun { context.Progress().InitBar(int64(len(filesToDelete)), false) var size, totalSize int64 for _, file := range filesToDelete { size, err = context.PackagePool().Remove(file) if err != nil { return err } context.Progress().AddBar(1) totalSize += size } context.Progress().ShutdownBar() context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize)) } else { context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|") } } if !dryRun { context.Progress().ColoredPrintf("@{w!}Compacting database...@|") err = db.CompactDB() } else { context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|") } return err }