func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
var err error
if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
cmd.Usage()
return commander.ErrCommandError
}
downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
var (
mirrorName, archiveURL, distribution string
components []string
)
mirrorName = args[0]
if len(args) == 2 {
archiveURL, distribution, components, err = deb.ParsePPA(args[1], context.Config())
if err != nil {
return err
}
} else {
archiveURL, distribution, components = args[1], args[2], args[3:]
}
repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
downloadSources, downloadUdebs)
if err != nil {
return fmt.Errorf("unable to create mirror: %s", err)
}
repo.Filter = context.Flags().Lookup("filter").Value.String()
repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to create mirror: %s", err)
}
}
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
if err != nil {
return fmt.Errorf("unable to fetch mirror: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
if err != nil {
return fmt.Errorf("unable to add mirror: %s", err)
}
fmt.Printf("\nMirror %s successfully added.\nYou can run 'aptly mirror update %s' to download repository contents.\n", repo, repo.Name)
return err
}
func (a *Api) MirrorCreate(name, url, filter, distribution string, components []string, withSources, withUdebs, filterWithDeps, forceComponents bool) (*deb.RemoteRepo, error) {
repo, err := deb.NewRemoteRepo(name, url, distribution, components, a.Ctx().ArchitecturesList(), withSources, withUdebs)
if err != nil {
return nil, fmt.Errorf("unable to create mirror: %s", err)
}
repo.Filter = filter
repo.FilterWithDeps = filterWithDeps
repo.SkipComponentCheck = forceComponents
if repo.Filter != "" {
_, err := query.Parse(repo.Filter)
if err != nil {
return nil, fmt.Errorf("unable to create mirror: %s", err)
}
}
_, err = getVerifier(false)
if err != nil {
return nil, fmt.Errorf("unable to init GPG: %s", err)
}
err = repo.Fetch(a.Ctx().Downloader(), nil)
if err != nil {
return nil, fmt.Errorf("unable to fetch mirror: %s", err)
}
err = a.Ctx().CollectionFactory().RemoteRepoCollection().Add(repo)
if err != nil {
return nil, fmt.Errorf("unable to add mirror: %s", err)
}
return repo, nil
}
func aptlyRepoRemove(cmd *commander.Command, args []string) error {
var err error
if len(args) < 2 {
cmd.Usage()
return commander.ErrCommandError
}
name := args[0]
repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
context.Progress().Printf("Loading packages...\n")
list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
queries := make([]deb.PackageQuery, len(args)-1)
for i := 0; i < len(args)-1; i++ {
queries[i], err = query.Parse(args[i+1])
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
}
list.PrepareIndex()
toRemove, err := list.Filter(queries, false, nil, 0, nil)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
toRemove.ForEach(func(p *deb.Package) error {
list.Remove(p)
context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
return nil
})
if context.Flags().Lookup("dry-run").Value.Get().(bool) {
context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
} else {
repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
err = context.CollectionFactory().LocalRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to save: %s", err)
}
}
return err
}
func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
err = repo.CheckLock()
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
context.Flags().Visit(func(flag *flag.Flag) {
switch flag.Name {
case "filter":
repo.Filter = flag.Value.String()
case "filter-with-deps":
repo.FilterWithDeps = flag.Value.Get().(bool)
case "with-sources":
repo.DownloadSources = flag.Value.Get().(bool)
case "with-udebs":
repo.DownloadUdebs = flag.Value.Get().(bool)
}
})
if repo.IsFlat() && repo.DownloadUdebs {
return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
}
if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
}
if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
repo.Architectures = context.ArchitecturesList()
err = repo.Fetch(context.Downloader(), nil)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
}
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
fmt.Printf("Mirror %s successfully updated.\n", repo)
return err
}
// Common piece of code to show list of packages,
// with searching & details if requested
func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
result := []*deb.Package{}
list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
if err != nil {
c.Fail(404, err)
return
}
queryS := c.Request.URL.Query().Get("q")
if queryS != "" {
q, err := query.Parse(c.Request.URL.Query().Get("q"))
if err != nil {
c.Fail(400, err)
return
}
withDeps := c.Request.URL.Query().Get("withDeps") == "1"
architecturesList := []string{}
if withDeps {
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = list.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
return
}
}
list.PrepareIndex()
list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
if err != nil {
c.Fail(500, fmt.Errorf("unable to search: %s", err))
return
}
}
if c.Request.URL.Query().Get("format") == "details" {
list.ForEach(func(p *deb.Package) error {
result = append(result, p)
return nil
})
c.JSON(200, result)
} else {
c.JSON(200, list.Strings())
}
}
func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
name := args[0]
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
verifier, err := getVerifier(context.flags)
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
var filterQuery deb.PackageQuery
if repo.Filter != "" {
filterQuery, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch,
context.DependencyOptions(), filterQuery)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
return err
}
func aptlyPackageShow(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
q, err := query.Parse(args[0])
if err != nil {
return fmt.Errorf("unable to show: %s", err)
}
withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
w := bufio.NewWriter(os.Stdout)
result := q.Query(context.CollectionFactory().PackageCollection())
err = result.ForEach(func(p *deb.Package) error {
p.Stanza().WriteTo(w)
w.Flush()
fmt.Printf("\n")
if withFiles {
fmt.Printf("Files in the pool:\n")
for _, f := range p.Files() {
path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
if err != nil {
return err
}
fmt.Printf(" %s\n", path)
}
fmt.Printf("\n")
}
if withReferences {
fmt.Printf("References to package:\n")
printReferencesTo(p)
fmt.Printf("\n")
}
return nil
})
if err != nil {
return fmt.Errorf("unable to show: %s", err)
}
return err
}
func (a *Api) MirrorEdit(name string, visitor func(*deb.RemoteRepo) error) (*deb.RemoteRepo, error) {
collection := a.Ctx().CollectionFactory().RemoteRepoCollection()
collection.Lock()
defer collection.Unlock()
repo, err := collection.ByName(name)
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
err = repo.CheckLock()
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
err = visitor(repo)
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
if repo.IsFlat() && repo.DownloadUdebs {
return nil, fmt.Errorf("unable to edit: flat mirrors don't support udebs")
}
if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
}
if repo.Architectures != nil {
err = repo.Fetch(a.Ctx().Downloader(), nil)
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
}
err = collection.Update(repo)
if err != nil {
return nil, fmt.Errorf("unable to edit: %s", err)
}
return repo, nil
}
func aptlyPackageSearch(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
q, err := query.Parse(args[0])
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
result := q.Query(context.CollectionFactory().PackageCollection())
result.ForEach(func(p *deb.Package) error {
context.Progress().Printf("%s\n", p)
return nil
})
return err
}
func aptlyPackageSearch(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
q, err := query.Parse(args[0])
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
result := q.Query(context.CollectionFactory().PackageCollection())
if result.Len() == 0 {
return fmt.Errorf("no results")
}
format := context.Flags().Lookup("format").Value.String()
PrintPackageList(result, format)
return err
}
func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
context.flags.Visit(func(flag *flag.Flag) {
switch flag.Name {
case "filter":
repo.Filter = flag.Value.String()
case "filter-with-deps":
repo.FilterWithDeps = flag.Value.Get().(bool)
}
})
if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
}
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
fmt.Printf("Mirror %s successfully updated.\n", repo)
return err
}
func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
var err error
if len(args) != 2 {
cmd.Usage()
return commander.ErrCommandError
}
name := args[0]
command := cmd.Parent.Name()
var reflist *deb.PackageRefList
if command == "snapshot" {
snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
reflist = snapshot.RefList()
} else if command == "mirror" {
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
reflist = repo.RefList()
} else if command == "repo" {
repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
reflist = repo.RefList()
} else {
panic("unknown command")
}
list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
list.PrepareIndex()
q, err := query.Parse(args[1])
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
architecturesList := []string{}
if withDeps {
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = list.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
}
result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
if result.Len() == 0 {
return fmt.Errorf("no results")
}
result.ForEach(func(p *deb.Package) error {
context.Progress().Printf("%s\n", p)
return nil
})
return err
}
func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
var err error
if len(args) < 3 {
cmd.Usage()
return commander.ErrCommandError
}
withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
// Load <source> snapshot
source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to filter: %s", err)
}
err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
if err != nil {
return fmt.Errorf("unable to filter: %s", err)
}
// Convert snapshot to package list
context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
context.Progress().Printf("Building indexes...\n")
packageList.PrepareIndex()
// Calculate architectures
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = packageList.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 && withDeps {
return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
// Initial queries out of arguments
queries := make([]deb.PackageQuery, len(args)-2)
for i, arg := range args[2:] {
queries[i], err = query.Parse(arg)
if err != nil {
return fmt.Errorf("unable to parse query: %s", err)
}
}
// Filter with dependencies as requested
result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
if err != nil {
return fmt.Errorf("unable to filter: %s", err)
}
// Create <destination> snapshot
destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
err = context.CollectionFactory().SnapshotCollection().Add(destination)
if err != nil {
return fmt.Errorf("unable to create snapshot: %s", err)
}
context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
return err
}
func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
var err error
if len(args) < 3 {
cmd.Usage()
return commander.ErrCommandError
}
command := cmd.Name()
dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
var (
srcRefList *deb.PackageRefList
srcRepo *deb.LocalRepo
)
if command == "copy" || command == "move" {
srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
if srcRepo.UUID == dstRepo.UUID {
return fmt.Errorf("unable to %s: source and destination are the same", command)
}
err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
srcRefList = srcRepo.RefList()
} else if command == "import" {
var srcRemoteRepo *deb.RemoteRepo
srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
if srcRemoteRepo.RefList() == nil {
return fmt.Errorf("unable to %s: mirror not updated", command)
}
srcRefList = srcRemoteRepo.RefList()
} else {
panic("unexpected command")
}
context.Progress().Printf("Loading packages...\n")
dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
srcList.PrepareIndex()
var architecturesList []string
withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
if withDeps {
dstList.PrepareIndex()
// Calculate architectures
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = dstList.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
}
queries := make([]deb.PackageQuery, len(args)-2)
for i := 0; i < len(args)-2; i++ {
queries[i], err = query.Parse(args[i+2])
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
}
toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
var verb string
if command == "move" {
verb = "moved"
} else if command == "copy" {
verb = "copied"
} else if command == "import" {
verb = "imported"
}
err = toProcess.ForEach(func(p *deb.Package) error {
err = dstList.Add(p)
if err != nil {
return err
}
if command == "move" {
srcList.Remove(p)
}
context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
return nil
})
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
if context.flags.Lookup("dry-run").Value.Get().(bool) {
context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
} else {
dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
if err != nil {
return fmt.Errorf("unable to save: %s", err)
}
if command == "move" {
srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
if err != nil {
return fmt.Errorf("unable to save: %s", err)
}
}
}
return err
}
func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
var err error
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
name := args[0]
repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
force := context.Flags().Lookup("force").Value.Get().(bool)
if !force {
err = repo.CheckLock()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
if repo.Filter != "" {
context.Progress().Printf("Applying filter...\n")
var filterQuery deb.PackageQuery
filterQuery, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
var oldLen, newLen int
oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
}
var (
downloadSize int64
queue []deb.PackageDownloadTask
)
context.Progress().Printf("Building download queue...\n")
queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
defer func() {
// on any interruption, unlock the mirror
err := context.ReOpenDatabase()
if err == nil {
repo.MarkAsIdle()
context.CollectionFactory().RemoteRepoCollection().Update(repo)
}
}()
repo.MarkAsUpdating()
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = context.CloseDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
// Catch ^C
sigch := make(chan os.Signal)
signal.Notify(sigch, os.Interrupt)
count := len(queue)
context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
// Download from the queue
context.Progress().InitBar(downloadSize, true)
// Download all package files
ch := make(chan error, count)
// In separate goroutine (to avoid blocking main), push queue to downloader
go func() {
for _, task := range queue {
context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
}
// We don't need queue after this point
queue = nil
}()
// Wait for all downloads to finish
errors := make([]string, 0)
for count > 0 {
select {
case <-sigch:
signal.Stop(sigch)
return fmt.Errorf("unable to update: interrupted")
case err = <-ch:
if err != nil {
errors = append(errors, err.Error())
}
count--
}
}
context.Progress().ShutdownBar()
signal.Stop(sigch)
if len(errors) > 0 {
return fmt.Errorf("unable to update: download errors:\n %s\n", strings.Join(errors, "\n "))
}
err = context.ReOpenDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
repo.FinalizeDownload()
err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
return err
}
func aptlyRepoInclude(cmd *commander.Command, args []string) error {
var err error
if len(args) < 1 {
cmd.Usage()
return commander.ErrCommandError
}
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
if verifier == nil {
verifier = &utils.GpgVerifier{}
}
forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
if err != nil {
return fmt.Errorf("error parsing -repo template: %s", err)
}
uploaders := (*deb.Uploaders)(nil)
uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
if uploadersFile != "" {
uploaders, err = deb.NewUploadersFromFile(uploadersFile)
if err != nil {
return err
}
for i := range uploaders.Rules {
uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
if err != nil {
return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
}
}
}
reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
var changesFiles, failedFiles, processedFiles []string
changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
for _, path := range changesFiles {
var changes *deb.Changes
changes, err = deb.NewChanges(path)
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", path, err)
continue
}
err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}
err = changes.Prepare()
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}
repoName := &bytes.Buffer{}
err = repoTemplate.Execute(repoName, changes.Stanza)
if err != nil {
return fmt.Errorf("error applying template to repo: %s", err)
}
context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}
currentUploaders := uploaders
if repo.Uploaders != nil {
currentUploaders = repo.Uploaders
for i := range currentUploaders.Rules {
currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
if err != nil {
return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
}
}
}
if currentUploaders != nil {
if err = currentUploaders.IsAllowed(changes); err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
changes.ChangesName, changes.SignatureKeys, err)
changes.Cleanup()
continue
}
}
err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return fmt.Errorf("unable to load repo: %s", err)
}
list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
var restriction deb.PackageQuery
restriction, err = changes.PackageQuery()
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
changes.Cleanup()
continue
}
var processedFiles2, failedFiles2 []string
processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
context.CollectionFactory().PackageCollection(), reporter, restriction)
if err != nil {
return fmt.Errorf("unable to import package files: %s", err)
}
repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
err = context.CollectionFactory().LocalRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to save: %s", err)
}
err = changes.Cleanup()
if err != nil {
return err
}
for _, file := range failedFiles2 {
failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
}
for _, file := range processedFiles2 {
processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
}
processedFiles = append(processedFiles, path)
}
if !noRemoveFiles {
processedFiles = utils.StrSliceDeduplicate(processedFiles)
for _, file := range processedFiles {
err := os.Remove(file)
if err != nil {
return fmt.Errorf("unable to remove file: %s", err)
}
}
}
if len(failedFiles) > 0 {
context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
for _, file := range failedFiles {
context.Progress().ColoredPrintf(" %s", file)
}
return fmt.Errorf("some files failed to be added")
}
return err
}
func aptlySnapshotPull(cmd *commander.Command, args []string) error {
var err error
if len(args) < 4 {
cmd.Usage()
return commander.ErrCommandError
}
noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
// Load <name> snapshot
snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
// Load <source> snapshot
source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
context.Progress().Printf("Dependencies would be pulled into snapshot:\n %s\nfrom snapshot:\n %s\nand result would be saved as new snapshot %s.\n",
snapshot, source, args[2])
// Convert snapshot to package list
context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
context.Progress().Printf("Building indexes...\n")
packageList.PrepareIndex()
sourcePackageList.PrepareIndex()
// Calculate architectures
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = packageList.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
for _, arch := range architecturesList {
archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
}
// Initial queries out of arguments
queries := make([]deb.PackageQuery, len(args)-3)
for i, arg := range args[3:] {
queries[i], err = query.Parse(arg)
if err != nil {
return fmt.Errorf("unable to parse query: %s", err)
}
// Add architecture filter
queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
}
// Filter with dependencies as requested
result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
result.PrepareIndex()
alreadySeen := map[string]bool{}
result.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
for _, p := range pS {
packageList.Remove(p)
context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
}
}
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
packageList.Add(pkg)
context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
}
alreadySeen[key] = true
return nil
})
alreadySeen = nil
if context.Flags().Lookup("dry-run").Value.Get().(bool) {
context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
} else {
// Create <destination> snapshot
destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
err = context.CollectionFactory().SnapshotCollection().Add(destination)
if err != nil {
return fmt.Errorf("unable to create snapshot: %s", err)
}
context.Progress().Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
}
return err
}
func (a *Api) MirrorUpdate(name string, force, ignoreChecksums bool) error {
var err error
repo, err := a.MirrorShow(name)
if !force {
err = repo.CheckLock()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
verifier, err := getVerifier(false)
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(a.Ctx().Downloader(), verifier)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = repo.DownloadPackageIndexes(a.Ctx().Progress(), a.Ctx().Downloader(), a.Ctx().CollectionFactory(), ignoreChecksums)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
if repo.Filter != "" {
var filterQuery deb.PackageQuery
filterQuery, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
_, _, err = repo.ApplyFilter(a.Ctx().DependencyOptions(), filterQuery)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
var queue []deb.PackageDownloadTask
queue, _, err = repo.BuildDownloadQueue(a.Ctx().PackagePool())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
defer func() {
// on any interruption, unlock the mirror
err := a.Ctx().ReOpenDatabase()
if err == nil {
repo.MarkAsIdle()
a.Ctx().CollectionFactory().RemoteRepoCollection().Update(repo)
}
}()
repo.MarkAsUpdating()
collection := a.Ctx().CollectionFactory().RemoteRepoCollection()
collection.Lock()
defer collection.Unlock()
collection.Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = a.Ctx().CloseDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
count := len(queue)
ch := make(chan error, count)
go func() {
for _, task := range queue {
a.Ctx().Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreChecksums)
}
// We don't need queue after this point
queue = nil
}()
errors := make([]string, 0)
for count > 0 {
select {
case err = <-ch:
if err != nil {
errors = append(errors, err.Error())
}
count--
}
}
if len(errors) > 0 {
return fmt.Errorf("unable to update: download errors:\n %s\n", strings.Join(errors, "\n "))
}
err = a.Ctx().ReOpenDatabase()
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
repo.FinalizeDownload()
collection.Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
return nil
}