// LinkFromPool links package file from pool to dist's pool location
func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
poolDir, err := p.PoolDirectory()
if err != nil {
return err
}
for i, f := range p.Files() {
sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return err
}
relPath := filepath.Join("pool", component, poolDir)
publishedDirectory := filepath.Join(prefix, relPath)
err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath)
if err != nil {
return err
}
if p.IsSource {
p.Extra()["Directory"] = relPath
} else {
p.Files()[i].downloadPath = relPath
}
}
return nil
}
// LinkFromPool links package file from pool to dist's pool location
func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
poolDir, err := p.PoolDirectory()
if err != nil {
return err
}
for i, f := range p.Files {
sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return err
}
relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
if err != nil {
return err
}
dir := filepath.Dir(relPath)
if p.IsSource {
p.Extra["Directory"] = dir
} else {
p.Files[i].downloadPath = dir
}
}
return nil
}
// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix, component string,
publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
var err error
referencedFiles := []string{}
if progress != nil {
progress.Printf("Cleaning up prefix %#v component %#v...\n", prefix, component)
}
for _, r := range collection.list {
if r.Prefix == prefix && r.Component == component {
err = collection.LoadComplete(r, collectionFactory)
if err != nil {
return err
}
packageList, err := NewPackageListFromRefList(r.RefList(), collectionFactory.PackageCollection(), progress)
if err != nil {
return err
}
packageList.ForEach(func(p *Package) error {
poolDir, err := p.PoolDirectory()
if err != nil {
return err
}
for _, f := range p.Files() {
referencedFiles = append(referencedFiles, filepath.Join(poolDir, f.Filename))
}
return nil
})
}
}
sort.Strings(referencedFiles)
rootPath := filepath.Join(prefix, "pool", component)
existingFiles, err := publishedStorage.Filelist(rootPath)
if err != nil {
return err
}
sort.Strings(existingFiles)
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
for _, file := range filesToDelete {
err = publishedStorage.Remove(filepath.Join(rootPath, file))
if err != nil {
return err
}
}
return nil
}
// RemoveFiles removes files that were created by Publish
//
// It can remove prefix fully, and part of pool (for specific component)
func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool, progress aptly.Progress) error {
// I. Easy: remove whole prefix (meta+packages)
if removePrefix {
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"), progress)
if err != nil {
return err
}
return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"), progress)
}
// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
if err != nil {
return err
}
// III. Complex: there are no other publishes with the same prefix + component
if removePoolComponent {
err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component), progress)
if err != nil {
return err
}
} else {
/// IV: Hard: should have removed published files from the pool + component
/// that are unique to this published repo
}
return nil
}
// RemoveFiles removes files that were created by Publish
//
// It can remove prefix fully, and part of pool (for specific component)
func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix bool,
removePoolComponents []string, progress aptly.Progress) error {
// I. Easy: remove whole prefix (meta+packages)
if removePrefix {
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"), progress)
if err != nil {
return err
}
return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"), progress)
}
// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
if err != nil {
return err
}
// III. Complex: there are no other publishes with the same prefix + component
for _, component := range removePoolComponents {
err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", component), progress)
if err != nil {
return err
}
}
return nil
}
// RemoveFiles removes files that were created by Publish
//
// It can remove prefix fully, and part of pool (for specific component)
func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool) error {
if removePrefix {
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"))
if err != nil {
return err
}
return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"))
}
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
if err != nil {
return err
}
if removePoolComponent {
err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
if err != nil {
return err
}
}
return nil
}
// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
var err error
referencedFiles := map[string][]string{}
if progress != nil {
progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", "))
}
for _, r := range collection.list {
if r.Prefix == prefix {
matches := false
repoComponents := r.Components()
for _, component := range components {
if utils.StrSliceHasItem(repoComponents, component) {
matches = true
break
}
}
if !matches {
continue
}
err = collection.LoadComplete(r, collectionFactory)
if err != nil {
return err
}
for _, component := range components {
if utils.StrSliceHasItem(repoComponents, component) {
packageList, err := NewPackageListFromRefList(r.RefList(component), collectionFactory.PackageCollection(), progress)
if err != nil {
return err
}
packageList.ForEach(func(p *Package) error {
poolDir, err := p.PoolDirectory()
if err != nil {
return err
}
for _, f := range p.Files() {
referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, f.Filename))
}
return nil
})
}
}
}
}
for _, component := range components {
sort.Strings(referencedFiles[component])
rootPath := filepath.Join(prefix, "pool", component)
existingFiles, err := publishedStorage.Filelist(rootPath)
if err != nil {
return err
}
sort.Strings(existingFiles)
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
for _, file := range filesToDelete {
err = publishedStorage.Remove(filepath.Join(rootPath, file))
if err != nil {
return err
}
}
}
return nil
}
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage,
collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress) error {
err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
if err != nil {
return err
}
basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
err = publishedStorage.MkDir(basePath)
if err != nil {
return err
}
if progress != nil {
progress.Printf("Loading packages...\n")
}
// Load all packages
list, err := NewPackageListFromRefList(p.RefList(), collectionFactory.PackageCollection(), progress)
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
if list.Len() == 0 {
return fmt.Errorf("source is empty")
}
if !p.rePublishing {
if len(p.Architectures) == 0 {
p.Architectures = list.Architectures(true)
}
if len(p.Architectures) == 0 {
return fmt.Errorf("unable to figure out list of architectures, please supply explicit list")
}
sort.Strings(p.Architectures)
}
var suffix string
if p.rePublishing {
suffix = ".tmp"
}
generatedFiles := map[string]utils.ChecksumInfo{}
renameMap := map[string]string{}
if progress != nil {
progress.Printf("Generating metadata files and linking package files...\n")
}
// For all architectures, generate release file
for _, arch := range p.Architectures {
if progress != nil {
progress.InitBar(int64(list.Len()), false)
}
var relativePath string
if arch == "source" {
relativePath = filepath.Join(p.Component, "source", "Sources")
} else {
relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
}
err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
if err != nil {
return err
}
var packagesFile *os.File
packagesFile, err = publishedStorage.CreateFile(filepath.Join(basePath, relativePath+suffix))
if err != nil {
return fmt.Errorf("unable to creates Packages file: %s", err)
}
if suffix != "" {
renameMap[filepath.Join(basePath, relativePath+suffix)] = filepath.Join(basePath, relativePath)
}
bufWriter := bufio.NewWriter(packagesFile)
err = list.ForEach(func(pkg *Package) error {
if progress != nil {
progress.AddBar(1)
}
if pkg.MatchesArchitecture(arch) {
err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
if err != nil {
return err
}
err = pkg.Stanza().WriteTo(bufWriter)
if err != nil {
return err
}
err = bufWriter.WriteByte('\n')
if err != nil {
return err
}
pkg.files = nil
pkg.deps = nil
pkg.extra = nil
}
return nil
})
if err != nil {
return fmt.Errorf("unable to process packages: %s", err)
}
err = bufWriter.Flush()
if err != nil {
return fmt.Errorf("unable to write Packages file: %s", err)
}
err = utils.CompressFile(packagesFile)
if err != nil {
return fmt.Errorf("unable to compress Packages files: %s", err)
}
if suffix != "" {
renameMap[filepath.Join(basePath, relativePath+suffix+".gz")] = filepath.Join(basePath, relativePath+".gz")
renameMap[filepath.Join(basePath, relativePath+suffix+".bz2")] = filepath.Join(basePath, relativePath+".bz2")
}
packagesFile.Close()
var checksumInfo utils.ChecksumInfo
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath] = checksumInfo
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix+".gz"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath+".gz"] = checksumInfo
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix+".bz2"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath+".bz2"] = checksumInfo
if progress != nil {
progress.ShutdownBar()
}
}
release := make(Stanza)
if p.Origin == "" {
release["Origin"] = p.Prefix + " " + p.Distribution
} else {
release["Origin"] = p.Origin
}
if p.Label == "" {
release["Label"] = p.Prefix + " " + p.Distribution
} else {
release["Label"] = p.Label
}
release["Codename"] = p.Distribution
release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
release["Components"] = p.Component
release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
release["Description"] = " Generated by aptly\n"
release["MD5Sum"] = "\n"
release["SHA1"] = "\n"
release["SHA256"] = "\n"
for path, info := range generatedFiles {
release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
}
releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"+suffix))
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
if suffix != "" {
renameMap[filepath.Join(basePath, "Release"+suffix)] = filepath.Join(basePath, "Release")
}
bufWriter := bufio.NewWriter(releaseFile)
err = release.WriteTo(bufWriter)
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
err = bufWriter.Flush()
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
releaseFilename := releaseFile.Name()
releaseFile.Close()
// Signing files might output to console, so flush progress writer first
if progress != nil {
progress.Flush()
}
if signer != nil {
err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
if err != nil {
return fmt.Errorf("unable to sign Release file: %s", err)
}
err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"+suffix))
if err != nil {
return fmt.Errorf("unable to sign Release file: %s", err)
}
if suffix != "" {
renameMap[filepath.Join(basePath, "Release"+suffix+".gpg")] = filepath.Join(basePath, "Release.gpg")
renameMap[filepath.Join(basePath, "InRelease"+suffix)] = filepath.Join(basePath, "InRelease")
}
}
for oldName, newName := range renameMap {
err = publishedStorage.RenameFile(oldName, newName)
if err != nil {
return fmt.Errorf("unable to rename: %s", err)
}
}
return nil
}
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer) error {
err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
if err != nil {
return err
}
basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
err = publishedStorage.MkDir(basePath)
if err != nil {
return err
}
// Load all packages
list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection)
if err != nil {
return fmt.Errorf("unable to load packages: %s", err)
}
if list.Len() == 0 {
return fmt.Errorf("snapshot is empty")
}
if len(p.Architectures) == 0 {
p.Architectures = list.Architectures(true)
}
if len(p.Architectures) == 0 {
return fmt.Errorf("unable to figure out list of architectures, please supply explicit list")
}
sort.Strings(p.Architectures)
generatedFiles := map[string]utils.ChecksumInfo{}
// For all architectures, generate release file
for _, arch := range p.Architectures {
var relativePath string
if arch == "source" {
relativePath = filepath.Join(p.Component, "source", "Sources")
} else {
relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
}
err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
if err != nil {
return err
}
packagesFile, err := publishedStorage.CreateFile(filepath.Join(basePath, relativePath))
if err != nil {
return fmt.Errorf("unable to creates Packages file: %s", err)
}
bufWriter := bufio.NewWriter(packagesFile)
err = list.ForEach(func(pkg *Package) error {
if pkg.MatchesArchitecture(arch) {
err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
if err != nil {
return err
}
err = pkg.Stanza().WriteTo(bufWriter)
if err != nil {
return err
}
err = bufWriter.WriteByte('\n')
if err != nil {
return err
}
}
return nil
})
if err != nil {
return fmt.Errorf("unable to process packages: %s", err)
}
err = bufWriter.Flush()
if err != nil {
return fmt.Errorf("unable to write Packages file: %s", err)
}
err = utils.CompressFile(packagesFile)
if err != nil {
return fmt.Errorf("unable to compress Packages files: %s", err)
}
packagesFile.Close()
checksumInfo, err := publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath] = checksumInfo
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath+".gz"] = checksumInfo
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath+".bz2"] = checksumInfo
}
release := make(Stanza)
release["Origin"] = p.Prefix + " " + p.Distribution
release["Label"] = p.Prefix + " " + p.Distribution
release["Codename"] = p.Distribution
release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
release["Components"] = p.Component
release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
release["Description"] = " Generated by aptly\n"
release["MD5Sum"] = "\n"
release["SHA1"] = "\n"
release["SHA256"] = "\n"
for path, info := range generatedFiles {
release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
}
releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"))
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
bufWriter := bufio.NewWriter(releaseFile)
err = release.WriteTo(bufWriter)
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
err = bufWriter.Flush()
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
releaseFilename := releaseFile.Name()
releaseFile.Close()
if signer != nil {
err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
if err != nil {
return fmt.Errorf("unable to sign Release file: %s", err)
}
err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"))
if err != nil {
return fmt.Errorf("unable to sign Release file: %s", err)
}
}
return nil
}
