func removeAll(stor storage.Storage) error {
names, err := storage.List(stor, "")
if err != nil {
return err
}
// Remove all the objects in parallel so that we incur fewer round-trips.
// If we're in danger of having hundreds of objects,
// we'll want to change this to limit the number
// of concurrent operations.
var wg sync.WaitGroup
wg.Add(len(names))
errc := make(chan error, len(names))
for _, name := range names {
name := name
go func() {
defer wg.Done()
if err := stor.Remove(name); err != nil {
errc <- err
}
}()
}
wg.Wait()
select {
case err := <-errc:
return fmt.Errorf("cannot delete all provider state: %v", err)
default:
}
return nil
}
// CreateStateFile creates an empty state file on the given storage, and
// returns its URL.
func CreateStateFile(stor storage.Storage) (string, error) {
err := putState(stor, []byte{})
if err != nil {
return "", fmt.Errorf("cannot create initial state file: %v", err)
}
return stor.URL(StateFile)
}
// metadataUnchanged returns true if the content of metadata for stream in stor is the same
// as generatedMetadata, ignoring the "updated" attribute.
func metadataUnchanged(stor storage.Storage, stream string, generatedMetadata []byte) (bool, error) {
mdPath := ProductMetadataPath(stream)
filePath := path.Join(storage.BaseToolsPath, mdPath)
existingDataReader, err := stor.Get(filePath)
// If the file can't be retrieved, consider it has changed.
if err != nil {
return false, nil
}
defer existingDataReader.Close()
existingData, err := ioutil.ReadAll(existingDataReader)
if err != nil {
return false, err
}
// To do the comparison, we unmarshall the metadata, clear the
// updated value, and marshall back to a string.
existingMetadata, err := removeMetadataUpdated(existingData)
if err != nil {
return false, err
}
newMetadata, err := removeMetadataUpdated(generatedMetadata)
if err != nil {
return false, err
}
return existingMetadata == newMetadata, nil
}
// syncBuiltTools copies to storage a tools tarball and cloned copies for each series.
func syncBuiltTools(stor storage.Storage, stream string, builtTools *BuiltAgent, fakeSeries ...string) (*coretools.Tools, error) {
if err := cloneToolsForSeries(builtTools, stream, fakeSeries...); err != nil {
return nil, err
}
syncContext := &SyncContext{
Source: builtTools.Dir,
TargetToolsFinder: StorageToolsFinder{stor},
TargetToolsUploader: StorageToolsUploader{stor, false, false},
AllVersions: true,
Stream: stream,
MajorVersion: builtTools.Version.Major,
MinorVersion: -1,
}
logger.Debugf("uploading agent binaries to cloud storage")
err := SyncTools(syncContext)
if err != nil {
return nil, err
}
url, err := stor.URL(builtTools.StorageName)
if err != nil {
return nil, err
}
return &coretools.Tools{
Version: builtTools.Version,
URL: url,
Size: builtTools.Size,
SHA256: builtTools.Sha256Hash,
}, nil
}
func restoreBootstrapVerificationFile(c *gc.C, stor storage.Storage) {
content := "juju-core storage writing verified: ok\n"
contentReader := strings.NewReader(content)
err := stor.Put("bootstrap-verify", contentReader,
int64(len(content)))
c.Assert(err, gc.IsNil)
}
// SyncBuiltTools copies to storage a tools tarball and cloned copies for each series.
func SyncBuiltTools(stor storage.Storage, builtTools *BuiltTools, fakeSeries ...string) (*coretools.Tools, error) {
if err := cloneToolsForSeries(builtTools, fakeSeries...); err != nil {
return nil, err
}
syncContext := &SyncContext{
Source: builtTools.Dir,
Target: stor,
AllVersions: true,
Dev: builtTools.Version.IsDev(),
MajorVersion: builtTools.Version.Major,
MinorVersion: -1,
}
logger.Debugf("uploading tools to cloud storage")
err := SyncTools(syncContext)
if err != nil {
return nil, err
}
url, err := stor.URL(builtTools.StorageName)
if err != nil {
return nil, err
}
return &coretools.Tools{
Version: builtTools.Version,
URL: url,
Size: builtTools.Size,
SHA256: builtTools.Sha256Hash,
}, nil
}
// RemoveFakeToolsMetadata deletes the fake simplestreams tools metadata from the supplied storage.
func RemoveFakeToolsMetadata(c *gc.C, stor storage.Storage) {
files := []string{simplestreams.UnsignedIndex, envtools.ProductMetadataPath}
for _, file := range files {
toolspath := path.Join("tools", file)
err := stor.Remove(toolspath)
c.Check(err, gc.IsNil)
}
}
// RemoveFakeToolsMetadata deletes the fake simplestreams tools metadata from the supplied storage.
func RemoveFakeToolsMetadata(c *gc.C, stor storage.Storage) {
files, err := stor.List("tools/streams")
c.Assert(err, jc.ErrorIsNil)
for _, file := range files {
err = stor.Remove(file)
c.Check(err, jc.ErrorIsNil)
}
}
// RemoveTools deletes all tools from the supplied storage.
func RemoveTools(c *gc.C, stor storage.Storage, toolsDir string) {
names, err := storage.List(stor, fmt.Sprintf("tools/%s/juju-", toolsDir))
c.Assert(err, jc.ErrorIsNil)
c.Logf("removing files: %v", names)
for _, name := range names {
err = stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
// RemoveTools deletes all tools from the supplied storage.
func RemoveTools(c *gc.C, stor storage.Storage) {
names, err := storage.List(stor, "tools/releases/juju-")
c.Assert(err, gc.IsNil)
c.Logf("removing files: %v", names)
for _, name := range names {
err = stor.Remove(name)
c.Check(err, gc.IsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
// VerifyStorage writes the bootstrap init file to the storage to indicate
// that the storage is writable.
func VerifyStorage(stor storage.Storage) error {
reader := strings.NewReader(verificationContent)
err := stor.Put(VerificationFilename, reader,
int64(len(verificationContent)))
if err != nil {
logger.Warningf("failed to write bootstrap-verify file: %v", err)
return VerifyStorageError
}
return nil
}
func uploadMetadataFile(stor storage.Storage, metadataDir, fileName string, size int64) error {
fullSourceFilename := filepath.Join(metadataDir, fileName)
logger.Debugf("uploading metadata file %s", fileName)
f, err := os.Open(fullSourceFilename)
if err != nil {
return err
}
defer f.Close()
destMetadataDir := path.Join(storage.BaseImagesPath, simplestreams.StreamsDir)
return stor.Put(path.Join(destMetadataDir, fileName), f, size)
}
func UseTestImageMetadata(c *gc.C, stor storage.Storage) {
files := map[string]string{
"images/streams/v1/index.json": indexData,
"images/streams/v1/com.ubuntu.cloud:released:maas.json": imagesData,
}
for f, d := range files {
rdr := strings.NewReader(d)
err := stor.Put(f, rdr, int64(len(d)))
c.Assert(err, jc.ErrorIsNil)
}
}
func UseTestImageData(stor envstorage.Storage, cred *identity.Credentials) {
// Put some image metadata files into the public storage.
t := template.Must(template.New("").Parse(indexData))
var metadata bytes.Buffer
if err := t.Execute(&metadata, cred); err != nil {
panic(fmt.Errorf("cannot generate index metdata: %v", err))
}
data := metadata.Bytes()
stor.Put(simplestreams.UnsignedIndex("v1", 1), bytes.NewReader(data), int64(len(data)))
stor.Put(
productMetadatafile, strings.NewReader(imagesData), int64(len(imagesData)))
}
func storageCopy(source storage.Storage, sourcePath string, target storage.Storage, targetPath string) error {
rc, err := storage.Get(source, sourcePath)
if err != nil {
return err
}
var buf bytes.Buffer
_, err = io.Copy(&buf, rc)
rc.Close()
if err != nil {
return err
}
return target.Put(targetPath, &buf, int64(buf.Len()))
}
func uploadFakeToolsVersion(stor storage.Storage, toolsDir string, vers version.Binary) (*coretools.Tools, error) {
logger.Infof("uploading FAKE tools %s", vers)
tgz, checksum := makeFakeTools(vers)
size := int64(len(tgz))
name := envtools.StorageName(vers, toolsDir)
if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
return nil, err
}
url, err := stor.URL(name)
if err != nil {
return nil, err
}
return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
func SignTestTools(stor storage.Storage) error {
files, err := stor.List("")
if err != nil {
return err
}
for _, file := range files {
if strings.HasSuffix(file, sstesting.UnsignedJsonSuffix) {
// only sign .json files and data
if err := SignFileData(stor, file); err != nil {
return err
}
}
}
return nil
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage) {
c.Logf("removing fake tools")
toolsVersion := version.Current
name := envtools.StorageName(toolsVersion)
err := stor.Remove(name)
c.Check(err, gc.IsNil)
defaultSeries := coretesting.FakeDefaultSeries
if version.Current.Series != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion)
err := stor.Remove(name)
c.Check(err, gc.IsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
c.Logf("removing fake tools")
toolsVersion := version.Current
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
defaultSeries := coretesting.FakeDefaultSeries
if series.HostSeries() != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
func uploadFakeToolsVersion(stor storage.Storage, vers version.Binary) (*coretools.Tools, error) {
logger.Infof("uploading FAKE tools %s", vers)
tgz, checksum := coretesting.TarGz(
coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()))
size := int64(len(tgz))
name := envtools.StorageName(vers)
if err := stor.Put(name, bytes.NewReader(tgz), size); err != nil {
return nil, err
}
url, err := stor.URL(name)
if err != nil {
return nil, err
}
return &coretools.Tools{URL: url, Version: vers, Size: size, SHA256: checksum}, nil
}
// RemoveFakeTools deletes the fake tools from the supplied storage.
func RemoveFakeTools(c *gc.C, stor storage.Storage, toolsDir string) {
c.Logf("removing fake tools")
toolsVersion := version.Binary{
Number: jujuversion.Current,
Arch: arch.HostArch(),
Series: series.HostSeries(),
}
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
defaultSeries := series.LatestLts()
if series.HostSeries() != defaultSeries {
toolsVersion.Series = defaultSeries
name := envtools.StorageName(toolsVersion, toolsDir)
err := stor.Remove(name)
c.Check(err, jc.ErrorIsNil)
}
RemoveFakeToolsMetadata(c, stor)
}
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error {
toolsName := envtools.StorageName(tool.Version)
logger.Infof("copying %v", toolsName)
resp, err := utils.GetValidatingHTTPClient().Get(tool.URL)
if err != nil {
return err
}
buf := &bytes.Buffer{}
srcFile := resp.Body
defer srcFile.Close()
tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf))
if err != nil {
return err
}
sizeInKB := (tool.Size + 512) / 1024
logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB)
logger.Infof("download %dkB, uploading", sizeInKB)
return dest.Put(toolsName, buf, tool.Size)
}
// writeMetadata generates some basic simplestreams metadata using the specified cloud and image details and writes
// it to the supplied store.
func writeMetadata(metadata []*ImageMetadata, cloudSpec []simplestreams.CloudSpec,
metadataStore storage.Storage) error {
index, products, err := MarshalImageMetadataJSON(metadata, cloudSpec, time.Now())
if err != nil {
return err
}
metadataInfo := []MetadataFile{
{IndexStoragePath(), index},
{ProductMetadataStoragePath(), products},
}
for _, md := range metadataInfo {
err = metadataStore.Put(md.Path, bytes.NewReader(md.Data), int64(len(md.Data)))
if err != nil {
return err
}
}
return nil
}
func SignFileData(stor storage.Storage, fileName string) error {
r, err := stor.Get(fileName)
if err != nil {
return err
}
defer r.Close()
fileData, err := ioutil.ReadAll(r)
if err != nil {
return err
}
signedName, signedContent, err := sstesting.SignMetadata(fileName, fileData)
if err != nil {
return err
}
err = stor.Put(signedName, strings.NewReader(string(signedContent)), int64(len(string(signedContent))))
if err != nil {
return err
}
return nil
}
// WriteMetadata writes the given tools metadata to the given storage.
func WriteMetadata(stor storage.Storage, metadata []*ToolsMetadata, writeMirrors ShouldWriteMirrors) error {
updated := time.Now()
index, products, err := MarshalToolsMetadataJSON(metadata, updated)
if err != nil {
return err
}
metadataInfo := []MetadataFile{
{simplestreams.UnsignedIndex, index},
{ProductMetadataPath, products},
}
if writeMirrors {
mirrorsUpdated := updated.Format("20060102") // YYYYMMDD
mirrorsInfo := strings.Replace(PublicMirrorsInfo, "{{updated}}", mirrorsUpdated, -1)
metadataInfo = append(metadataInfo, MetadataFile{simplestreams.UnsignedMirror, []byte(mirrorsInfo)})
}
for _, md := range metadataInfo {
logger.Infof("Writing %s", "tools/"+md.Path)
err = stor.Put(path.Join(storage.BaseToolsPath, md.Path), bytes.NewReader(md.Data), int64(len(md.Data)))
if err != nil {
return err
}
}
return nil
}
func checkRemoveAll(c *gc.C, stor storage.Storage) {
contents := []byte("File contents.")
aFile := "a-file.txt"
err := stor.Put(aFile, bytes.NewBuffer(contents), int64(len(contents)))
c.Assert(err, gc.IsNil)
err = stor.Put("empty-file", bytes.NewBuffer(nil), 0)
c.Assert(err, gc.IsNil)
err = stor.RemoveAll()
c.Assert(err, gc.IsNil)
files, err := storage.List(stor, "")
c.Assert(err, gc.IsNil)
c.Check(files, gc.HasLen, 0)
_, err = storage.Get(stor, aFile)
c.Assert(err, gc.NotNil)
c.Check(err, gc.ErrorMatches, fmt.Sprintf("file %q not found", aFile))
}
// UploadToStorage uploads tools and metadata for the specified versions to storage.
func UploadToStorage(c *gc.C, stor storage.Storage, stream string, versions ...version.Binary) map[version.Binary]string {
uploaded := map[version.Binary]string{}
if len(versions) == 0 {
return uploaded
}
var err error
for _, vers := range versions {
filename := fmt.Sprintf("tools/%s/tools-%s.tar.gz", stream, vers.String())
// Put a file in images since the dummy storage provider requires a
// file to exist before the URL can be found. This is to ensure it behaves
// the same way as MAAS.
err = stor.Put(filename, strings.NewReader("dummy"), 5)
c.Assert(err, jc.ErrorIsNil)
uploaded[vers], err = stor.URL(filename)
c.Assert(err, jc.ErrorIsNil)
}
objects := generateMetadata(c, stream, versions...)
for _, object := range objects {
toolspath := path.Join("tools", object.path)
err = stor.Put(toolspath, bytes.NewReader(object.data), int64(len(object.data)))
c.Assert(err, jc.ErrorIsNil)
}
return uploaded
}
func RemoveTestImageData(stor envstorage.Storage) {
stor.Remove(simplestreams.UnsignedIndex("v1", 1))
stor.Remove(productMetadatafile)
}
func RemoveTestImageData(stor storage.Storage) {
stor.Remove(simplestreams.DefaultIndexPath + ".json")
stor.Remove(productMetadatafile)
}
func RemoveTestImageData(stor envstorage.Storage) {
stor.RemoveAll()
}