func createImageMetadata(c *gc.C) (sourceDir string, destDir string, destStor storage.Storage, metadata *imagemetadata.ImageMetadata) {
destDir = c.MkDir()
var err error
destStor, err = filestorage.NewFileStorageWriter(destDir)
c.Assert(err, gc.IsNil)
// Generate some metadata.
sourceDir = c.MkDir()
im := []*imagemetadata.ImageMetadata{
{
Id: "1234",
Arch: "amd64",
Version: "13.04",
},
}
cloudSpec := &simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
}
im[0].RegionName = cloudSpec.Region
im[0].Endpoint = cloudSpec.Endpoint
var sourceStor storage.Storage
sourceStor, err = filestorage.NewFileStorageWriter(sourceDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor)
c.Assert(err, gc.IsNil)
return sourceDir, destDir, destStor, im[0]
}
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
metadataDir, metadata := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 3)
c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
// This data source does not require to contain signed data.
// However, it may still contain it.
// Since we will always try to read signed data first,
// we want to be able to try to read this signed data
// with a user provided key.
// for this test, user provided key is empty.
// Bugs #1542127, #1542131
c.Assert(datasources[0].PublicSigningKey(), gc.Equals, "")
c.Assert(env.instanceConfig, gc.NotNil)
c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
// TestBootstrapImageMetadataFromAllSources tests that we are looking for
// image metadata in all data sources available to environment.
// Abandoning look up too soon led to misleading bootstrap failures:
// Juju reported no images available for a particular configuration,
// despite image metadata in other data sources compatible with the same configuration as well.
// Related to bug#1560625.
func (s *bootstrapSuite) TestBootstrapImageMetadataFromAllSources(c *gc.C) {
s.PatchValue(&series.HostSeries, func() string { return "raring" })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
// Ensure that we can find at least one image metadata
// early on in the image metadata lookup.
// We should continue looking despite it.
metadataDir, _ := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := bootstrapEnvironWithRegion{
newEnviron("foo", useDefaultKeys, nil),
simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
},
}
s.setDummyStorage(c, env.bootstrapEnviron)
bootstrapCons := constraints.MustParse("arch=amd64")
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
BootstrapConstraints: bootstrapCons,
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
for _, source := range datasources {
// make sure we looked in each and all...
c.Assert(c.GetTestLog(), jc.Contains, fmt.Sprintf("image metadata in %s", source.Description()))
}
}
func (s *uploadSuite) testStorageToolsUploaderWriteMirrors(c *gc.C, writeMirrors envtools.ShouldWriteMirrors) {
storageDir := c.MkDir()
stor, err := filestorage.NewFileStorageWriter(storageDir)
c.Assert(err, jc.ErrorIsNil)
uploader := &sync.StorageToolsUploader{
Storage: stor,
WriteMetadata: true,
WriteMirrors: writeMirrors,
}
err = uploader.UploadTools(
"released",
"released",
&coretools.Tools{
Version: version.Current,
Size: 7,
SHA256: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
}, []byte("content"))
c.Assert(err, jc.ErrorIsNil)
mirrorsPath := simplestreams.MirrorsPath(envtools.StreamsVersionV1) + simplestreams.UnsignedSuffix
r, err := stor.Get(path.Join(storage.BaseToolsPath, mirrorsPath))
if writeMirrors == envtools.WriteMirrors {
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
r.Close()
c.Assert(err, jc.ErrorIsNil)
c.Assert(string(data), jc.Contains, `"mirrors":`)
} else {
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
}
func (s *generateSuite) TestWriteIndexRegionOnce(c *gc.C) {
existingImageMetadata := []*imagemetadata.ImageMetadata{
{
Id: "1234",
Arch: "amd64",
Version: "13.04",
VirtType: "kvm",
},
}
cloudSpec := &simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
}
dir := c.MkDir()
targetStorage, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, jc.ErrorIsNil)
err = imagemetadata.MergeAndWriteMetadata("raring", existingImageMetadata, cloudSpec, targetStorage)
c.Assert(err, jc.ErrorIsNil)
newImageMetadata := []*imagemetadata.ImageMetadata{
{
Id: "abcd",
Arch: "amd64",
Version: "13.04",
VirtType: "lxd",
},
}
err = imagemetadata.MergeAndWriteMetadata("raring", newImageMetadata, cloudSpec, targetStorage)
c.Assert(err, jc.ErrorIsNil)
foundIndex, _ := testing.ParseIndexMetadataFromStorage(c, targetStorage)
expectedCloudSpecs := []simplestreams.CloudSpec{*cloudSpec}
c.Assert(foundIndex.Clouds, jc.SameContents, expectedCloudSpecs)
}
func (c *SyncToolsCommand) Run(ctx *cmd.Context) (resultErr error) {
// Register writer for output on screen.
loggo.RegisterWriter("synctools", cmd.NewCommandLogWriter("juju.environs.sync", ctx.Stdout, ctx.Stderr), loggo.INFO)
defer loggo.RemoveWriter("synctools")
environ, cleanup, err := environFromName(ctx, c.EnvName, &resultErr, "Sync-tools")
if err != nil {
return err
}
defer cleanup()
target := environ.Storage()
if c.localDir != "" {
target, err = filestorage.NewFileStorageWriter(c.localDir)
if err != nil {
return err
}
}
// Prepare syncing.
sctx := &sync.SyncContext{
Target: target,
AllVersions: c.allVersions,
MajorVersion: c.majorVersion,
MinorVersion: c.minorVersion,
DryRun: c.dryRun,
Dev: c.dev,
Public: c.public,
Source: c.source,
}
return syncTools(sctx)
}
func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) {
metadata := []*tools.ToolsMetadata{{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Path: "path1",
}, {
Release: "raring",
Version: "1.2.3",
Arch: "amd64",
Path: "path2",
}}
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, gc.IsNil)
out, err := tools.ReadMetadata(stor)
c.Assert(out, gc.HasLen, 0)
c.Assert(err, gc.IsNil) // non-existence is not an error
err = tools.WriteMetadata(stor, metadata, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
out, err = tools.ReadMetadata(stor)
for _, md := range out {
// FullPath is set by ReadMetadata.
c.Assert(md.FullPath, gc.Not(gc.Equals), "")
md.FullPath = ""
}
c.Assert(out, gc.DeepEquals, metadata)
}
func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) {
dir := c.MkDir()
existingToolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abc",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
newToolsList := coretools.List{
existingToolsList[0],
{
Version: version.MustParseBinary("2.1.0-raring-amd64"),
Size: 789,
SHA256: "def",
},
}
err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
requiredToolsList := append(existingToolsList, newToolsList[1])
metadata := toolstesting.ParseMetadataFromDir(c, dir, false)
assertMetadataMatches(c, dir, requiredToolsList, metadata)
}
func (cs *NewAPIStateSuite) TestNewAPIState(c *gc.C) {
cfg, err := config.New(config.NoDefaults, dummy.SampleConfig())
c.Assert(err, jc.ErrorIsNil)
ctx := envtesting.BootstrapContext(c)
env, err := environs.Prepare(cfg, ctx, configstore.NewMem())
c.Assert(err, jc.ErrorIsNil)
storageDir := c.MkDir()
cs.PatchValue(&envtools.DefaultBaseURL, storageDir)
stor, err := filestorage.NewFileStorageWriter(storageDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
c.Assert(err, jc.ErrorIsNil)
cfg = env.Config()
cfg, err = cfg.Apply(map[string]interface{}{
"secret": "fnord",
})
c.Assert(err, jc.ErrorIsNil)
err = env.SetConfig(cfg)
c.Assert(err, jc.ErrorIsNil)
st, err := juju.NewAPIState(dummy.AdminUserTag(), env, api.DialOpts{})
c.Assert(st, gc.NotNil)
// the secrets will not be updated, as they already exist
attrs, err := st.Client().EnvironmentGet()
c.Assert(attrs["secret"], gc.Equals, "pork")
c.Assert(st.Close(), gc.IsNil)
}
func (s *NewAPIClientSuite) bootstrapModel(c *gc.C) (environs.Environ, jujuclient.ClientStore) {
const controllerName = "local.my-controller"
store := jujuclienttesting.NewMemStore()
ctx := envtesting.BootstrapContext(c)
env, err := environs.Prepare(ctx, store, environs.PrepareParams{
ControllerName: controllerName,
BaseConfig: dummy.SampleConfig(),
CloudName: "dummy",
})
c.Assert(err, jc.ErrorIsNil)
storageDir := c.MkDir()
s.PatchValue(&envtools.DefaultBaseURL, storageDir)
stor, err := filestorage.NewFileStorageWriter(storageDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
c.Assert(err, jc.ErrorIsNil)
return env, store
}
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
metadataDir, metadata := createImageMetadata(c)
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
MetadataDir: metadataDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 3)
c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
c.Assert(env.instanceConfig, gc.NotNil)
c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
func (s *OpenSuite) TestNewDummyEnviron(c *gc.C) {
s.PatchValue(&jujuversion.Current, testing.FakeVersionNumber)
// matches *Settings.Map()
cfg, err := config.New(config.NoDefaults, dummySampleConfig())
c.Assert(err, jc.ErrorIsNil)
ctx := envtesting.BootstrapContext(c)
cache := jujuclienttesting.NewMemStore()
controllerCfg := testing.FakeControllerConfig()
env, err := bootstrap.Prepare(ctx, cache, bootstrap.PrepareParams{
ControllerConfig: controllerCfg,
ControllerName: cfg.Name(),
ModelConfig: cfg.AllAttrs(),
Cloud: dummy.SampleCloudSpec(),
AdminSecret: "admin-secret",
})
c.Assert(err, jc.ErrorIsNil)
storageDir := c.MkDir()
s.PatchValue(&envtools.DefaultBaseURL, storageDir)
stor, err := filestorage.NewFileStorageWriter(storageDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, cfg.AgentStream(), cfg.AgentStream())
err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{
ControllerConfig: controllerCfg,
AdminSecret: "admin-secret",
CAPrivateKey: testing.CAKey,
})
c.Assert(err, jc.ErrorIsNil)
// New controller should have been added to collection.
foundController, err := cache.ControllerByName(cfg.Name())
c.Assert(err, jc.ErrorIsNil)
c.Assert(foundController.ControllerUUID, gc.DeepEquals, controllerCfg.ControllerUUID())
}
// PrimeAgentVersion writes the configuration file and tools with version
// vers for an agent with the given entity name. It returns the agent's
// configuration and the current tools.
func (s *AgentSuite) PrimeAgentVersion(c *gc.C, tag names.Tag, password string, vers version.Binary) (agent.ConfigSetterWriter, *coretools.Tools) {
c.Logf("priming agent %s", tag.String())
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), "released", vers)
err = envtools.MergeAndWriteMetadata(stor, "released", "released", coretools.List{agentTools}, envtools.DoNotWriteMirrors)
tools1, err := agenttools.ChangeAgentTools(s.DataDir(), tag.String(), vers)
c.Assert(err, jc.ErrorIsNil)
c.Assert(tools1, gc.DeepEquals, agentTools)
stateInfo := s.MongoInfo(c)
apiInfo := s.APIInfo(c)
paths := agent.DefaultPaths
paths.DataDir = s.DataDir()
conf, err := agent.NewAgentConfig(
agent.AgentConfigParams{
Paths: paths,
Tag: tag,
UpgradedToVersion: vers.Number,
Password: password,
Nonce: agent.BootstrapNonce,
StateAddresses: stateInfo.Addrs,
APIAddresses: apiInfo.Addrs,
CACert: stateInfo.CACert,
Model: apiInfo.ModelTag,
})
c.Assert(err, jc.ErrorIsNil)
conf.SetPassword(password)
c.Assert(conf.Write(), gc.IsNil)
s.primeAPIHostPorts(c)
return conf, agentTools
}
func makeTools(c *gc.C, metadataDir, stream string, versionStrings []string, withCheckSum bool) coretools.List {
toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath, stream)
c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
var toolsList coretools.List
for _, versionString := range versionStrings {
binary, err := version.ParseBinary(versionString)
if err != nil {
c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
}
path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
data := binary.String()
err = ioutil.WriteFile(path, []byte(data), 0644)
c.Assert(err, jc.ErrorIsNil)
tool := &coretools.Tools{
Version: binary,
URL: path,
}
if withCheckSum {
tool.Size, tool.SHA256 = SHA256sum(c, path)
}
toolsList = append(toolsList, tool)
}
// Write the tools metadata.
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, jc.ErrorIsNil)
err = tools.MergeAndWriteMetadata(stor, stream, stream, toolsList, false)
c.Assert(err, jc.ErrorIsNil)
return toolsList
}
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
toolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
expected := toolsList
// Add tools with an unknown series. Do not add an entry in the
// expected list as these tools should be ignored.
vers, err := version.ParseBinary("3.2.1-xuanhuaceratops-amd64")
c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
toolsList = append(toolsList, &coretools.Tools{
Version: vers,
Size: 456,
SHA256: "wqe",
})
dir := c.MkDir()
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, jc.ErrorIsNil)
err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", false)
assertMetadataMatches(c, dir, "proposed", expected, metadata)
}
func (s *UpgradeJujuSuite) setUpEnvAndTools(c *gc.C, currentVersion string, agentVersion string, tools []string) {
current := version.MustParseBinary(currentVersion)
s.PatchValue(&jujuversion.Current, current.Number)
s.PatchValue(&arch.HostArch, func() string { return current.Arch })
s.PatchValue(&series.HostSeries, func() string { return current.Series })
toolsDir := c.MkDir()
updateAttrs := map[string]interface{}{
"agent-version": agentVersion,
"agent-metadata-url": "file://" + toolsDir + "/tools",
}
err := s.State.UpdateModelConfig(updateAttrs, nil, nil)
c.Assert(err, jc.ErrorIsNil)
versions := make([]version.Binary, len(tools))
for i, v := range tools {
versions[i], err = version.ParseBinary(v)
if err != nil {
c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
}
}
if len(versions) > 0 {
stor, err := filestorage.NewFileStorageWriter(toolsDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.MustUploadFakeToolsVersions(stor, s.Environ.Config().AgentStream(), versions...)
}
}
func (c *ImageMetadataCommand) Run(context *cmd.Context) error {
if err := c.setParams(context); err != nil {
return err
}
out := context.Stdout
im := &imagemetadata.ImageMetadata{
Id: c.ImageId,
Arch: c.Arch,
}
cloudSpec := simplestreams.CloudSpec{
Region: c.Region,
Endpoint: c.Endpoint,
}
targetStorage, err := filestorage.NewFileStorageWriter(c.Dir)
if err != nil {
return err
}
err = imagemetadata.MergeAndWriteMetadata(c.Series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage)
if err != nil {
return fmt.Errorf("image metadata files could not be created: %v", err)
}
dir := context.AbsPath(c.Dir)
dest := filepath.Join(dir, storage.BaseImagesPath, "streams", "v1")
fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, dir, dir))
return nil
}
func (*metadataHelperSuite) writeMetadataMultipleStream(c *gc.C) (storage.StorageReader, map[string][]*tools.ToolsMetadata) {
metadata := map[string][]*tools.ToolsMetadata{
"released": {{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Path: "path1",
}},
"proposed": {{
Release: "raring",
Version: "1.2.3",
Arch: "amd64",
Path: "path2",
}},
}
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
out, err := tools.ReadAllMetadata(stor)
c.Assert(out, gc.HasLen, 0)
c.Assert(err, jc.ErrorIsNil) // non-existence is not an error
err = tools.WriteMetadata(stor, metadata, []string{"released", "proposed"}, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
return stor, metadata
}
func (s *metadataHelperSuite) TestReadWriteMetadataUnchanged(c *gc.C) {
metadata := map[string][]*tools.ToolsMetadata{
"released": {{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Path: "path1",
}, {
Release: "raring",
Version: "1.2.3",
Arch: "amd64",
Path: "path2",
}},
}
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
s.PatchValue(tools.WriteMetadataFiles, func(stor storage.Storage, metadataInfo []tools.MetadataFile) error {
// The product data is the same, we only write the indices.
c.Assert(metadataInfo, gc.HasLen, 2)
c.Assert(metadataInfo[0].Path, gc.Equals, "streams/v1/index2.json")
c.Assert(metadataInfo[1].Path, gc.Equals, "streams/v1/index.json")
return nil
})
err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
}
func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) {
var versionStrings = []string{
"1.2.3-precise-amd64",
"2.0.1-raring-amd64",
}
dir := c.MkDir()
toolstesting.MakeTools(c, dir, "proposed", versionStrings)
toolsList := coretools.List{
{
// If sha256/size is already known, do not recalculate
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
// The URL is not used for generating metadata.
URL: "bogus://",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, jc.ErrorIsNil)
writeMirrors := tools.DoNotWriteMirrors
if withMirrors {
writeMirrors = tools.WriteMirrors
}
err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, writeMirrors)
c.Assert(err, jc.ErrorIsNil)
metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", withMirrors)
assertMetadataMatches(c, dir, "proposed", toolsList, metadata)
// No release stream generated so there will not be a legacy index file created.
_, err = writer.Get("tools/streams/v1/index.json")
c.Assert(err, gc.NotNil)
}
func (*metadataHelperSuite) TestReadWriteMetadataSingleStream(c *gc.C) {
metadata := map[string][]*tools.ToolsMetadata{
"released": {{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Path: "path1",
}, {
Release: "raring",
Version: "1.2.3",
Arch: "amd64",
Path: "path2",
}},
}
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, jc.ErrorIsNil)
out, err := tools.ReadAllMetadata(stor)
c.Assert(err, jc.ErrorIsNil) // non-existence is not an error
c.Assert(out, gc.HasLen, 0)
err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
c.Assert(err, jc.ErrorIsNil)
// Read back what was just written.
out, err = tools.ReadAllMetadata(stor)
c.Assert(err, jc.ErrorIsNil)
for _, outMetadata := range out {
for _, md := range outMetadata {
// FullPath is set by ReadAllMetadata.
c.Assert(md.FullPath, gc.Not(gc.Equals), "")
md.FullPath = ""
}
}
c.Assert(out, jc.DeepEquals, metadata)
}
func (s *NewAPIClientSuite) bootstrapEnv(c *gc.C, envName string, store configstore.Storage) {
if store == nil {
store = configstore.NewMem()
}
ctx := envtesting.BootstrapContext(c)
c.Logf("env name: %s", envName)
env, err := environs.PrepareFromName(envName, ctx, store)
c.Assert(err, jc.ErrorIsNil)
storageDir := c.MkDir()
s.PatchValue(&envtools.DefaultBaseURL, storageDir)
stor, err := filestorage.NewFileStorageWriter(storageDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
c.Assert(err, jc.ErrorIsNil)
info, err := store.ReadInfo(envName)
c.Assert(err, jc.ErrorIsNil)
creds := info.APICredentials()
creds.User = dummy.AdminUserTag().Name()
c.Logf("set creds: %#v", creds)
info.SetAPICredentials(creds)
err = info.Write()
c.Assert(err, jc.ErrorIsNil)
c.Logf("creds: %#v", info.APICredentials())
info, err = store.ReadInfo(envName)
c.Assert(err, jc.ErrorIsNil)
c.Logf("read creds: %#v", info.APICredentials())
c.Logf("store: %#v", store)
}
func (s *uploadSuite) TestUploadErrors(c *gc.C) {
destDir := c.MkDir()
destStor, err := filestorage.NewFileStorageWriter(destDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.UploadImageMetadata(destStor, "foo")
c.Assert(err, jc.Satisfies, os.IsNotExist)
}
func (s *syncSuite) setUpTest(c *gc.C) {
if runtime.GOOS == "windows" {
c.Skip("issue 1403084: Currently does not work because of jujud problems")
}
s.FakeJujuXDGDataHomeSuite.SetUpTest(c)
s.ToolsFixture.SetUpTest(c)
// It's important that this be v1.8.x to match the test data.
s.PatchValue(&jujuversion.Current, version.MustParse("1.8.3"))
// Create a source storage.
baseDir := c.MkDir()
stor, err := filestorage.NewFileStorageWriter(baseDir)
c.Assert(err, jc.ErrorIsNil)
s.storage = stor
// Create a local tools directory.
s.localStorage = c.MkDir()
// Populate both local and default tools locations with the public tools.
versionStrings := make([]string, len(vAll))
for i, vers := range vAll {
versionStrings[i] = vers.String()
}
toolstesting.MakeTools(c, baseDir, "released", versionStrings)
toolstesting.MakeTools(c, s.localStorage, "released", versionStrings)
// Switch the default tools location.
baseURL, err := s.storage.URL(storage.BaseToolsPath)
c.Assert(err, jc.ErrorIsNil)
s.PatchValue(&envtools.DefaultBaseURL, baseURL)
}
func (s *bootstrapSuite) TestBootstrapMetadataImagesMissing(c *gc.C) {
environs.UnregisterImageDataSourceFunc("bootstrap metadata")
noImagesDir := c.MkDir()
stor, err := filestorage.NewFileStorageWriter(noImagesDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.UploadFakeTools(c, stor, "released", "released")
env := newEnviron("foo", useDefaultKeys, nil)
s.setDummyStorage(c, env)
err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
ControllerConfig: coretesting.FakeControllerConfig(),
AdminSecret: "admin-secret",
CAPrivateKey: coretesting.CAKey,
MetadataDir: noImagesDir,
})
c.Assert(err, jc.ErrorIsNil)
c.Assert(env.bootstrapCount, gc.Equals, 1)
datasources, err := environs.ImageMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(datasources, gc.HasLen, 2)
c.Assert(datasources[0].Description(), gc.Equals, "default cloud images")
c.Assert(datasources[1].Description(), gc.Equals, "default ubuntu cloud images")
}
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
for i, test := range upgradeJujuTests {
c.Logf("\ntest %d: %s", i, test.about)
s.Reset(c)
tools.DefaultBaseURL = ""
// Set up apparent CLI version and initialize the command.
s.PatchValue(&version.Current, version.MustParseBinary(test.currentVersion))
com := &UpgradeJujuCommand{}
if err := coretesting.InitCommand(envcmd.Wrap(com), test.args); err != nil {
if test.expectInitErr != "" {
c.Check(err, gc.ErrorMatches, test.expectInitErr)
} else {
c.Check(err, jc.ErrorIsNil)
}
continue
}
// Set up state and environ, and run the command.
toolsDir := c.MkDir()
updateAttrs := map[string]interface{}{
"agent-version": test.agentVersion,
"agent-metadata-url": "file://" + toolsDir + "/tools",
}
err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
c.Assert(err, jc.ErrorIsNil)
versions := make([]version.Binary, len(test.tools))
for i, v := range test.tools {
versions[i] = version.MustParseBinary(v)
}
if len(versions) > 0 {
stor, err := filestorage.NewFileStorageWriter(toolsDir)
c.Assert(err, jc.ErrorIsNil)
envtesting.MustUploadFakeToolsVersions(stor, s.Environ.Config().AgentStream(), versions...)
}
err = com.Run(coretesting.Context(c))
if test.expectErr != "" {
c.Check(err, gc.ErrorMatches, test.expectErr)
continue
} else if !c.Check(err, jc.ErrorIsNil) {
continue
}
// Check expected changes to environ/state.
cfg, err := s.State.EnvironConfig()
c.Check(err, jc.ErrorIsNil)
agentVersion, ok := cfg.AgentVersion()
c.Check(ok, jc.IsTrue)
c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))
for _, uploaded := range test.expectUploaded {
// Substitute latest LTS for placeholder in expected series for uploaded tools
uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)
vers := version.MustParseBinary(uploaded)
s.checkToolsUploaded(c, vers, agentVersion)
}
}
}
// startServer starts a new local storage server
// using a temporary directory and returns the listener,
// a base URL for the server and the directory path.
func startServer(c *gc.C) (listener net.Listener, url, dataDir string) {
dataDir = c.MkDir()
embedded, err := filestorage.NewFileStorageWriter(dataDir)
c.Assert(err, gc.IsNil)
listener, err = httpstorage.Serve("localhost:0", embedded)
c.Assert(err, gc.IsNil)
return listener, fmt.Sprintf("http://%s/", listener.Addr()), dataDir
}
func (s *filestorageSuite) SetUpTest(c *gc.C) {
s.dir = c.MkDir()
var err error
s.reader, err = filestorage.NewFileStorageReader(s.dir)
c.Assert(err, jc.ErrorIsNil)
s.writer, err = filestorage.NewFileStorageWriter(s.dir)
c.Assert(err, jc.ErrorIsNil)
}
// setLocalStorage creates a filestorage so tools can
// be synced and so forth without having a machine agent
// running.
func (env *localEnviron) setLocalStorage() error {
storage, err := filestorage.NewFileStorageWriter(env.config.storageDir())
if err != nil {
return err
}
env.localStorage = storage
return nil
}
// populateTools stores uploaded tools in provider storage
// and updates the tools metadata.
//
// TODO(axw) store tools in gridfs, catalogue in state.
func (c *BootstrapCommand) populateTools(env environs.Environ) error {
agentConfig := c.CurrentConfig()
dataDir := agentConfig.DataDir()
tools, err := agenttools.ReadTools(dataDir, version.Current)
if err != nil {
return err
}
if !strings.HasPrefix(tools.URL, "file://") {
// Nothing to do since the tools were not uploaded.
return nil
}
// This is a hack: providers using localstorage (local, manual)
// can't use storage during bootstrap as the localstorage worker
// isn't running. Use filestorage instead.
var stor storage.Storage
storageDir := agentConfig.Value(agent.StorageDir)
if storageDir != "" {
stor, err = filestorage.NewFileStorageWriter(storageDir)
if err != nil {
return err
}
} else {
stor = env.Storage()
}
// Create a temporary directory to contain source and cloned tools.
tempDir, err := ioutil.TempDir("", "juju-sync-tools")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
destTools := filepath.Join(tempDir, filepath.FromSlash(envtools.StorageName(tools.Version)))
if err := os.MkdirAll(filepath.Dir(destTools), 0700); err != nil {
return err
}
srcTools := filepath.Join(
agenttools.SharedToolsDir(dataDir, version.Current),
"tools.tar.gz",
)
if err := utils.CopyFile(destTools, srcTools); err != nil {
return err
}
// Until we catalogue tools in state, we clone the tools
// for each of the supported series of the same OS.
otherSeries := version.OSSupportedSeries(version.Current.OS)
_, err = sync.SyncBuiltTools(stor, &sync.BuiltTools{
Version: tools.Version,
Dir: tempDir,
StorageName: envtools.StorageName(tools.Version),
Sha256Hash: tools.SHA256,
Size: tools.Size,
}, otherSeries...)
return err
}