func (s *URLsSuite) TestToolsSources(c *gc.C) {
env := s.env(c, "config-tools-metadata-url")
sources, err := tools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-tools-metadata-url/", "https://streams.canonical.com/juju/tools/"})
}
func (s *URLsSuite) TestToolsMetadataURLsRegisteredFuncs(c *gc.C) {
tools.RegisterToolsDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id0", "betwixt/releases", utils.NoVerifySSLHostnames), nil
})
tools.RegisterToolsDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
return simplestreams.NewURLDataSource("id1", "yoink", utils.NoVerifySSLHostnames), nil
})
// overwrite the one previously registered against id1
tools.RegisterToolsDataSourceFunc("id1", func(environs.Environ) (simplestreams.DataSource, error) {
// NotSupported errors do not cause GetMetadataSources to fail,
// they just cause the datasource function to be ignored.
return nil, errors.NewNotSupported(nil, "oyvey")
})
defer tools.UnregisterToolsDataSourceFunc("id0")
defer tools.UnregisterToolsDataSourceFunc("id1")
env := s.env(c, "config-tools-metadata-url")
sources, err := tools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-tools-metadata-url/",
"betwixt/releases/",
"https://streams.canonical.com/juju/tools/",
})
}
func (s *SimpleStreamsToolsSuite) TestFindToolsFiltering(c *gc.C) {
var tw loggo.TestWriter
c.Assert(loggo.RegisterWriter("filter-tester", &tw, loggo.TRACE), gc.IsNil)
defer loggo.RemoveWriter("filter-tester")
logger := loggo.GetLogger("juju.environs")
defer logger.SetLogLevel(logger.LogLevel())
logger.SetLogLevel(loggo.TRACE)
_, err := envtools.FindTools(
s.env, 1, -1, "released", coretools.Filter{Number: version.Number{Major: 1, Minor: 2, Patch: 3}})
c.Assert(err, jc.Satisfies, errors.IsNotFound)
// This is slightly overly prescriptive, but feel free to change or add
// messages. This still helps to ensure that all log messages are
// properly formed.
messages := []jc.SimpleMessage{
{loggo.INFO, "reading tools with major version 1"},
{loggo.INFO, "filtering tools by version: \\d+\\.\\d+\\.\\d+"},
{loggo.TRACE, "no architecture specified when finding tools, looking for "},
{loggo.TRACE, "no series specified when finding tools, looking for \\[.*\\]"},
}
sources, err := envtools.GetMetadataSources(s.env)
c.Assert(err, jc.ErrorIsNil)
for i := 0; i < 2*len(sources); i++ {
messages = append(messages,
jc.SimpleMessage{loggo.TRACE, `fetchData failed for .*`},
jc.SimpleMessage{loggo.TRACE, `cannot load index .*`})
}
c.Check(tw.Log(), jc.LogMatches, messages)
}
func (s *SimpleStreamsToolsSuite) TestFindToolsFiltering(c *gc.C) {
tw := &loggo.TestWriter{}
c.Assert(loggo.RegisterWriter("filter-tester", tw, loggo.DEBUG), gc.IsNil)
defer loggo.RemoveWriter("filter-tester")
_, err := envtools.FindTools(
s.env, 1, -1, coretools.Filter{Number: version.Number{Major: 1, Minor: 2, Patch: 3}}, envtools.DoNotAllowRetry)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
// This is slightly overly prescriptive, but feel free to change or add
// messages. This still helps to ensure that all log messages are
// properly formed.
messages := []jc.SimpleMessage{
{loggo.INFO, "reading tools with major version 1"},
{loggo.INFO, "filtering tools by version: \\d+\\.\\d+\\.\\d+"},
{loggo.DEBUG, "no architecture specified when finding tools, looking for any"},
{loggo.DEBUG, "no series specified when finding tools, looking for any"},
}
sources, err := envtools.GetMetadataSources(s.env)
c.Assert(err, gc.IsNil)
for i := 0; i < 2*len(sources); i++ {
messages = append(messages,
jc.SimpleMessage{loggo.DEBUG, `fetchData failed for .*`},
jc.SimpleMessage{loggo.DEBUG, `cannot load index .*`})
}
c.Check(tw.Log, jc.LogMatches, messages)
}
func (s *environSuite) TestGetToolsMetadataSources(c *gc.C) {
testConfig := minimalConfig(c)
environ, err := local.Provider.Open(testConfig)
c.Assert(err, jc.ErrorIsNil)
sources, err := tools.GetMetadataSources(environ)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sources, gc.HasLen, 0)
}
func (s *localServerSuite) TestGetToolsMetadataSources(c *gc.C) {
s.PatchValue(&tools.DefaultBaseURL, "")
env := s.Prepare(c)
sources, err := tools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sources, gc.HasLen, 0)
}
func (s *environSuite) TestEnvironSupportsCustomSources(c *gc.C) {
sources, err := tools.GetMetadataSources(s.env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 2)
url, err := sources[0].URL("")
c.Assert(err, gc.IsNil)
c.Assert(strings.Contains(url, "/tools"), jc.IsTrue)
}
func (t *localServerSuite) TestGetToolsMetadataSources(c *gc.C) {
env := t.Prepare(c)
sources, err := tools.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 1)
url, err := sources[0].URL("")
// The control bucket URL contains the bucket name.
c.Assert(strings.Contains(url, ec2.ControlBucketName(env)+"/tools"), jc.IsTrue)
}
func (suite *maas2EnvironSuite) TestGetToolsMetadataSources(c *gc.C) {
// Add a dummy file to storage so we can use that to check the
// obtained source later.
env := suite.makeEnviron(c, newFakeControllerWithFiles(
&fakeFile{name: "agent-prefix-tools/filename", contents: makeRandomBytes(10)},
))
sources, err := envtools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sources, gc.HasLen, 0)
}
func (s *URLsSuite) TestToolsMetadataURLsRegisteredFuncsError(c *gc.C) {
tools.RegisterToolsDataSourceFunc("id0", func(environs.Environ) (simplestreams.DataSource, error) {
// Non-NotSupported errors cause GetMetadataSources to fail.
return nil, errors.New("oyvey!")
})
defer tools.UnregisterToolsDataSourceFunc("id0")
env := s.env(c, "config-tools-metadata-url")
_, err := tools.GetMetadataSources(env)
c.Assert(err, gc.ErrorMatches, "oyvey!")
}
func (s *environSuite) TestGetToolsMetadataSources(c *gc.C) {
testConfig := minimalConfig(c)
environ, err := local.Provider.Open(testConfig)
c.Assert(err, gc.IsNil)
sources, err := tools.GetMetadataSources(environ)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 1)
url, err := sources[0].URL("")
c.Assert(err, gc.IsNil)
c.Assert(strings.Contains(url, "/tools"), jc.IsTrue)
}
func (suite *environSuite) TestGetToolsMetadataSources(c *gc.C) {
env := suite.makeEnviron()
// Add a dummy file to storage so we can use that to check the
// obtained source later.
data := makeRandomBytes(10)
stor := NewStorage(env)
err := stor.Put("tools/filename", bytes.NewBuffer([]byte(data)), int64(len(data)))
c.Assert(err, jc.ErrorIsNil)
sources, err := envtools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
c.Assert(sources, gc.HasLen, 0)
}
func (s *URLsSuite) TestToolsURLsNoConfigURL(c *gc.C) {
env := s.env(c, "")
sources, err := tools.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
// Put a file in tools since the dummy storage provider requires a
// file to exist before the URL can be found. This is to ensure it behaves
// the same way as MAAS.
err = env.Storage().Put("tools/dummy", strings.NewReader("dummy"), 5)
c.Assert(err, gc.IsNil)
privateStorageURL, err := env.Storage().URL("tools")
c.Assert(err, gc.IsNil)
sstesting.AssertExpectedSources(c, sources, []string{
privateStorageURL, "https://streams.canonical.com/juju/tools/"})
}
func (s *localServerSuite) TestGetToolsMetadataSources(c *gc.C) {
env := s.Prepare(c)
sources, err := tools.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(len(sources), gc.Equals, 1)
var urls = make([]string, len(sources))
for i, source := range sources {
url, err := source.URL("")
c.Assert(err, gc.IsNil)
urls[i] = url
}
// The control bucket URL contains the bucket name.
c.Assert(strings.Contains(urls[0], joyent.ControlBucketName(env)+"/tools"), jc.IsTrue)
}
func (s *localServerSuite) TestGetToolsMetadataSources(c *gc.C) {
env := s.Open(c)
sources, err := tools.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
c.Assert(sources, gc.HasLen, 3)
var urls = make([]string, len(sources))
for i, source := range sources {
url, err := source.URL("")
c.Assert(err, gc.IsNil)
urls[i] = url
}
// The tools-metadata-url ends with "/juju-dist-test/tools/".
c.Check(strings.HasSuffix(urls[0], "/juju-dist-test/tools/"), jc.IsTrue)
// The control bucket URL contains the bucket name.
c.Check(strings.Contains(urls[1], openstack.ControlBucketName(env)+"/tools"), jc.IsTrue)
// Check that the URL from keystone parses.
_, err = url.Parse(urls[2])
c.Assert(err, gc.IsNil)
}
func (s *URLsSuite) TestToolsSources(c *gc.C) {
env := s.env(c, "config-tools-metadata-url")
sources, err := tools.GetMetadataSources(env)
c.Assert(err, gc.IsNil)
// Put a file in tools since the dummy storage provider requires a
// file to exist before the URL can be found. This is to ensure it behaves
// the same way as MAAS.
err = env.Storage().Put("tools/dummy", strings.NewReader("dummy"), 5)
c.Assert(err, gc.IsNil)
privateStorageURL, err := env.Storage().URL("tools")
c.Assert(err, gc.IsNil)
sstesting.AssertExpectedSources(c, sources, []string{
"config-tools-metadata-url/", privateStorageURL, "https://streams.canonical.com/juju/tools/"})
haveExpectedSources := false
for _, source := range sources {
if allowRetry, ok := storage.TestingGetAllowRetry(source); ok {
haveExpectedSources = true
c.Assert(allowRetry, jc.IsFalse)
}
}
c.Assert(haveExpectedSources, jc.IsTrue)
}
func (c *validateToolsMetadataCommand) Run(context *cmd.Context) error {
var params *simplestreams.MetadataLookupParams
if c.providerType == "" {
store, err := configstore.Default()
if err != nil {
return err
}
environ, err := c.prepare(context, store)
if err == nil {
mdLookup, ok := environ.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support tools metadata validation", environ.Config().Type())
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
params.Sources, err = tools.GetMetadataSources(environ)
if err != nil {
return err
}
} else {
if c.metadataDir == "" {
return err
}
params = &simplestreams.MetadataLookupParams{
Architectures: arch.AllSupportedArches,
}
}
} else {
prov, err := environs.Provider(c.providerType)
if err != nil {
return err
}
mdLookup, ok := prov.(simplestreams.MetadataValidator)
if !ok {
return fmt.Errorf("%s provider does not support tools metadata validation", c.providerType)
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
}
if c.series != "" {
params.Series = c.series
}
if c.region != "" {
params.Region = c.region
}
if c.endpoint != "" {
params.Endpoint = c.endpoint
}
if c.metadataDir != "" {
if _, err := os.Stat(c.metadataDir); err != nil {
return err
}
toolsURL, err := tools.ToolsURL(c.metadataDir)
if err != nil {
return err
}
params.Sources = []simplestreams.DataSource{simplestreams.NewURLDataSource(
"local metadata directory", toolsURL, utils.VerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false),
}
}
params.Stream = c.stream
versions, resolveInfo, err := tools.ValidateToolsMetadata(&tools.ToolsMetadataLookupParams{
MetadataLookupParams: *params,
Version: c.exactVersion,
Major: c.major,
Minor: c.minor,
})
if err != nil {
if resolveInfo != nil {
metadata := map[string]interface{}{
"Resolve Metadata": *resolveInfo,
}
if metadataYaml, yamlErr := cmd.FormatYaml(metadata); yamlErr == nil {
err = fmt.Errorf("%v\n%v", err, string(metadataYaml))
}
}
return err
}
if len(versions) > 0 {
metadata := map[string]interface{}{
"Matching Tools Versions": versions,
"Resolve Metadata": *resolveInfo,
}
c.out.Write(context, metadata)
} else {
var sources []string
for _, s := range params.Sources {
url, err := s.URL("")
if err == nil {
sources = append(sources, fmt.Sprintf("- %s (%s)", s.Description(), url))
}
}
return fmt.Errorf("no matching tools using sources:\n%s", strings.Join(sources, "\n"))
}
return nil
}
func (s *localHTTPSServerSuite) TestFetchFromToolsMetadataSources(c *gc.C) {
// Setup a custom URL for image metadata
customStorage := openstack.CreateCustomStorage(s.env, "custom-tools-metadata")
customURL, err := customStorage.URL("")
c.Assert(err, gc.IsNil)
c.Check(customURL[:8], gc.Equals, "https://")
config, err := s.env.Config().Apply(
map[string]interface{}{"tools-metadata-url": customURL},
)
c.Assert(err, gc.IsNil)
err = s.env.SetConfig(config)
c.Assert(err, gc.IsNil)
sources, err := tools.GetMetadataSources(s.env)
c.Assert(err, gc.IsNil)
c.Assert(sources, gc.HasLen, 4)
// Make sure there is something to download from each location
private := "private-tools-content"
// The Private data storage always tacks on "tools/" to the URL stream,
// so add it in here
err = s.env.Storage().Put("tools/"+private, bytes.NewBufferString(private), int64(len(private)))
c.Assert(err, gc.IsNil)
keystone := "keystone-tools-content"
// The keystone entry just points at the root of the Swift storage, and
// we have to create a container to upload any data. So we just point
// into a subdirectory for the data we are downloading
keystoneContainer := "tools-test"
keystoneStorage := openstack.CreateCustomStorage(s.env, "tools-test")
err = keystoneStorage.Put(keystone, bytes.NewBufferString(keystone), int64(len(keystone)))
c.Assert(err, gc.IsNil)
custom := "custom-tools-content"
err = customStorage.Put(custom, bytes.NewBufferString(custom), int64(len(custom)))
c.Assert(err, gc.IsNil)
// Read from the Config entry's tools-metadata-url
contentReader, url, err := sources[0].Fetch(custom)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err := ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Assert(string(content), gc.Equals, custom)
c.Check(url[:8], gc.Equals, "https://")
// Read from the private bucket
contentReader, url, err = sources[1].Fetch(private)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err = ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Check(string(content), gc.Equals, private)
//c.Check(url[:8], gc.Equals, "https://")
c.Check(strings.HasSuffix(url, "tools/"+private), jc.IsTrue)
// Check the entry we got from keystone
// Now fetch the data, and verify the contents.
contentReader, url, err = sources[2].Fetch(keystoneContainer + "/" + keystone)
c.Assert(err, gc.IsNil)
defer contentReader.Close()
content, err = ioutil.ReadAll(contentReader)
c.Assert(err, gc.IsNil)
c.Assert(string(content), gc.Equals, keystone)
c.Check(url[:8], gc.Equals, "https://")
keystoneURL, err := keystoneStorage.URL(keystone)
c.Assert(err, gc.IsNil)
c.Check(url, gc.Equals, keystoneURL)
// We *don't* test Fetch for sources[3] because it points to
// streams.canonical.com
}
func (s *URLsSuite) TestToolsURLsNoConfigURL(c *gc.C) {
env := s.env(c, "")
sources, err := tools.GetMetadataSources(env)
c.Assert(err, jc.ErrorIsNil)
sstesting.AssertExpectedSources(c, sources, []sstesting.SourceDetails{{"https://streams.canonical.com/juju/tools/", keys.JujuPublicKey}})
}
func (c *validateToolsMetadataCommand) Run(context *cmd.Context) error {
var params *simplestreams.MetadataLookupParams
if c.providerType == "" {
environ, err := c.prepare(context)
if err == nil {
mdLookup, ok := environ.(simplestreams.MetadataValidator)
if !ok {
return errors.Errorf("%s provider does not support tools metadata validation", environ.Config().Type())
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
params.Sources, err = tools.GetMetadataSources(environ)
if err != nil {
return err
}
} else {
if c.metadataDir == "" {
return err
}
params = &simplestreams.MetadataLookupParams{}
}
} else {
prov, err := environs.Provider(c.providerType)
if err != nil {
return err
}
mdLookup, ok := prov.(simplestreams.MetadataValidator)
if !ok {
return errors.Errorf("%s provider does not support tools metadata validation", c.providerType)
}
params, err = mdLookup.MetadataLookupParams(c.region)
if err != nil {
return err
}
}
if len(params.Architectures) == 0 {
params.Architectures = arch.AllSupportedArches
}
if c.series != "" {
params.Series = c.series
}
if c.region != "" {
params.Region = c.region
}
if c.endpoint != "" {
params.Endpoint = c.endpoint
}
if c.metadataDir != "" {
if _, err := os.Stat(c.metadataDir); err != nil {
return err
}
toolsURL, err := tools.ToolsURL(c.metadataDir)
if err != nil {
return err
}
params.Sources = toolsDataSources(toolsURL)
}
params.Stream = c.stream
versions, resolveInfo, err := tools.ValidateToolsMetadata(&tools.ToolsMetadataLookupParams{
MetadataLookupParams: *params,
Version: c.exactVersion,
Major: c.major,
Minor: c.minor,
})
if err != nil {
if resolveInfo != nil {
metadata := map[string]interface{}{
"Resolve Metadata": *resolveInfo,
}
buff := &bytes.Buffer{}
if yamlErr := cmd.FormatYaml(buff, metadata); yamlErr == nil {
err = errors.Errorf("%v\n%v", err, buff.String())
}
}
return err
}
if len(versions) > 0 {
metadata := map[string]interface{}{
"Matching Tools Versions": versions,
"Resolve Metadata": *resolveInfo,
}
c.out.Write(context, metadata)
} else {
var sources []string
for _, s := range params.Sources {
url, err := s.URL("")
if err == nil {
sources = append(sources, fmt.Sprintf("- %s (%s)", s.Description(), url))
}
}
return errors.Errorf("no matching tools using sources:\n%s", strings.Join(sources, "\n"))
}
return nil
}