Example #1
0
func createImageMetadata(c *gc.C) (sourceDir string, destDir string, destStor storage.Storage, metadata *imagemetadata.ImageMetadata) {
	destDir = c.MkDir()
	var err error
	destStor, err = filestorage.NewFileStorageWriter(destDir)
	c.Assert(err, gc.IsNil)

	// Generate some metadata.
	sourceDir = c.MkDir()
	im := []*imagemetadata.ImageMetadata{
		{
			Id:      "1234",
			Arch:    "amd64",
			Version: "13.04",
		},
	}
	cloudSpec := &simplestreams.CloudSpec{
		Region:   "region",
		Endpoint: "endpoint",
	}
	im[0].RegionName = cloudSpec.Region
	im[0].Endpoint = cloudSpec.Endpoint
	var sourceStor storage.Storage
	sourceStor, err = filestorage.NewFileStorageWriter(sourceDir)
	c.Assert(err, gc.IsNil)
	err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor)
	c.Assert(err, gc.IsNil)
	return sourceDir, destDir, destStor, im[0]
}
Example #2
0
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	metadataDir, metadata := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		MetadataDir: metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)
	c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 3)
	c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
	// This data source does not require to contain signed data.
	// However, it may still contain it.
	// Since we will always try to read signed data first,
	// we want to be able to try to read this signed data
	// with a user provided key.
	// for this test, user provided key is empty.
	// Bugs #1542127, #1542131
	c.Assert(datasources[0].PublicSigningKey(), gc.Equals, "")
	c.Assert(env.instanceConfig, gc.NotNil)
	c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
	c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
Example #3
0
// TestBootstrapImageMetadataFromAllSources tests that we are looking for
// image metadata in all data sources available to environment.
// Abandoning look up too soon led to misleading bootstrap failures:
// Juju reported no images available for a particular configuration,
// despite image metadata in other data sources compatible with the same configuration as well.
// Related to bug#1560625.
func (s *bootstrapSuite) TestBootstrapImageMetadataFromAllSources(c *gc.C) {
	s.PatchValue(&series.HostSeries, func() string { return "raring" })
	s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })

	// Ensure that we can find at least one image metadata
	// early on in the image metadata lookup.
	// We should continue looking despite it.
	metadataDir, _ := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := bootstrapEnvironWithRegion{
		newEnviron("foo", useDefaultKeys, nil),
		simplestreams.CloudSpec{
			Region:   "region",
			Endpoint: "endpoint",
		},
	}
	s.setDummyStorage(c, env.bootstrapEnviron)

	bootstrapCons := constraints.MustParse("arch=amd64")
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		BootstrapConstraints: bootstrapCons,
		MetadataDir:          metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	for _, source := range datasources {
		// make sure we looked in each and all...
		c.Assert(c.GetTestLog(), jc.Contains, fmt.Sprintf("image metadata in %s", source.Description()))
	}
}
Example #4
0
func (s *uploadSuite) testStorageToolsUploaderWriteMirrors(c *gc.C, writeMirrors envtools.ShouldWriteMirrors) {
	storageDir := c.MkDir()
	stor, err := filestorage.NewFileStorageWriter(storageDir)
	c.Assert(err, jc.ErrorIsNil)

	uploader := &sync.StorageToolsUploader{
		Storage:       stor,
		WriteMetadata: true,
		WriteMirrors:  writeMirrors,
	}

	err = uploader.UploadTools(
		"released",
		"released",
		&coretools.Tools{
			Version: version.Current,
			Size:    7,
			SHA256:  "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
		}, []byte("content"))
	c.Assert(err, jc.ErrorIsNil)

	mirrorsPath := simplestreams.MirrorsPath(envtools.StreamsVersionV1) + simplestreams.UnsignedSuffix
	r, err := stor.Get(path.Join(storage.BaseToolsPath, mirrorsPath))
	if writeMirrors == envtools.WriteMirrors {
		c.Assert(err, jc.ErrorIsNil)
		data, err := ioutil.ReadAll(r)
		r.Close()
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(string(data), jc.Contains, `"mirrors":`)
	} else {
		c.Assert(err, jc.Satisfies, errors.IsNotFound)
	}
}
Example #5
0
func (s *generateSuite) TestWriteIndexRegionOnce(c *gc.C) {
	existingImageMetadata := []*imagemetadata.ImageMetadata{
		{
			Id:       "1234",
			Arch:     "amd64",
			Version:  "13.04",
			VirtType: "kvm",
		},
	}
	cloudSpec := &simplestreams.CloudSpec{
		Region:   "region",
		Endpoint: "endpoint",
	}
	dir := c.MkDir()
	targetStorage, err := filestorage.NewFileStorageWriter(dir)
	c.Assert(err, jc.ErrorIsNil)
	err = imagemetadata.MergeAndWriteMetadata("raring", existingImageMetadata, cloudSpec, targetStorage)
	c.Assert(err, jc.ErrorIsNil)
	newImageMetadata := []*imagemetadata.ImageMetadata{
		{
			Id:       "abcd",
			Arch:     "amd64",
			Version:  "13.04",
			VirtType: "lxd",
		},
	}
	err = imagemetadata.MergeAndWriteMetadata("raring", newImageMetadata, cloudSpec, targetStorage)
	c.Assert(err, jc.ErrorIsNil)

	foundIndex, _ := testing.ParseIndexMetadataFromStorage(c, targetStorage)
	expectedCloudSpecs := []simplestreams.CloudSpec{*cloudSpec}
	c.Assert(foundIndex.Clouds, jc.SameContents, expectedCloudSpecs)
}
Example #6
0
func (c *SyncToolsCommand) Run(ctx *cmd.Context) (resultErr error) {
	// Register writer for output on screen.
	loggo.RegisterWriter("synctools", cmd.NewCommandLogWriter("juju.environs.sync", ctx.Stdout, ctx.Stderr), loggo.INFO)
	defer loggo.RemoveWriter("synctools")
	environ, cleanup, err := environFromName(ctx, c.EnvName, &resultErr, "Sync-tools")
	if err != nil {
		return err
	}
	defer cleanup()
	target := environ.Storage()
	if c.localDir != "" {
		target, err = filestorage.NewFileStorageWriter(c.localDir)
		if err != nil {
			return err
		}
	}

	// Prepare syncing.
	sctx := &sync.SyncContext{
		Target:       target,
		AllVersions:  c.allVersions,
		MajorVersion: c.majorVersion,
		MinorVersion: c.minorVersion,
		DryRun:       c.dryRun,
		Dev:          c.dev,
		Public:       c.public,
		Source:       c.source,
	}
	return syncTools(sctx)
}
Example #7
0
func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) {
	metadata := []*tools.ToolsMetadata{{
		Release: "precise",
		Version: "1.2.3",
		Arch:    "amd64",
		Path:    "path1",
	}, {
		Release: "raring",
		Version: "1.2.3",
		Arch:    "amd64",
		Path:    "path2",
	}}

	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, gc.IsNil)
	out, err := tools.ReadMetadata(stor)
	c.Assert(out, gc.HasLen, 0)
	c.Assert(err, gc.IsNil) // non-existence is not an error
	err = tools.WriteMetadata(stor, metadata, tools.DoNotWriteMirrors)
	c.Assert(err, gc.IsNil)
	out, err = tools.ReadMetadata(stor)
	for _, md := range out {
		// FullPath is set by ReadMetadata.
		c.Assert(md.FullPath, gc.Not(gc.Equals), "")
		md.FullPath = ""
	}
	c.Assert(out, gc.DeepEquals, metadata)
}
Example #8
0
func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) {
	dir := c.MkDir()
	existingToolsList := coretools.List{
		{
			Version: version.MustParseBinary("1.2.3-precise-amd64"),
			Size:    123,
			SHA256:  "abc",
		}, {
			Version: version.MustParseBinary("2.0.1-raring-amd64"),
			Size:    456,
			SHA256:  "xyz",
		},
	}
	writer, err := filestorage.NewFileStorageWriter(dir)
	c.Assert(err, gc.IsNil)
	err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors)
	c.Assert(err, gc.IsNil)
	newToolsList := coretools.List{
		existingToolsList[0],
		{
			Version: version.MustParseBinary("2.1.0-raring-amd64"),
			Size:    789,
			SHA256:  "def",
		},
	}
	err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors)
	c.Assert(err, gc.IsNil)
	requiredToolsList := append(existingToolsList, newToolsList[1])
	metadata := toolstesting.ParseMetadataFromDir(c, dir, false)
	assertMetadataMatches(c, dir, requiredToolsList, metadata)
}
Example #9
0
func (cs *NewAPIStateSuite) TestNewAPIState(c *gc.C) {
	cfg, err := config.New(config.NoDefaults, dummy.SampleConfig())
	c.Assert(err, jc.ErrorIsNil)
	ctx := envtesting.BootstrapContext(c)
	env, err := environs.Prepare(cfg, ctx, configstore.NewMem())
	c.Assert(err, jc.ErrorIsNil)

	storageDir := c.MkDir()
	cs.PatchValue(&envtools.DefaultBaseURL, storageDir)
	stor, err := filestorage.NewFileStorageWriter(storageDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
	c.Assert(err, jc.ErrorIsNil)

	cfg = env.Config()
	cfg, err = cfg.Apply(map[string]interface{}{
		"secret": "fnord",
	})
	c.Assert(err, jc.ErrorIsNil)
	err = env.SetConfig(cfg)
	c.Assert(err, jc.ErrorIsNil)

	st, err := juju.NewAPIState(dummy.AdminUserTag(), env, api.DialOpts{})
	c.Assert(st, gc.NotNil)

	// the secrets will not be updated, as they already exist
	attrs, err := st.Client().EnvironmentGet()
	c.Assert(attrs["secret"], gc.Equals, "pork")

	c.Assert(st.Close(), gc.IsNil)
}
Example #10
0
func (s *NewAPIClientSuite) bootstrapModel(c *gc.C) (environs.Environ, jujuclient.ClientStore) {
	const controllerName = "local.my-controller"

	store := jujuclienttesting.NewMemStore()

	ctx := envtesting.BootstrapContext(c)

	env, err := environs.Prepare(ctx, store, environs.PrepareParams{
		ControllerName: controllerName,
		BaseConfig:     dummy.SampleConfig(),
		CloudName:      "dummy",
	})
	c.Assert(err, jc.ErrorIsNil)

	storageDir := c.MkDir()
	s.PatchValue(&envtools.DefaultBaseURL, storageDir)
	stor, err := filestorage.NewFileStorageWriter(storageDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
	c.Assert(err, jc.ErrorIsNil)

	return env, store
}
Example #11
0
func (s *bootstrapSuite) TestBootstrapMetadata(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	metadataDir, metadata := createImageMetadata(c)
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		MetadataDir: metadataDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)
	c.Assert(envtools.DefaultBaseURL, gc.Equals, metadataDir)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 3)
	c.Assert(datasources[0].Description(), gc.Equals, "bootstrap metadata")
	c.Assert(env.instanceConfig, gc.NotNil)
	c.Assert(env.instanceConfig.CustomImageMetadata, gc.HasLen, 1)
	c.Assert(env.instanceConfig.CustomImageMetadata[0], gc.DeepEquals, metadata[0])
}
Example #12
0
func (s *OpenSuite) TestNewDummyEnviron(c *gc.C) {
	s.PatchValue(&jujuversion.Current, testing.FakeVersionNumber)
	// matches *Settings.Map()
	cfg, err := config.New(config.NoDefaults, dummySampleConfig())
	c.Assert(err, jc.ErrorIsNil)
	ctx := envtesting.BootstrapContext(c)
	cache := jujuclienttesting.NewMemStore()
	controllerCfg := testing.FakeControllerConfig()
	env, err := bootstrap.Prepare(ctx, cache, bootstrap.PrepareParams{
		ControllerConfig: controllerCfg,
		ControllerName:   cfg.Name(),
		ModelConfig:      cfg.AllAttrs(),
		Cloud:            dummy.SampleCloudSpec(),
		AdminSecret:      "admin-secret",
	})
	c.Assert(err, jc.ErrorIsNil)

	storageDir := c.MkDir()
	s.PatchValue(&envtools.DefaultBaseURL, storageDir)
	stor, err := filestorage.NewFileStorageWriter(storageDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, cfg.AgentStream(), cfg.AgentStream())
	err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{
		ControllerConfig: controllerCfg,
		AdminSecret:      "admin-secret",
		CAPrivateKey:     testing.CAKey,
	})
	c.Assert(err, jc.ErrorIsNil)

	// New controller should have been added to collection.
	foundController, err := cache.ControllerByName(cfg.Name())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(foundController.ControllerUUID, gc.DeepEquals, controllerCfg.ControllerUUID())
}
Example #13
0
// PrimeAgentVersion writes the configuration file and tools with version
// vers for an agent with the given entity name. It returns the agent's
// configuration and the current tools.
func (s *AgentSuite) PrimeAgentVersion(c *gc.C, tag names.Tag, password string, vers version.Binary) (agent.ConfigSetterWriter, *coretools.Tools) {
	c.Logf("priming agent %s", tag.String())
	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	agentTools := envtesting.PrimeTools(c, stor, s.DataDir(), "released", vers)
	err = envtools.MergeAndWriteMetadata(stor, "released", "released", coretools.List{agentTools}, envtools.DoNotWriteMirrors)
	tools1, err := agenttools.ChangeAgentTools(s.DataDir(), tag.String(), vers)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(tools1, gc.DeepEquals, agentTools)

	stateInfo := s.MongoInfo(c)
	apiInfo := s.APIInfo(c)
	paths := agent.DefaultPaths
	paths.DataDir = s.DataDir()
	conf, err := agent.NewAgentConfig(
		agent.AgentConfigParams{
			Paths:             paths,
			Tag:               tag,
			UpgradedToVersion: vers.Number,
			Password:          password,
			Nonce:             agent.BootstrapNonce,
			StateAddresses:    stateInfo.Addrs,
			APIAddresses:      apiInfo.Addrs,
			CACert:            stateInfo.CACert,
			Model:             apiInfo.ModelTag,
		})
	c.Assert(err, jc.ErrorIsNil)
	conf.SetPassword(password)
	c.Assert(conf.Write(), gc.IsNil)
	s.primeAPIHostPorts(c)
	return conf, agentTools
}
Example #14
0
func makeTools(c *gc.C, metadataDir, stream string, versionStrings []string, withCheckSum bool) coretools.List {
	toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath, stream)
	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
	var toolsList coretools.List
	for _, versionString := range versionStrings {
		binary, err := version.ParseBinary(versionString)
		if err != nil {
			c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
		}
		path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
		data := binary.String()
		err = ioutil.WriteFile(path, []byte(data), 0644)
		c.Assert(err, jc.ErrorIsNil)
		tool := &coretools.Tools{
			Version: binary,
			URL:     path,
		}
		if withCheckSum {
			tool.Size, tool.SHA256 = SHA256sum(c, path)
		}
		toolsList = append(toolsList, tool)
	}
	// Write the tools metadata.
	stor, err := filestorage.NewFileStorageWriter(metadataDir)
	c.Assert(err, jc.ErrorIsNil)
	err = tools.MergeAndWriteMetadata(stor, stream, stream, toolsList, false)
	c.Assert(err, jc.ErrorIsNil)
	return toolsList
}
Example #15
0
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
	toolsList := coretools.List{
		{
			Version: version.MustParseBinary("1.2.3-precise-amd64"),
			Size:    123,
			SHA256:  "abcd",
		}, {
			Version: version.MustParseBinary("2.0.1-raring-amd64"),
			Size:    456,
			SHA256:  "xyz",
		},
	}
	expected := toolsList

	// Add tools with an unknown series. Do not add an entry in the
	// expected list as these tools should be ignored.
	vers, err := version.ParseBinary("3.2.1-xuanhuaceratops-amd64")
	c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
	toolsList = append(toolsList, &coretools.Tools{
		Version: vers,
		Size:    456,
		SHA256:  "wqe",
	})

	dir := c.MkDir()
	writer, err := filestorage.NewFileStorageWriter(dir)
	c.Assert(err, jc.ErrorIsNil)
	err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, tools.DoNotWriteMirrors)
	c.Assert(err, jc.ErrorIsNil)
	metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", false)
	assertMetadataMatches(c, dir, "proposed", expected, metadata)
}
Example #16
0
func (s *UpgradeJujuSuite) setUpEnvAndTools(c *gc.C, currentVersion string, agentVersion string, tools []string) {
	current := version.MustParseBinary(currentVersion)
	s.PatchValue(&jujuversion.Current, current.Number)
	s.PatchValue(&arch.HostArch, func() string { return current.Arch })
	s.PatchValue(&series.HostSeries, func() string { return current.Series })

	toolsDir := c.MkDir()
	updateAttrs := map[string]interface{}{
		"agent-version":      agentVersion,
		"agent-metadata-url": "file://" + toolsDir + "/tools",
	}

	err := s.State.UpdateModelConfig(updateAttrs, nil, nil)
	c.Assert(err, jc.ErrorIsNil)
	versions := make([]version.Binary, len(tools))
	for i, v := range tools {
		versions[i], err = version.ParseBinary(v)
		if err != nil {
			c.Assert(err, jc.Satisfies, series.IsUnknownOSForSeriesError)
		}
	}
	if len(versions) > 0 {
		stor, err := filestorage.NewFileStorageWriter(toolsDir)
		c.Assert(err, jc.ErrorIsNil)
		envtesting.MustUploadFakeToolsVersions(stor, s.Environ.Config().AgentStream(), versions...)
	}
}
Example #17
0
func (c *ImageMetadataCommand) Run(context *cmd.Context) error {
	if err := c.setParams(context); err != nil {
		return err
	}
	out := context.Stdout
	im := &imagemetadata.ImageMetadata{
		Id:   c.ImageId,
		Arch: c.Arch,
	}
	cloudSpec := simplestreams.CloudSpec{
		Region:   c.Region,
		Endpoint: c.Endpoint,
	}
	targetStorage, err := filestorage.NewFileStorageWriter(c.Dir)
	if err != nil {
		return err
	}
	err = imagemetadata.MergeAndWriteMetadata(c.Series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage)
	if err != nil {
		return fmt.Errorf("image metadata files could not be created: %v", err)
	}
	dir := context.AbsPath(c.Dir)
	dest := filepath.Join(dir, storage.BaseImagesPath, "streams", "v1")
	fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, dir, dir))
	return nil
}
Example #18
0
func (*metadataHelperSuite) writeMetadataMultipleStream(c *gc.C) (storage.StorageReader, map[string][]*tools.ToolsMetadata) {
	metadata := map[string][]*tools.ToolsMetadata{
		"released": {{
			Release: "precise",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path1",
		}},
		"proposed": {{
			Release: "raring",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path2",
		}},
	}

	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	out, err := tools.ReadAllMetadata(stor)
	c.Assert(out, gc.HasLen, 0)
	c.Assert(err, jc.ErrorIsNil) // non-existence is not an error
	err = tools.WriteMetadata(stor, metadata, []string{"released", "proposed"}, tools.DoNotWriteMirrors)
	c.Assert(err, jc.ErrorIsNil)
	return stor, metadata
}
Example #19
0
func (s *metadataHelperSuite) TestReadWriteMetadataUnchanged(c *gc.C) {
	metadata := map[string][]*tools.ToolsMetadata{
		"released": {{
			Release: "precise",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path1",
		}, {
			Release: "raring",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path2",
		}},
	}

	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
	c.Assert(err, jc.ErrorIsNil)

	s.PatchValue(tools.WriteMetadataFiles, func(stor storage.Storage, metadataInfo []tools.MetadataFile) error {
		// The product data is the same, we only write the indices.
		c.Assert(metadataInfo, gc.HasLen, 2)
		c.Assert(metadataInfo[0].Path, gc.Equals, "streams/v1/index2.json")
		c.Assert(metadataInfo[1].Path, gc.Equals, "streams/v1/index.json")
		return nil
	})
	err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
	c.Assert(err, jc.ErrorIsNil)
}
Example #20
0
func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) {
	var versionStrings = []string{
		"1.2.3-precise-amd64",
		"2.0.1-raring-amd64",
	}
	dir := c.MkDir()
	toolstesting.MakeTools(c, dir, "proposed", versionStrings)

	toolsList := coretools.List{
		{
			// If sha256/size is already known, do not recalculate
			Version: version.MustParseBinary("1.2.3-precise-amd64"),
			Size:    123,
			SHA256:  "abcd",
		}, {
			Version: version.MustParseBinary("2.0.1-raring-amd64"),
			// The URL is not used for generating metadata.
			URL: "bogus://",
		},
	}
	writer, err := filestorage.NewFileStorageWriter(dir)
	c.Assert(err, jc.ErrorIsNil)
	writeMirrors := tools.DoNotWriteMirrors
	if withMirrors {
		writeMirrors = tools.WriteMirrors
	}
	err = tools.MergeAndWriteMetadata(writer, "proposed", "proposed", toolsList, writeMirrors)
	c.Assert(err, jc.ErrorIsNil)
	metadata := toolstesting.ParseMetadataFromDir(c, dir, "proposed", withMirrors)
	assertMetadataMatches(c, dir, "proposed", toolsList, metadata)

	// No release stream generated so there will not be a legacy index file created.
	_, err = writer.Get("tools/streams/v1/index.json")
	c.Assert(err, gc.NotNil)
}
Example #21
0
func (*metadataHelperSuite) TestReadWriteMetadataSingleStream(c *gc.C) {
	metadata := map[string][]*tools.ToolsMetadata{
		"released": {{
			Release: "precise",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path1",
		}, {
			Release: "raring",
			Version: "1.2.3",
			Arch:    "amd64",
			Path:    "path2",
		}},
	}

	stor, err := filestorage.NewFileStorageWriter(c.MkDir())
	c.Assert(err, jc.ErrorIsNil)
	out, err := tools.ReadAllMetadata(stor)
	c.Assert(err, jc.ErrorIsNil) // non-existence is not an error
	c.Assert(out, gc.HasLen, 0)
	err = tools.WriteMetadata(stor, metadata, []string{"released"}, tools.DoNotWriteMirrors)
	c.Assert(err, jc.ErrorIsNil)

	// Read back what was just written.
	out, err = tools.ReadAllMetadata(stor)
	c.Assert(err, jc.ErrorIsNil)
	for _, outMetadata := range out {
		for _, md := range outMetadata {
			// FullPath is set by ReadAllMetadata.
			c.Assert(md.FullPath, gc.Not(gc.Equals), "")
			md.FullPath = ""
		}
	}
	c.Assert(out, jc.DeepEquals, metadata)
}
Example #22
0
func (s *NewAPIClientSuite) bootstrapEnv(c *gc.C, envName string, store configstore.Storage) {
	if store == nil {
		store = configstore.NewMem()
	}
	ctx := envtesting.BootstrapContext(c)
	c.Logf("env name: %s", envName)
	env, err := environs.PrepareFromName(envName, ctx, store)
	c.Assert(err, jc.ErrorIsNil)

	storageDir := c.MkDir()
	s.PatchValue(&envtools.DefaultBaseURL, storageDir)
	stor, err := filestorage.NewFileStorageWriter(storageDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	err = bootstrap.Bootstrap(ctx, env, bootstrap.BootstrapParams{})
	c.Assert(err, jc.ErrorIsNil)
	info, err := store.ReadInfo(envName)
	c.Assert(err, jc.ErrorIsNil)
	creds := info.APICredentials()
	creds.User = dummy.AdminUserTag().Name()
	c.Logf("set creds: %#v", creds)
	info.SetAPICredentials(creds)
	err = info.Write()
	c.Assert(err, jc.ErrorIsNil)
	c.Logf("creds: %#v", info.APICredentials())
	info, err = store.ReadInfo(envName)
	c.Assert(err, jc.ErrorIsNil)
	c.Logf("read creds: %#v", info.APICredentials())
	c.Logf("store: %#v", store)
}
Example #23
0
func (s *uploadSuite) TestUploadErrors(c *gc.C) {
	destDir := c.MkDir()
	destStor, err := filestorage.NewFileStorageWriter(destDir)
	c.Assert(err, gc.IsNil)
	err = imagemetadata.UploadImageMetadata(destStor, "foo")
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}
Example #24
0
func (s *syncSuite) setUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("issue 1403084: Currently does not work because of jujud problems")
	}
	s.FakeJujuXDGDataHomeSuite.SetUpTest(c)
	s.ToolsFixture.SetUpTest(c)

	// It's important that this be v1.8.x to match the test data.
	s.PatchValue(&jujuversion.Current, version.MustParse("1.8.3"))

	// Create a source storage.
	baseDir := c.MkDir()
	stor, err := filestorage.NewFileStorageWriter(baseDir)
	c.Assert(err, jc.ErrorIsNil)
	s.storage = stor

	// Create a local tools directory.
	s.localStorage = c.MkDir()

	// Populate both local and default tools locations with the public tools.
	versionStrings := make([]string, len(vAll))
	for i, vers := range vAll {
		versionStrings[i] = vers.String()
	}
	toolstesting.MakeTools(c, baseDir, "released", versionStrings)
	toolstesting.MakeTools(c, s.localStorage, "released", versionStrings)

	// Switch the default tools location.
	baseURL, err := s.storage.URL(storage.BaseToolsPath)
	c.Assert(err, jc.ErrorIsNil)
	s.PatchValue(&envtools.DefaultBaseURL, baseURL)
}
Example #25
0
func (s *bootstrapSuite) TestBootstrapMetadataImagesMissing(c *gc.C) {
	environs.UnregisterImageDataSourceFunc("bootstrap metadata")

	noImagesDir := c.MkDir()
	stor, err := filestorage.NewFileStorageWriter(noImagesDir)
	c.Assert(err, jc.ErrorIsNil)
	envtesting.UploadFakeTools(c, stor, "released", "released")

	env := newEnviron("foo", useDefaultKeys, nil)
	s.setDummyStorage(c, env)
	err = bootstrap.Bootstrap(envtesting.BootstrapContext(c), env, bootstrap.BootstrapParams{
		ControllerConfig: coretesting.FakeControllerConfig(),
		AdminSecret:      "admin-secret",
		CAPrivateKey:     coretesting.CAKey,
		MetadataDir:      noImagesDir,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(env.bootstrapCount, gc.Equals, 1)

	datasources, err := environs.ImageMetadataSources(env)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(datasources, gc.HasLen, 2)
	c.Assert(datasources[0].Description(), gc.Equals, "default cloud images")
	c.Assert(datasources[1].Description(), gc.Equals, "default ubuntu cloud images")
}
Example #26
0
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
	for i, test := range upgradeJujuTests {
		c.Logf("\ntest %d: %s", i, test.about)
		s.Reset(c)
		tools.DefaultBaseURL = ""

		// Set up apparent CLI version and initialize the command.
		s.PatchValue(&version.Current, version.MustParseBinary(test.currentVersion))
		com := &UpgradeJujuCommand{}
		if err := coretesting.InitCommand(envcmd.Wrap(com), test.args); err != nil {
			if test.expectInitErr != "" {
				c.Check(err, gc.ErrorMatches, test.expectInitErr)
			} else {
				c.Check(err, jc.ErrorIsNil)
			}
			continue
		}

		// Set up state and environ, and run the command.
		toolsDir := c.MkDir()
		updateAttrs := map[string]interface{}{
			"agent-version":      test.agentVersion,
			"agent-metadata-url": "file://" + toolsDir + "/tools",
		}
		err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
		c.Assert(err, jc.ErrorIsNil)
		versions := make([]version.Binary, len(test.tools))
		for i, v := range test.tools {
			versions[i] = version.MustParseBinary(v)
		}
		if len(versions) > 0 {
			stor, err := filestorage.NewFileStorageWriter(toolsDir)
			c.Assert(err, jc.ErrorIsNil)
			envtesting.MustUploadFakeToolsVersions(stor, s.Environ.Config().AgentStream(), versions...)
		}

		err = com.Run(coretesting.Context(c))
		if test.expectErr != "" {
			c.Check(err, gc.ErrorMatches, test.expectErr)
			continue
		} else if !c.Check(err, jc.ErrorIsNil) {
			continue
		}

		// Check expected changes to environ/state.
		cfg, err := s.State.EnvironConfig()
		c.Check(err, jc.ErrorIsNil)
		agentVersion, ok := cfg.AgentVersion()
		c.Check(ok, jc.IsTrue)
		c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))

		for _, uploaded := range test.expectUploaded {
			// Substitute latest LTS for placeholder in expected series for uploaded tools
			uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)
			vers := version.MustParseBinary(uploaded)
			s.checkToolsUploaded(c, vers, agentVersion)
		}
	}
}
Example #27
0
// startServer starts a new local storage server
// using a temporary directory and returns the listener,
// a base URL for the server and the directory path.
func startServer(c *gc.C) (listener net.Listener, url, dataDir string) {
	dataDir = c.MkDir()
	embedded, err := filestorage.NewFileStorageWriter(dataDir)
	c.Assert(err, gc.IsNil)
	listener, err = httpstorage.Serve("localhost:0", embedded)
	c.Assert(err, gc.IsNil)
	return listener, fmt.Sprintf("http://%s/", listener.Addr()), dataDir
}
Example #28
0
func (s *filestorageSuite) SetUpTest(c *gc.C) {
	s.dir = c.MkDir()
	var err error
	s.reader, err = filestorage.NewFileStorageReader(s.dir)
	c.Assert(err, jc.ErrorIsNil)
	s.writer, err = filestorage.NewFileStorageWriter(s.dir)
	c.Assert(err, jc.ErrorIsNil)
}
Example #29
0
// setLocalStorage creates a filestorage so tools can
// be synced and so forth without having a machine agent
// running.
func (env *localEnviron) setLocalStorage() error {
	storage, err := filestorage.NewFileStorageWriter(env.config.storageDir())
	if err != nil {
		return err
	}
	env.localStorage = storage
	return nil
}
Example #30
0
// populateTools stores uploaded tools in provider storage
// and updates the tools metadata.
//
// TODO(axw) store tools in gridfs, catalogue in state.
func (c *BootstrapCommand) populateTools(env environs.Environ) error {
	agentConfig := c.CurrentConfig()
	dataDir := agentConfig.DataDir()
	tools, err := agenttools.ReadTools(dataDir, version.Current)
	if err != nil {
		return err
	}
	if !strings.HasPrefix(tools.URL, "file://") {
		// Nothing to do since the tools were not uploaded.
		return nil
	}

	// This is a hack: providers using localstorage (local, manual)
	// can't use storage during bootstrap as the localstorage worker
	// isn't running. Use filestorage instead.
	var stor storage.Storage
	storageDir := agentConfig.Value(agent.StorageDir)
	if storageDir != "" {
		stor, err = filestorage.NewFileStorageWriter(storageDir)
		if err != nil {
			return err
		}
	} else {
		stor = env.Storage()
	}

	// Create a temporary directory to contain source and cloned tools.
	tempDir, err := ioutil.TempDir("", "juju-sync-tools")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	destTools := filepath.Join(tempDir, filepath.FromSlash(envtools.StorageName(tools.Version)))
	if err := os.MkdirAll(filepath.Dir(destTools), 0700); err != nil {
		return err
	}
	srcTools := filepath.Join(
		agenttools.SharedToolsDir(dataDir, version.Current),
		"tools.tar.gz",
	)
	if err := utils.CopyFile(destTools, srcTools); err != nil {
		return err
	}

	// Until we catalogue tools in state, we clone the tools
	// for each of the supported series of the same OS.
	otherSeries := version.OSSupportedSeries(version.Current.OS)
	_, err = sync.SyncBuiltTools(stor, &sync.BuiltTools{
		Version:     tools.Version,
		Dir:         tempDir,
		StorageName: envtools.StorageName(tools.Version),
		Sha256Hash:  tools.SHA256,
		Size:        tools.Size,
	}, otherSeries...)
	return err
}