func createImageMetadata(c *gc.C) (sourceDir string, destDir string, destStor storage.Storage, metadata *imagemetadata.ImageMetadata) {
destDir = c.MkDir()
var err error
destStor, err = filestorage.NewFileStorageWriter(destDir)
c.Assert(err, gc.IsNil)
// Generate some metadata.
sourceDir = c.MkDir()
im := []*imagemetadata.ImageMetadata{
{
Id: "1234",
Arch: "amd64",
Version: "13.04",
},
}
cloudSpec := &simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
}
im[0].RegionName = cloudSpec.Region
im[0].Endpoint = cloudSpec.Endpoint
var sourceStor storage.Storage
sourceStor, err = filestorage.NewFileStorageWriter(sourceDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor)
c.Assert(err, gc.IsNil)
return sourceDir, destDir, destStor, im[0]
}
func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) {
dir := c.MkDir()
existingToolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abc",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
newToolsList := coretools.List{
existingToolsList[0],
{
Version: version.MustParseBinary("2.1.0-raring-amd64"),
Size: 789,
SHA256: "def",
},
}
err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
requiredToolsList := append(existingToolsList, newToolsList[1])
metadata := ttesting.ParseMetadataFromDir(c, dir, false)
assertMetadataMatches(c, dir, requiredToolsList, metadata)
}
func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) {
metadata := []*tools.ToolsMetadata{{
Release: "precise",
Version: "1.2.3",
Arch: "amd64",
Path: "path1",
}, {
Release: "raring",
Version: "1.2.3",
Arch: "amd64",
Path: "path2",
}}
stor, err := filestorage.NewFileStorageWriter(c.MkDir())
c.Assert(err, gc.IsNil)
out, err := tools.ReadMetadata(stor)
c.Assert(out, gc.HasLen, 0)
c.Assert(err, gc.IsNil) // non-existence is not an error
err = tools.WriteMetadata(stor, metadata, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
out, err = tools.ReadMetadata(stor)
for _, md := range out {
// FullPath is set by ReadMetadata.
c.Assert(md.FullPath, gc.Not(gc.Equals), "")
md.FullPath = ""
}
c.Assert(out, gc.DeepEquals, metadata)
}
func (c *SyncToolsCommand) Run(ctx *cmd.Context) (resultErr error) {
// Register writer for output on screen.
loggo.RegisterWriter("synctools", cmd.NewCommandLogWriter("juju.environs.sync", ctx.Stdout, ctx.Stderr), loggo.INFO)
defer loggo.RemoveWriter("synctools")
environ, cleanup, err := environFromName(ctx, c.EnvName, &resultErr, "Sync-tools")
if err != nil {
return err
}
defer cleanup()
target := environ.Storage()
if c.localDir != "" {
target, err = filestorage.NewFileStorageWriter(c.localDir)
if err != nil {
return err
}
}
// Prepare syncing.
sctx := &sync.SyncContext{
Target: target,
AllVersions: c.allVersions,
MajorVersion: c.majorVersion,
MinorVersion: c.minorVersion,
DryRun: c.dryRun,
Dev: c.dev,
Public: c.public,
Source: c.source,
}
return syncTools(sctx)
}
func (s *uploadSuite) TestUploadErrors(c *gc.C) {
destDir := c.MkDir()
destStor, err := filestorage.NewFileStorageWriter(destDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.UploadImageMetadata(destStor, "foo")
c.Assert(err, jc.Satisfies, os.IsNotExist)
}
func makeTools(c *gc.C, metadataDir, subdir string, versionStrings []string, withCheckSum bool) coretools.List {
toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath)
if subdir != "" {
toolsDir = filepath.Join(toolsDir, subdir)
}
c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
var toolsList coretools.List
for _, versionString := range versionStrings {
binary := version.MustParseBinary(versionString)
path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
data := binary.String()
err := ioutil.WriteFile(path, []byte(data), 0644)
c.Assert(err, gc.IsNil)
tool := &coretools.Tools{
Version: binary,
URL: path,
}
if withCheckSum {
tool.Size, tool.SHA256 = SHA256sum(c, path)
}
toolsList = append(toolsList, tool)
}
// Write the tools metadata.
stor, err := filestorage.NewFileStorageWriter(metadataDir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(stor, toolsList, false)
c.Assert(err, gc.IsNil)
return toolsList
}
func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) {
var versionStrings = []string{
"1.2.3-precise-amd64",
"2.0.1-raring-amd64",
}
dir := c.MkDir()
ttesting.MakeTools(c, dir, "releases", versionStrings)
toolsList := coretools.List{
{
// If sha256/size is already known, do not recalculate
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
// The URL is not used for generating metadata.
URL: "bogus://",
},
}
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
writeMirrors := tools.DoNotWriteMirrors
if withMirrors {
writeMirrors = tools.WriteMirrors
}
err = tools.MergeAndWriteMetadata(writer, toolsList, writeMirrors)
c.Assert(err, gc.IsNil)
metadata := ttesting.ParseMetadataFromDir(c, dir, withMirrors)
assertMetadataMatches(c, dir, toolsList, metadata)
}
func (c *ImageMetadataCommand) Run(context *cmd.Context) error {
if err := c.setParams(context); err != nil {
return err
}
out := context.Stdout
im := &imagemetadata.ImageMetadata{
Id: c.ImageId,
Arch: c.Arch,
}
cloudSpec := simplestreams.CloudSpec{
Region: c.Region,
Endpoint: c.Endpoint,
}
targetStorage, err := filestorage.NewFileStorageWriter(c.Dir)
if err != nil {
return err
}
err = imagemetadata.MergeAndWriteMetadata(c.Series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage)
if err != nil {
return fmt.Errorf("image metadata files could not be created: %v", err)
}
dir := context.AbsPath(c.Dir)
dest := filepath.Join(dir, storage.BaseImagesPath, "streams", "v1")
fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, dir, dir))
return nil
}
// startServer starts a new local storage server
// using a temporary directory and returns the listener,
// a base URL for the server and the directory path.
func startServer(c *gc.C) (listener net.Listener, url, dataDir string) {
dataDir = c.MkDir()
embedded, err := filestorage.NewFileStorageWriter(dataDir)
c.Assert(err, gc.IsNil)
listener, err = httpstorage.Serve("localhost:0", embedded)
c.Assert(err, gc.IsNil)
return listener, fmt.Sprintf("http://%s/", listener.Addr()), dataDir
}
func (s *filestorageSuite) SetUpTest(c *gc.C) {
s.dir = c.MkDir()
var err error
s.reader, err = filestorage.NewFileStorageReader(s.dir)
c.Assert(err, gc.IsNil)
s.writer, err = filestorage.NewFileStorageWriter(s.dir)
c.Assert(err, gc.IsNil)
}
// CreateLocalTestStorage returns the listener, which needs to be closed, and
// the storage that is backed by a directory created in the running test's temp
// directory.
func CreateLocalTestStorage(c *gc.C) (closer io.Closer, stor storage.Storage, dataDir string) {
dataDir = c.MkDir()
underlying, err := filestorage.NewFileStorageWriter(dataDir)
c.Assert(err, gc.IsNil)
listener, err := httpstorage.Serve("localhost:0", underlying)
c.Assert(err, gc.IsNil)
stor = httpstorage.Client(listener.Addr().String())
closer = listener
return
}
func (s *bootstrapSuite) SetUpTest(c *gc.C) {
s.JujuConnSuite.SetUpTest(c)
s.env = &localStorageEnviron{
Environ: s.Conn.Environ,
storageDir: c.MkDir(),
}
storage, err := filestorage.NewFileStorageWriter(s.env.storageDir)
c.Assert(err, gc.IsNil)
s.env.storage = storage
}
func (s *ValidateToolsMetadataSuite) makeLocalMetadata(c *gc.C, version, region, series, endpoint string) error {
tm := []*tools.ToolsMetadata{{
Version: version,
Arch: "amd64",
Release: series,
}}
targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir)
c.Assert(err, gc.IsNil)
err = tools.WriteMetadata(targetStorage, tm, false)
if err != nil {
return err
}
return nil
}
// uploadToStorage uploads the tools from the specified directory to environment storage.
func (h *toolsHandler) uploadToStorage(uploadedTools *tools.Tools, toolsDir,
toolsFilename string, fakeSeries ...string) (*tools.Tools, bool, error) {
// SyncTools requires simplestreams metadata to find the tools to upload.
stor, err := filestorage.NewFileStorageWriter(toolsDir)
if err != nil {
return nil, false, fmt.Errorf("cannot create metadata storage: %v", err)
}
// Generate metadata for the fake series. The URL for each fake series
// record points to the same tools tarball.
allToolsMetadata := []*tools.Tools{uploadedTools}
for _, series := range fakeSeries {
vers := uploadedTools.Version
vers.Series = series
allToolsMetadata = append(allToolsMetadata, &tools.Tools{
Version: vers,
URL: uploadedTools.URL,
Size: uploadedTools.Size,
SHA256: uploadedTools.SHA256,
})
}
err = envtools.MergeAndWriteMetadata(stor, allToolsMetadata, false)
if err != nil {
return nil, false, fmt.Errorf("cannot get environment config: %v", err)
}
// Create the environment so we can get the storage to which we upload the tools.
envConfig, err := h.state.EnvironConfig()
if err != nil {
return nil, false, fmt.Errorf("cannot get environment config: %v", err)
}
env, err := environs.New(envConfig)
if err != nil {
return nil, false, fmt.Errorf("cannot access environment: %v", err)
}
// Now perform the upload.
builtTools := &sync.BuiltTools{
Version: uploadedTools.Version,
Dir: toolsDir,
StorageName: toolsFilename,
Size: uploadedTools.Size,
Sha256Hash: uploadedTools.SHA256,
}
uploadedTools, err = sync.SyncBuiltTools(env.Storage(), builtTools, fakeSeries...)
if err != nil {
return nil, false, err
}
return uploadedTools, !envConfig.SSLHostnameVerification(), nil
}
// startServerTLS starts a new TLS-based local storage server
// using a temporary directory and returns the listener,
// a base URL for the server and the directory path.
func startServerTLS(c *gc.C) (listener net.Listener, url, dataDir string) {
dataDir = c.MkDir()
embedded, err := filestorage.NewFileStorageWriter(dataDir)
c.Assert(err, gc.IsNil)
hostnames := []string{"127.0.0.1"}
listener, err = httpstorage.ServeTLS(
"127.0.0.1:0",
embedded,
coretesting.CACert,
coretesting.CAKey,
hostnames,
testAuthkey,
)
c.Assert(err, gc.IsNil)
return listener, fmt.Sprintf("http://localhost:%d/", listener.Addr().(*net.TCPAddr).Port), dataDir
}
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified
// series and generates corresponding metadata.
func cloneToolsForSeries(toolsInfo *BuiltTools, series ...string) error {
// Copy the tools to the target storage, recording a Tools struct for each one.
var targetTools coretools.List
targetTools = append(targetTools, &coretools.Tools{
Version: toolsInfo.Version,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
putTools := func(vers version.Binary) (string, error) {
name := envtools.StorageName(vers)
src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
dest := filepath.Join(toolsInfo.Dir, name)
err := utils.CopyFile(dest, src)
if err != nil {
return "", err
}
// Append to targetTools the attributes required to write out tools metadata.
targetTools = append(targetTools, &coretools.Tools{
Version: vers,
Size: toolsInfo.Size,
SHA256: toolsInfo.Sha256Hash,
})
return name, nil
}
logger.Debugf("generating tarballs for %v", series)
for _, series := range series {
_, err := ubuntu.SeriesVersion(series)
if err != nil {
return err
}
if series != toolsInfo.Version.Series {
fakeVersion := toolsInfo.Version
fakeVersion.Series = series
if _, err := putTools(fakeVersion); err != nil {
return err
}
}
}
// The tools have been copied to a temp location from which they will be uploaded,
// now write out the matching simplestreams metadata so that SyncTools can find them.
metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
if err != nil {
return err
}
logger.Debugf("generating tools metadata")
return envtools.MergeAndWriteMetadata(metadataStore, targetTools, false)
}
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, version, series string) error {
tm := []*ToolsMetadata{{
Version: version,
Release: series,
Arch: "amd64",
Path: "/tools/tools.tar.gz",
Size: 1234,
FileType: "tar.gz",
SHA256: "f65a92b3b41311bdf398663ee1c5cd0c",
}}
stor, err := filestorage.NewFileStorageWriter(s.metadataDir)
c.Assert(err, gc.IsNil)
err = WriteMetadata(stor, tm, false)
c.Assert(err, gc.IsNil)
return nil
}
func (s *syncSuite) setUpTest(c *gc.C) {
s.LoggingSuite.SetUpTest(c)
s.ToolsFixture.SetUpTest(c)
s.origVersion = version.Current
// It's important that this be v1.8.x to match the test data.
version.Current.Number = version.MustParse("1.8.3")
// Create a target environments.yaml.
fakeHome := coretesting.MakeFakeHome(c, `
environments:
test-target:
type: dummy
state-server: false
authorized-keys: "not-really-one"
`)
s.AddCleanup(func(*gc.C) { fakeHome.Restore() })
var err error
s.targetEnv, err = environs.PrepareFromName("test-target", coretesting.Context(c), configstore.NewMem())
c.Assert(err, gc.IsNil)
envtesting.RemoveAllTools(c, s.targetEnv)
// Create a source storage.
baseDir := c.MkDir()
stor, err := filestorage.NewFileStorageWriter(baseDir)
c.Assert(err, gc.IsNil)
s.storage = stor
// Create a local tools directory.
s.localStorage = c.MkDir()
// Populate both local and default tools locations with the public tools.
versionStrings := make([]string, len(vAll))
for i, vers := range vAll {
versionStrings[i] = vers.String()
}
ttesting.MakeTools(c, baseDir, "releases", versionStrings)
ttesting.MakeTools(c, s.localStorage, "releases", versionStrings)
// Switch the default tools location.
baseURL, err := s.storage.URL(storage.BaseToolsPath)
c.Assert(err, gc.IsNil)
s.PatchValue(&envtools.DefaultBaseURL, baseURL)
}
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint, stream string) error {
metadata := []*imagemetadata.ImageMetadata{
{
Id: id,
Arch: "amd64",
Stream: stream,
},
}
cloudSpec := simplestreams.CloudSpec{
Region: region,
Endpoint: endpoint,
}
targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.MergeAndWriteMetadata(series, metadata, &cloudSpec, targetStorage)
if err != nil {
return err
}
return nil
}
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
toolsList := coretools.List{
{
Version: version.MustParseBinary("1.2.3-precise-amd64"),
Size: 123,
SHA256: "abcd",
}, {
Version: version.MustParseBinary("2.0.1-raring-amd64"),
Size: 456,
SHA256: "xyz",
},
}
dir := c.MkDir()
writer, err := filestorage.NewFileStorageWriter(dir)
c.Assert(err, gc.IsNil)
err = tools.MergeAndWriteMetadata(writer, toolsList, tools.DoNotWriteMirrors)
c.Assert(err, gc.IsNil)
metadata := ttesting.ParseMetadataFromDir(c, dir, false)
assertMetadataMatches(c, dir, toolsList, metadata)
}
func (c *ToolsMetadataCommand) Run(context *cmd.Context) error {
loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO)
defer loggo.RemoveWriter("toolsmetadata")
if c.metadataDir == "" {
c.metadataDir = osenv.JujuHome()
} else {
c.metadataDir = context.AbsPath(c.metadataDir)
}
sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir)
if err != nil {
return err
}
fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir)
const minorVersion = -1
toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion)
if err == envtools.ErrNoTools {
var source string
source, err = envtools.ToolsURL(envtools.DefaultBaseURL)
if err != nil {
return err
}
sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames)
toolsList, err = envtools.FindToolsForCloud(
[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
version.Current.Major, minorVersion, coretools.Filter{})
}
if err != nil {
return err
}
targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir)
if err != nil {
return err
}
writeMirrors := envtools.DoNotWriteMirrors
if c.public {
writeMirrors = envtools.WriteMirrors
}
return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors)
}
// getMockBuildTools returns a sync.BuildToolsTarballFunc implementation which generates
// a fake tools tarball.
func (s *UpgradeJujuSuite) getMockBuildTools(c *gc.C) sync.BuildToolsTarballFunc {
return func(forceVersion *version.Number) (*sync.BuiltTools, error) {
// UploadFakeToolsVersions requires a storage to write to.
stor, err := filestorage.NewFileStorageWriter(s.toolsDir)
c.Assert(err, gc.IsNil)
vers := version.Current
if forceVersion != nil {
vers.Number = *forceVersion
}
versions := []version.Binary{vers}
uploadedTools, err := envtesting.UploadFakeToolsVersions(stor, versions...)
c.Assert(err, gc.IsNil)
agentTools := uploadedTools[0]
return &sync.BuiltTools{
Dir: s.toolsDir,
StorageName: envtools.StorageName(vers),
Version: vers,
Size: agentTools.Size,
Sha256Hash: agentTools.SHA256,
}, nil
}
}
// createImageMetadata creates some image metadata in a local directory.
func createImageMetadata(c *gc.C) (string, []*imagemetadata.ImageMetadata) {
// Generate some image metadata.
im := []*imagemetadata.ImageMetadata{
{
Id: "1234",
Arch: "amd64",
Version: "13.04",
RegionName: "region",
Endpoint: "endpoint",
},
}
cloudSpec := &simplestreams.CloudSpec{
Region: "region",
Endpoint: "endpoint",
}
sourceDir := c.MkDir()
sourceStor, err := filestorage.NewFileStorageWriter(sourceDir)
c.Assert(err, gc.IsNil)
err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor)
c.Assert(err, gc.IsNil)
return sourceDir, im
}
func (s *storageWorker) serveStorage(storageAddr, storageDir string, config *config) (net.Listener, error) {
authenticated := len(config.caCertPEM) > 0 && len(config.caKeyPEM) > 0
scheme := "http://"
if authenticated {
scheme = "https://"
}
logger.Infof("serving storage from %s to %s%s", storageDir, scheme, storageAddr)
storage, err := filestorage.NewFileStorageWriter(storageDir)
if err != nil {
return nil, err
}
if authenticated {
return httpstorage.ServeTLS(
storageAddr,
storage,
config.caCertPEM,
config.caKeyPEM,
config.hostnames,
config.authkey,
)
}
return httpstorage.Serve(storageAddr, storage)
}
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
s.PatchValue(&sync.BuildToolsTarball, s.getMockBuildTools(c))
oldVersion := version.Current
defer func() {
version.Current = oldVersion
}()
for i, test := range upgradeJujuTests {
c.Logf("\ntest %d: %s", i, test.about)
s.Reset(c)
// Set up apparent CLI version and initialize the command.
version.Current = version.MustParseBinary(test.currentVersion)
com := &UpgradeJujuCommand{}
if err := coretesting.InitCommand(com, test.args); err != nil {
if test.expectInitErr != "" {
c.Check(err, gc.ErrorMatches, test.expectInitErr)
} else {
c.Check(err, gc.IsNil)
}
continue
}
// Set up state and environ, and run the command.
toolsDir := c.MkDir()
updateAttrs := map[string]interface{}{
"agent-version": test.agentVersion,
"tools-metadata-url": "file://" + toolsDir,
}
err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
c.Assert(err, gc.IsNil)
versions := make([]version.Binary, len(test.tools))
for i, v := range test.tools {
versions[i] = version.MustParseBinary(v)
}
if len(versions) > 0 {
envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...)
stor, err := filestorage.NewFileStorageWriter(toolsDir)
c.Assert(err, gc.IsNil)
envtesting.MustUploadFakeToolsVersions(stor, versions...)
}
err = com.Run(coretesting.Context(c))
if test.expectErr != "" {
c.Check(err, gc.ErrorMatches, test.expectErr)
continue
} else if !c.Check(err, gc.IsNil) {
continue
}
// Check expected changes to environ/state.
cfg, err := s.State.EnvironConfig()
c.Check(err, gc.IsNil)
agentVersion, ok := cfg.AgentVersion()
c.Check(ok, gc.Equals, true)
c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))
for _, uploaded := range test.expectUploaded {
// Substitute latest LTS for placeholder in expected series for uploaded tools
uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)
vers := version.MustParseBinary(uploaded)
r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers))
if !c.Check(err, gc.IsNil) {
continue
}
data, err := ioutil.ReadAll(r)
r.Close()
c.Check(err, gc.IsNil)
expectContent := version.Current
expectContent.Number = agentVersion
checkToolsContent(c, data, "jujud contents "+expectContent.String())
}
}
}