func createImageMetadata(c *gc.C) (sourceDir string, destDir string, destStor storage.Storage, metadata *imagemetadata.ImageMetadata) { destDir = c.MkDir() var err error destStor, err = filestorage.NewFileStorageWriter(destDir) c.Assert(err, gc.IsNil) // Generate some metadata. sourceDir = c.MkDir() im := []*imagemetadata.ImageMetadata{ { Id: "1234", Arch: "amd64", Version: "13.04", }, } cloudSpec := &simplestreams.CloudSpec{ Region: "region", Endpoint: "endpoint", } im[0].RegionName = cloudSpec.Region im[0].Endpoint = cloudSpec.Endpoint var sourceStor storage.Storage sourceStor, err = filestorage.NewFileStorageWriter(sourceDir) c.Assert(err, gc.IsNil) err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor) c.Assert(err, gc.IsNil) return sourceDir, destDir, destStor, im[0] }
func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) { dir := c.MkDir() existingToolsList := coretools.List{ { Version: version.MustParseBinary("1.2.3-precise-amd64"), Size: 123, SHA256: "abc", }, { Version: version.MustParseBinary("2.0.1-raring-amd64"), Size: 456, SHA256: "xyz", }, } writer, err := filestorage.NewFileStorageWriter(dir) c.Assert(err, gc.IsNil) err = tools.MergeAndWriteMetadata(writer, existingToolsList, tools.DoNotWriteMirrors) c.Assert(err, gc.IsNil) newToolsList := coretools.List{ existingToolsList[0], { Version: version.MustParseBinary("2.1.0-raring-amd64"), Size: 789, SHA256: "def", }, } err = tools.MergeAndWriteMetadata(writer, newToolsList, tools.DoNotWriteMirrors) c.Assert(err, gc.IsNil) requiredToolsList := append(existingToolsList, newToolsList[1]) metadata := ttesting.ParseMetadataFromDir(c, dir, false) assertMetadataMatches(c, dir, requiredToolsList, metadata) }
func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) { metadata := []*tools.ToolsMetadata{{ Release: "precise", Version: "1.2.3", Arch: "amd64", Path: "path1", }, { Release: "raring", Version: "1.2.3", Arch: "amd64", Path: "path2", }} stor, err := filestorage.NewFileStorageWriter(c.MkDir()) c.Assert(err, gc.IsNil) out, err := tools.ReadMetadata(stor) c.Assert(out, gc.HasLen, 0) c.Assert(err, gc.IsNil) // non-existence is not an error err = tools.WriteMetadata(stor, metadata, tools.DoNotWriteMirrors) c.Assert(err, gc.IsNil) out, err = tools.ReadMetadata(stor) for _, md := range out { // FullPath is set by ReadMetadata. c.Assert(md.FullPath, gc.Not(gc.Equals), "") md.FullPath = "" } c.Assert(out, gc.DeepEquals, metadata) }
func (c *SyncToolsCommand) Run(ctx *cmd.Context) (resultErr error) { // Register writer for output on screen. loggo.RegisterWriter("synctools", cmd.NewCommandLogWriter("juju.environs.sync", ctx.Stdout, ctx.Stderr), loggo.INFO) defer loggo.RemoveWriter("synctools") environ, cleanup, err := environFromName(ctx, c.EnvName, &resultErr, "Sync-tools") if err != nil { return err } defer cleanup() target := environ.Storage() if c.localDir != "" { target, err = filestorage.NewFileStorageWriter(c.localDir) if err != nil { return err } } // Prepare syncing. sctx := &sync.SyncContext{ Target: target, AllVersions: c.allVersions, MajorVersion: c.majorVersion, MinorVersion: c.minorVersion, DryRun: c.dryRun, Dev: c.dev, Public: c.public, Source: c.source, } return syncTools(sctx) }
func (s *uploadSuite) TestUploadErrors(c *gc.C) { destDir := c.MkDir() destStor, err := filestorage.NewFileStorageWriter(destDir) c.Assert(err, gc.IsNil) err = imagemetadata.UploadImageMetadata(destStor, "foo") c.Assert(err, jc.Satisfies, os.IsNotExist) }
func makeTools(c *gc.C, metadataDir, subdir string, versionStrings []string, withCheckSum bool) coretools.List { toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath) if subdir != "" { toolsDir = filepath.Join(toolsDir, subdir) } c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil) var toolsList coretools.List for _, versionString := range versionStrings { binary := version.MustParseBinary(versionString) path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary)) data := binary.String() err := ioutil.WriteFile(path, []byte(data), 0644) c.Assert(err, gc.IsNil) tool := &coretools.Tools{ Version: binary, URL: path, } if withCheckSum { tool.Size, tool.SHA256 = SHA256sum(c, path) } toolsList = append(toolsList, tool) } // Write the tools metadata. stor, err := filestorage.NewFileStorageWriter(metadataDir) c.Assert(err, gc.IsNil) err = tools.MergeAndWriteMetadata(stor, toolsList, false) c.Assert(err, gc.IsNil) return toolsList }
func (s *simplestreamsSuite) assertWriteMetadata(c *gc.C, withMirrors bool) { var versionStrings = []string{ "1.2.3-precise-amd64", "2.0.1-raring-amd64", } dir := c.MkDir() ttesting.MakeTools(c, dir, "releases", versionStrings) toolsList := coretools.List{ { // If sha256/size is already known, do not recalculate Version: version.MustParseBinary("1.2.3-precise-amd64"), Size: 123, SHA256: "abcd", }, { Version: version.MustParseBinary("2.0.1-raring-amd64"), // The URL is not used for generating metadata. URL: "bogus://", }, } writer, err := filestorage.NewFileStorageWriter(dir) c.Assert(err, gc.IsNil) writeMirrors := tools.DoNotWriteMirrors if withMirrors { writeMirrors = tools.WriteMirrors } err = tools.MergeAndWriteMetadata(writer, toolsList, writeMirrors) c.Assert(err, gc.IsNil) metadata := ttesting.ParseMetadataFromDir(c, dir, withMirrors) assertMetadataMatches(c, dir, toolsList, metadata) }
func (c *ImageMetadataCommand) Run(context *cmd.Context) error { if err := c.setParams(context); err != nil { return err } out := context.Stdout im := &imagemetadata.ImageMetadata{ Id: c.ImageId, Arch: c.Arch, } cloudSpec := simplestreams.CloudSpec{ Region: c.Region, Endpoint: c.Endpoint, } targetStorage, err := filestorage.NewFileStorageWriter(c.Dir) if err != nil { return err } err = imagemetadata.MergeAndWriteMetadata(c.Series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage) if err != nil { return fmt.Errorf("image metadata files could not be created: %v", err) } dir := context.AbsPath(c.Dir) dest := filepath.Join(dir, storage.BaseImagesPath, "streams", "v1") fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, dir, dir)) return nil }
// startServer starts a new local storage server // using a temporary directory and returns the listener, // a base URL for the server and the directory path. func startServer(c *gc.C) (listener net.Listener, url, dataDir string) { dataDir = c.MkDir() embedded, err := filestorage.NewFileStorageWriter(dataDir) c.Assert(err, gc.IsNil) listener, err = httpstorage.Serve("localhost:0", embedded) c.Assert(err, gc.IsNil) return listener, fmt.Sprintf("http://%s/", listener.Addr()), dataDir }
func (s *filestorageSuite) SetUpTest(c *gc.C) { s.dir = c.MkDir() var err error s.reader, err = filestorage.NewFileStorageReader(s.dir) c.Assert(err, gc.IsNil) s.writer, err = filestorage.NewFileStorageWriter(s.dir) c.Assert(err, gc.IsNil) }
// CreateLocalTestStorage returns the listener, which needs to be closed, and // the storage that is backed by a directory created in the running test's temp // directory. func CreateLocalTestStorage(c *gc.C) (closer io.Closer, stor storage.Storage, dataDir string) { dataDir = c.MkDir() underlying, err := filestorage.NewFileStorageWriter(dataDir) c.Assert(err, gc.IsNil) listener, err := httpstorage.Serve("localhost:0", underlying) c.Assert(err, gc.IsNil) stor = httpstorage.Client(listener.Addr().String()) closer = listener return }
func (s *bootstrapSuite) SetUpTest(c *gc.C) { s.JujuConnSuite.SetUpTest(c) s.env = &localStorageEnviron{ Environ: s.Conn.Environ, storageDir: c.MkDir(), } storage, err := filestorage.NewFileStorageWriter(s.env.storageDir) c.Assert(err, gc.IsNil) s.env.storage = storage }
func (s *ValidateToolsMetadataSuite) makeLocalMetadata(c *gc.C, version, region, series, endpoint string) error { tm := []*tools.ToolsMetadata{{ Version: version, Arch: "amd64", Release: series, }} targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir) c.Assert(err, gc.IsNil) err = tools.WriteMetadata(targetStorage, tm, false) if err != nil { return err } return nil }
// uploadToStorage uploads the tools from the specified directory to environment storage. func (h *toolsHandler) uploadToStorage(uploadedTools *tools.Tools, toolsDir, toolsFilename string, fakeSeries ...string) (*tools.Tools, bool, error) { // SyncTools requires simplestreams metadata to find the tools to upload. stor, err := filestorage.NewFileStorageWriter(toolsDir) if err != nil { return nil, false, fmt.Errorf("cannot create metadata storage: %v", err) } // Generate metadata for the fake series. The URL for each fake series // record points to the same tools tarball. allToolsMetadata := []*tools.Tools{uploadedTools} for _, series := range fakeSeries { vers := uploadedTools.Version vers.Series = series allToolsMetadata = append(allToolsMetadata, &tools.Tools{ Version: vers, URL: uploadedTools.URL, Size: uploadedTools.Size, SHA256: uploadedTools.SHA256, }) } err = envtools.MergeAndWriteMetadata(stor, allToolsMetadata, false) if err != nil { return nil, false, fmt.Errorf("cannot get environment config: %v", err) } // Create the environment so we can get the storage to which we upload the tools. envConfig, err := h.state.EnvironConfig() if err != nil { return nil, false, fmt.Errorf("cannot get environment config: %v", err) } env, err := environs.New(envConfig) if err != nil { return nil, false, fmt.Errorf("cannot access environment: %v", err) } // Now perform the upload. builtTools := &sync.BuiltTools{ Version: uploadedTools.Version, Dir: toolsDir, StorageName: toolsFilename, Size: uploadedTools.Size, Sha256Hash: uploadedTools.SHA256, } uploadedTools, err = sync.SyncBuiltTools(env.Storage(), builtTools, fakeSeries...) if err != nil { return nil, false, err } return uploadedTools, !envConfig.SSLHostnameVerification(), nil }
// startServerTLS starts a new TLS-based local storage server // using a temporary directory and returns the listener, // a base URL for the server and the directory path. func startServerTLS(c *gc.C) (listener net.Listener, url, dataDir string) { dataDir = c.MkDir() embedded, err := filestorage.NewFileStorageWriter(dataDir) c.Assert(err, gc.IsNil) hostnames := []string{"127.0.0.1"} listener, err = httpstorage.ServeTLS( "127.0.0.1:0", embedded, coretesting.CACert, coretesting.CAKey, hostnames, testAuthkey, ) c.Assert(err, gc.IsNil) return listener, fmt.Sprintf("http://localhost:%d/", listener.Addr().(*net.TCPAddr).Port), dataDir }
// cloneToolsForSeries copies the built tools tarball into a tarball for the specified // series and generates corresponding metadata. func cloneToolsForSeries(toolsInfo *BuiltTools, series ...string) error { // Copy the tools to the target storage, recording a Tools struct for each one. var targetTools coretools.List targetTools = append(targetTools, &coretools.Tools{ Version: toolsInfo.Version, Size: toolsInfo.Size, SHA256: toolsInfo.Sha256Hash, }) putTools := func(vers version.Binary) (string, error) { name := envtools.StorageName(vers) src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName) dest := filepath.Join(toolsInfo.Dir, name) err := utils.CopyFile(dest, src) if err != nil { return "", err } // Append to targetTools the attributes required to write out tools metadata. targetTools = append(targetTools, &coretools.Tools{ Version: vers, Size: toolsInfo.Size, SHA256: toolsInfo.Sha256Hash, }) return name, nil } logger.Debugf("generating tarballs for %v", series) for _, series := range series { _, err := ubuntu.SeriesVersion(series) if err != nil { return err } if series != toolsInfo.Version.Series { fakeVersion := toolsInfo.Version fakeVersion.Series = series if _, err := putTools(fakeVersion); err != nil { return err } } } // The tools have been copied to a temp location from which they will be uploaded, // now write out the matching simplestreams metadata so that SyncTools can find them. metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir) if err != nil { return err } logger.Debugf("generating tools metadata") return envtools.MergeAndWriteMetadata(metadataStore, targetTools, false) }
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, version, series string) error { tm := []*ToolsMetadata{{ Version: version, Release: series, Arch: "amd64", Path: "/tools/tools.tar.gz", Size: 1234, FileType: "tar.gz", SHA256: "f65a92b3b41311bdf398663ee1c5cd0c", }} stor, err := filestorage.NewFileStorageWriter(s.metadataDir) c.Assert(err, gc.IsNil) err = WriteMetadata(stor, tm, false) c.Assert(err, gc.IsNil) return nil }
func (s *syncSuite) setUpTest(c *gc.C) { s.LoggingSuite.SetUpTest(c) s.ToolsFixture.SetUpTest(c) s.origVersion = version.Current // It's important that this be v1.8.x to match the test data. version.Current.Number = version.MustParse("1.8.3") // Create a target environments.yaml. fakeHome := coretesting.MakeFakeHome(c, ` environments: test-target: type: dummy state-server: false authorized-keys: "not-really-one" `) s.AddCleanup(func(*gc.C) { fakeHome.Restore() }) var err error s.targetEnv, err = environs.PrepareFromName("test-target", coretesting.Context(c), configstore.NewMem()) c.Assert(err, gc.IsNil) envtesting.RemoveAllTools(c, s.targetEnv) // Create a source storage. baseDir := c.MkDir() stor, err := filestorage.NewFileStorageWriter(baseDir) c.Assert(err, gc.IsNil) s.storage = stor // Create a local tools directory. s.localStorage = c.MkDir() // Populate both local and default tools locations with the public tools. versionStrings := make([]string, len(vAll)) for i, vers := range vAll { versionStrings[i] = vers.String() } ttesting.MakeTools(c, baseDir, "releases", versionStrings) ttesting.MakeTools(c, s.localStorage, "releases", versionStrings) // Switch the default tools location. baseURL, err := s.storage.URL(storage.BaseToolsPath) c.Assert(err, gc.IsNil) s.PatchValue(&envtools.DefaultBaseURL, baseURL) }
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint, stream string) error { metadata := []*imagemetadata.ImageMetadata{ { Id: id, Arch: "amd64", Stream: stream, }, } cloudSpec := simplestreams.CloudSpec{ Region: region, Endpoint: endpoint, } targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir) c.Assert(err, gc.IsNil) err = imagemetadata.MergeAndWriteMetadata(series, metadata, &cloudSpec, targetStorage) if err != nil { return err } return nil }
func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) { toolsList := coretools.List{ { Version: version.MustParseBinary("1.2.3-precise-amd64"), Size: 123, SHA256: "abcd", }, { Version: version.MustParseBinary("2.0.1-raring-amd64"), Size: 456, SHA256: "xyz", }, } dir := c.MkDir() writer, err := filestorage.NewFileStorageWriter(dir) c.Assert(err, gc.IsNil) err = tools.MergeAndWriteMetadata(writer, toolsList, tools.DoNotWriteMirrors) c.Assert(err, gc.IsNil) metadata := ttesting.ParseMetadataFromDir(c, dir, false) assertMetadataMatches(c, dir, toolsList, metadata) }
func (c *ToolsMetadataCommand) Run(context *cmd.Context) error { loggo.RegisterWriter("toolsmetadata", cmd.NewCommandLogWriter("juju.environs.tools", context.Stdout, context.Stderr), loggo.INFO) defer loggo.RemoveWriter("toolsmetadata") if c.metadataDir == "" { c.metadataDir = osenv.JujuHome() } else { c.metadataDir = context.AbsPath(c.metadataDir) } sourceStorage, err := filestorage.NewFileStorageReader(c.metadataDir) if err != nil { return err } fmt.Fprintf(context.Stdout, "Finding tools in %s\n", c.metadataDir) const minorVersion = -1 toolsList, err := envtools.ReadList(sourceStorage, version.Current.Major, minorVersion) if err == envtools.ErrNoTools { var source string source, err = envtools.ToolsURL(envtools.DefaultBaseURL) if err != nil { return err } sourceDataSource := simplestreams.NewURLDataSource("local source", source, utils.VerifySSLHostnames) toolsList, err = envtools.FindToolsForCloud( []simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{}, version.Current.Major, minorVersion, coretools.Filter{}) } if err != nil { return err } targetStorage, err := filestorage.NewFileStorageWriter(c.metadataDir) if err != nil { return err } writeMirrors := envtools.DoNotWriteMirrors if c.public { writeMirrors = envtools.WriteMirrors } return mergeAndWriteMetadata(targetStorage, toolsList, writeMirrors) }
// getMockBuildTools returns a sync.BuildToolsTarballFunc implementation which generates // a fake tools tarball. func (s *UpgradeJujuSuite) getMockBuildTools(c *gc.C) sync.BuildToolsTarballFunc { return func(forceVersion *version.Number) (*sync.BuiltTools, error) { // UploadFakeToolsVersions requires a storage to write to. stor, err := filestorage.NewFileStorageWriter(s.toolsDir) c.Assert(err, gc.IsNil) vers := version.Current if forceVersion != nil { vers.Number = *forceVersion } versions := []version.Binary{vers} uploadedTools, err := envtesting.UploadFakeToolsVersions(stor, versions...) c.Assert(err, gc.IsNil) agentTools := uploadedTools[0] return &sync.BuiltTools{ Dir: s.toolsDir, StorageName: envtools.StorageName(vers), Version: vers, Size: agentTools.Size, Sha256Hash: agentTools.SHA256, }, nil } }
// createImageMetadata creates some image metadata in a local directory. func createImageMetadata(c *gc.C) (string, []*imagemetadata.ImageMetadata) { // Generate some image metadata. im := []*imagemetadata.ImageMetadata{ { Id: "1234", Arch: "amd64", Version: "13.04", RegionName: "region", Endpoint: "endpoint", }, } cloudSpec := &simplestreams.CloudSpec{ Region: "region", Endpoint: "endpoint", } sourceDir := c.MkDir() sourceStor, err := filestorage.NewFileStorageWriter(sourceDir) c.Assert(err, gc.IsNil) err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, sourceStor) c.Assert(err, gc.IsNil) return sourceDir, im }
func (s *storageWorker) serveStorage(storageAddr, storageDir string, config *config) (net.Listener, error) { authenticated := len(config.caCertPEM) > 0 && len(config.caKeyPEM) > 0 scheme := "http://" if authenticated { scheme = "https://" } logger.Infof("serving storage from %s to %s%s", storageDir, scheme, storageAddr) storage, err := filestorage.NewFileStorageWriter(storageDir) if err != nil { return nil, err } if authenticated { return httpstorage.ServeTLS( storageAddr, storage, config.caCertPEM, config.caKeyPEM, config.hostnames, config.authkey, ) } return httpstorage.Serve(storageAddr, storage) }
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) { s.PatchValue(&sync.BuildToolsTarball, s.getMockBuildTools(c)) oldVersion := version.Current defer func() { version.Current = oldVersion }() for i, test := range upgradeJujuTests { c.Logf("\ntest %d: %s", i, test.about) s.Reset(c) // Set up apparent CLI version and initialize the command. version.Current = version.MustParseBinary(test.currentVersion) com := &UpgradeJujuCommand{} if err := coretesting.InitCommand(com, test.args); err != nil { if test.expectInitErr != "" { c.Check(err, gc.ErrorMatches, test.expectInitErr) } else { c.Check(err, gc.IsNil) } continue } // Set up state and environ, and run the command. toolsDir := c.MkDir() updateAttrs := map[string]interface{}{ "agent-version": test.agentVersion, "tools-metadata-url": "file://" + toolsDir, } err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil) c.Assert(err, gc.IsNil) versions := make([]version.Binary, len(test.tools)) for i, v := range test.tools { versions[i] = version.MustParseBinary(v) } if len(versions) > 0 { envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...) stor, err := filestorage.NewFileStorageWriter(toolsDir) c.Assert(err, gc.IsNil) envtesting.MustUploadFakeToolsVersions(stor, versions...) } err = com.Run(coretesting.Context(c)) if test.expectErr != "" { c.Check(err, gc.ErrorMatches, test.expectErr) continue } else if !c.Check(err, gc.IsNil) { continue } // Check expected changes to environ/state. cfg, err := s.State.EnvironConfig() c.Check(err, gc.IsNil) agentVersion, ok := cfg.AgentVersion() c.Check(ok, gc.Equals, true) c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion)) for _, uploaded := range test.expectUploaded { // Substitute latest LTS for placeholder in expected series for uploaded tools uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1) vers := version.MustParseBinary(uploaded) r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers)) if !c.Check(err, gc.IsNil) { continue } data, err := ioutil.ReadAll(r) r.Close() c.Check(err, gc.IsNil) expectContent := version.Current expectContent.Number = agentVersion checkToolsContent(c, data, "jujud contents "+expectContent.String()) } } }