func (s *filestorageSuite) TestGetRefusesTemp(c *gc.C) {
s.createFile(c, ".tmp/test-file")
_, err := storage.Get(s.reader, ".tmp/test-file")
c.Check(err, gc.NotNil)
c.Check(err, jc.Satisfies, os.IsNotExist)
s.createFile(c, ".tmp/foo/test-file")
_, err = storage.Get(s.reader, ".tmp/foo/test-file")
c.Check(err, gc.NotNil)
c.Check(err, jc.Satisfies, os.IsNotExist)
}
func (s *storageSuite) TestGet(c *gc.C) {
stor, storageDir := s.makeStorage(c)
data := []byte("abc\000def")
err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
c.Assert(err, gc.IsNil)
for _, name := range []string{"b", filepath.Join("a", "b")} {
err = ioutil.WriteFile(filepath.Join(storageDir, name), data, 0644)
c.Assert(err, gc.IsNil)
r, err := storage.Get(stor, name)
c.Assert(err, gc.IsNil)
out, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
c.Assert(out, gc.DeepEquals, data)
}
_, err = storage.Get(stor, "notthere")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
func (*storageSuite) TestGetReturnsNotFoundIf404(c *gc.C) {
container := "container"
filename := "blobname"
response := makeResponse("not found", http.StatusNotFound)
azStorage, transport := makeFakeStorage(container, "account", "")
transport.AddExchange(response, nil)
_, err := storage.Get(azStorage, filename)
c.Assert(err, gc.NotNil)
c.Check(err, jc.Satisfies, errors.IsNotFound)
}
// LoadState reads state from the given storage.
func LoadState(stor storage.StorageReader) (*BootstrapState, error) {
r, err := storage.Get(stor, StateFile)
if err != nil {
if errors.IsNotFound(err) {
return nil, environs.ErrNotBootstrapped
}
return nil, err
}
return loadState(r)
}
// fetchToolsHash fetches the tools from storage and calculates
// its size in bytes and computes a SHA256 hash of its contents.
func fetchToolsHash(stor storage.StorageReader, ver version.Binary) (size int64, sha256hash hash.Hash, err error) {
r, err := storage.Get(stor, StorageName(ver))
if err != nil {
return 0, nil, err
}
defer r.Close()
sha256hash = sha256.New()
size, err = io.Copy(sha256hash, r)
return size, sha256hash, err
}
func (s *localServerSuite) TestRemoveAll(c *gc.C) {
env := s.Prepare(c)
stor := env.Storage()
for _, a := range []byte("abcdefghijklmnopqrstuvwxyz") {
content := []byte{a}
name := string(content)
err := stor.Put(name, bytes.NewBuffer(content),
int64(len(content)))
c.Assert(err, gc.IsNil)
}
reader, err := storage.Get(stor, "a")
c.Assert(err, gc.IsNil)
allContent, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Assert(string(allContent), gc.Equals, "a")
err = stor.RemoveAll()
c.Assert(err, gc.IsNil)
_, err = storage.Get(stor, "a")
c.Assert(err, gc.NotNil)
}
func assertMirrors(c *gc.C, stor storage.StorageReader, expectMirrors bool) {
r, err := storage.Get(stor, "tools/"+simplestreams.UnsignedMirror)
if err == nil {
defer r.Close()
}
if expectMirrors {
data, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
c.Assert(string(data), jc.Contains, `"mirrors":`)
} else {
c.Assert(err, gc.NotNil)
}
}
func (t *LiveTests) TestBootstrapVerifyStorage(c *gc.C) {
// Bootstrap automatically verifies that storage is writable.
t.BootstrapOnce(c)
environ := t.Env
stor := environ.Storage()
reader, err := storage.Get(stor, "bootstrap-verify")
c.Assert(err, gc.IsNil)
defer reader.Close()
contents, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(contents), gc.Equals,
"juju-core storage writing verified: ok\n")
}
func storageCopy(source storage.Storage, sourcePath string, target storage.Storage, targetPath string) error {
rc, err := storage.Get(source, sourcePath)
if err != nil {
return err
}
var buf bytes.Buffer
_, err = io.Copy(&buf, rc)
rc.Close()
if err != nil {
return err
}
return target.Put(targetPath, &buf, int64(buf.Len()))
}
func (s *localServerSuite) TestDeleteMoreThan100(c *gc.C) {
env := s.Prepare(c).(environs.EnvironStorage)
stor := env.Storage()
// 6*26 = 156 items
for _, a := range []byte("abcdef") {
for _, b := range []byte("abcdefghijklmnopqrstuvwxyz") {
content := []byte{a, b}
name := string(content)
err := stor.Put(name, bytes.NewBuffer(content),
int64(len(content)))
c.Assert(err, jc.ErrorIsNil)
}
}
reader, err := storage.Get(stor, "ab")
c.Assert(err, jc.ErrorIsNil)
allContent, err := ioutil.ReadAll(reader)
c.Assert(err, jc.ErrorIsNil)
c.Assert(string(allContent), gc.Equals, "ab")
err = stor.RemoveAll()
c.Assert(err, jc.ErrorIsNil)
_, err = storage.Get(stor, "ab")
c.Assert(err, gc.NotNil)
}
func (s *storageSuite) TestFileContentsAreBinary(c *gc.C) {
const filename = "myfile.bin"
data := []byte{0, 1, 255, 2, 254, 3}
stor := s.makeStorage("binary-test")
err := stor.Put(filename, bytes.NewReader(data), int64(len(data)))
c.Assert(err, gc.IsNil)
file, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
content, err := ioutil.ReadAll(file)
c.Assert(err, gc.IsNil)
c.Check(content, gc.DeepEquals, data)
}
func (s *verifyStorageSuite) TestVerifyStorage(c *gc.C) {
ctx := testing.Context(c)
environ, err := environs.PrepareFromName("test", ctx, configstore.NewMem())
c.Assert(err, gc.IsNil)
stor := environ.Storage()
err = environs.VerifyStorage(stor)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, environs.VerificationFilename)
c.Assert(err, gc.IsNil)
defer reader.Close()
contents, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(contents), gc.Equals,
"juju-core storage writing verified: ok\n")
}
func (s *storageSuite) TestWithExclusiveLocks(c *gc.C) {
stor, storageDir := s.makeStorage(c)
// None of the methods (apart from URL) should be able to do anything
// while an exclusive lock is held.
s.flock(c, flockExclusive, storageDir)
_, err := stor.URL("a")
c.Assert(err, gc.IsNil)
c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
c.Assert(stor.Remove("a"), gc.NotNil)
c.Assert(stor.RemoveAll(), gc.NotNil)
_, err = storage.Get(stor, "a")
c.Assert(err, gc.NotNil)
_, err = storage.List(stor, "")
c.Assert(err, gc.NotNil)
}
func (s *storageSuite) TestRemoveDeletesFile(c *gc.C) {
const filename = "doomed.txt"
stor := NewStorage(s.makeEnviron())
s.fakeStoredFile(stor, filename)
err := stor.Remove(filename)
c.Assert(err, gc.IsNil)
_, err = storage.Get(stor, filename)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
listing, err := storage.List(stor, filename)
c.Assert(err, gc.IsNil)
c.Assert(listing, gc.DeepEquals, []string{})
}
func (s *storageSuite) TestWithSharedLocks(c *gc.C) {
stor, storageDir := s.makeStorage(c)
// Get and List should be able to proceed with a shared lock.
// All other methods should fail.
createFiles(c, storageDir, "a")
s.flock(c, flockShared, storageDir)
_, err := storage.Get(stor, "a")
c.Assert(err, gc.IsNil)
_, err = storage.List(stor, "")
c.Assert(err, gc.IsNil)
c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
c.Assert(stor.Remove("a"), gc.NotNil)
c.Assert(stor.RemoveAll(), gc.NotNil)
}
func (s *storageSuite) TestPutStoresRetrievableFile(c *gc.C) {
const filename = "broken-toaster.jpg"
contents := []byte("Contents here")
length := int64(len(contents))
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(contents), length)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(buf, gc.DeepEquals, contents)
}
// CheckEnvironment checks if an environment has a bootstrap-verify
// that is written by juju-core commands (as compared to one being
// written by Python juju).
//
// If there is no bootstrap-verify file in the storage, it is still
// considered to be a Juju-core environment since early versions have
// not written it out.
//
// Returns InvalidEnvironmentError on failure, nil otherwise.
func CheckEnvironment(environ Environ) error {
stor := environ.Storage()
reader, err := storage.Get(stor, VerificationFilename)
if errors.IsNotFound(err) {
// When verification file does not exist, this is a juju-core
// environment.
return nil
} else if err != nil {
return err
} else if content, err := ioutil.ReadAll(reader); err != nil {
return err
} else if string(content) != verificationContent {
return InvalidEnvironmentError
}
return nil
}
func (suite *StateSuite) TestCreateStateFileWritesEmptyStateFile(c *gc.C) {
stor := suite.newStorage(c)
url, err := common.CreateStateFile(stor)
c.Assert(err, jc.ErrorIsNil)
reader, err := storage.Get(stor, common.StateFile)
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(reader)
c.Assert(err, jc.ErrorIsNil)
c.Check(string(data), gc.Equals, "")
c.Assert(url, gc.NotNil)
expectedURL, err := stor.URL(common.StateFile)
c.Assert(err, jc.ErrorIsNil)
c.Check(url, gc.Equals, expectedURL)
}
func (s *storageSuite) TestPutStopsAtGivenLength(c *gc.C) {
const filename = "xyzzyz.2.xls"
const length = 5
contents := []byte("abcdefghijklmnopqrstuvwxyz")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(contents), length)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, length)
}
func (suite *StateSuite) TestSaveStateWritesStateFile(c *gc.C) {
stor := suite.newStorage(c)
state := common.BootstrapState{
StateInstances: []instance.Id{instance.Id("an-instance-id")},
}
marshaledState, err := goyaml.Marshal(state)
c.Assert(err, jc.ErrorIsNil)
err = common.SaveState(stor, &state)
c.Assert(err, jc.ErrorIsNil)
loadedState, err := storage.Get(stor, common.StateFile)
c.Assert(err, jc.ErrorIsNil)
content, err := ioutil.ReadAll(loadedState)
c.Assert(err, jc.ErrorIsNil)
c.Check(content, gc.DeepEquals, marshaledState)
}
func (s *storageSuite) TestPutOverwritesFile(c *gc.C) {
const filename = "foo.bar"
stor := NewStorage(s.makeEnviron())
s.fakeStoredFile(stor, filename)
newContents := []byte("Overwritten")
err := stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(newContents))
c.Check(buf, gc.DeepEquals, newContents)
}
func (s *storageSuite) TestGetRetrievesFile(c *gc.C) {
const filename = "stored-data"
stor := s.makeStorage("get-retrieves-file")
file := s.fakeStoredFile(stor, filename)
base64Content, err := file.GetField("content")
c.Assert(err, gc.IsNil)
content, err := base64.StdEncoding.DecodeString(base64Content)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(content))
c.Check(buf, gc.DeepEquals, content)
}
func checkRemoveAll(c *gc.C, stor storage.Storage) {
contents := []byte("File contents.")
aFile := "a-file.txt"
err := stor.Put(aFile, bytes.NewBuffer(contents), int64(len(contents)))
c.Assert(err, gc.IsNil)
err = stor.Put("empty-file", bytes.NewBuffer(nil), 0)
c.Assert(err, gc.IsNil)
err = stor.RemoveAll()
c.Assert(err, gc.IsNil)
files, err := storage.List(stor, "")
c.Assert(err, gc.IsNil)
c.Check(files, gc.HasLen, 0)
_, err = storage.Get(stor, aFile)
c.Assert(err, gc.NotNil)
c.Check(err, gc.ErrorMatches, fmt.Sprintf("file %q not found", aFile))
}
func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte) {
r, err := storage.Get(stor, name)
c.Assert(err, gc.IsNil)
c.Check(r, gc.NotNil)
defer r.Close()
data, err := ioutil.ReadAll(r)
c.Check(err, gc.IsNil)
c.Check(data, gc.DeepEquals, contents)
url, err := stor.URL(name)
c.Assert(err, gc.IsNil)
resp, err := http.Get(url)
c.Assert(err, gc.IsNil)
data, err = ioutil.ReadAll(resp.Body)
c.Assert(err, gc.IsNil)
defer resp.Body.Close()
c.Assert(resp.StatusCode, gc.Equals, http.StatusOK, gc.Commentf("error response: %s", data))
c.Check(data, gc.DeepEquals, contents)
}
func (s *storageSuite) TestPutToExistingFileTruncatesAtGivenLength(c *gc.C) {
const filename = "a-file-which-is-mine"
oldContents := []byte("abcdefghijklmnopqrstuvwxyz")
newContents := []byte("xyz")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(oldContents), int64(len(oldContents)))
c.Assert(err, gc.IsNil)
err = stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(newContents))
c.Check(buf, gc.DeepEquals, newContents)
}
func (s *filestorageSuite) TestGet(c *gc.C) {
expectedpath, data := s.createFile(c, "test-file")
_, file := filepath.Split(expectedpath)
rc, err := storage.Get(s.reader, file)
c.Assert(err, jc.ErrorIsNil)
defer rc.Close()
c.Assert(err, jc.ErrorIsNil)
b, err := ioutil.ReadAll(rc)
c.Assert(err, jc.ErrorIsNil)
c.Assert(b, gc.DeepEquals, data)
// Get on a non-existant path returns errors.NotFound
_, err = s.reader.Get("nowhere")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
// Get on a directory returns errors.NotFound
s.createFile(c, "dir/file")
_, err = s.reader.Get("dir")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
func (s *storageSuite) TestNamesMayHaveSlashes(c *gc.C) {
const filename = "name/with/slashes"
content := []byte("File contents")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(content), int64(len(content)))
c.Assert(err, gc.IsNil)
// There's not much we can say about the anonymous URL, except that
// we get one.
anonURL, err := stor.URL(filename)
c.Assert(err, gc.IsNil)
c.Check(anonURL, gc.Matches, "http[s]*://.*")
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
data, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(data, gc.DeepEquals, content)
}
func (*storageSuite) TestGet(c *gc.C) {
blobContent := "test blob"
container := "container"
filename := "blobname"
response := makeResponse(blobContent, http.StatusOK)
azStorage, transport := makeFakeStorage(container, "account", "")
transport.AddExchange(response, nil)
reader, err := storage.Get(azStorage, filename)
c.Assert(err, gc.IsNil)
c.Assert(reader, gc.NotNil)
defer reader.Close()
context, err := azStorage.getStorageContext()
c.Assert(err, gc.IsNil)
c.Assert(transport.ExchangeCount, gc.Equals, 1)
c.Check(transport.Exchanges[0].Request.URL.String(), gc.Matches, context.GetFileURL(container, filename)+"?.*")
data, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(data), gc.Equals, blobContent)
}
func (s *UpgradeJujuSuite) TestUpgradeJuju(c *gc.C) {
oldVersion := version.Current
defer func() {
version.Current = oldVersion
}()
for i, test := range upgradeJujuTests {
c.Logf("\ntest %d: %s", i, test.about)
s.Reset(c)
// Set up apparent CLI version and initialize the command.
version.Current = version.MustParseBinary(test.currentVersion)
com := &UpgradeJujuCommand{}
if err := coretesting.InitCommand(envcmd.Wrap(com), test.args); err != nil {
if test.expectInitErr != "" {
c.Check(err, gc.ErrorMatches, test.expectInitErr)
} else {
c.Check(err, gc.IsNil)
}
continue
}
// Set up state and environ, and run the command.
toolsDir := c.MkDir()
updateAttrs := map[string]interface{}{
"agent-version": test.agentVersion,
"tools-metadata-url": "file://" + toolsDir,
}
err := s.State.UpdateEnvironConfig(updateAttrs, nil, nil)
c.Assert(err, gc.IsNil)
versions := make([]version.Binary, len(test.tools))
for i, v := range test.tools {
versions[i] = version.MustParseBinary(v)
}
if len(versions) > 0 {
envtesting.MustUploadFakeToolsVersions(s.Conn.Environ.Storage(), versions...)
stor, err := filestorage.NewFileStorageWriter(toolsDir)
c.Assert(err, gc.IsNil)
envtesting.MustUploadFakeToolsVersions(stor, versions...)
}
err = com.Run(coretesting.Context(c))
if test.expectErr != "" {
c.Check(err, gc.ErrorMatches, test.expectErr)
continue
} else if !c.Check(err, gc.IsNil) {
continue
}
// Check expected changes to environ/state.
cfg, err := s.State.EnvironConfig()
c.Check(err, gc.IsNil)
agentVersion, ok := cfg.AgentVersion()
c.Check(ok, gc.Equals, true)
c.Check(agentVersion, gc.Equals, version.MustParse(test.expectVersion))
for _, uploaded := range test.expectUploaded {
// Substitute latest LTS for placeholder in expected series for uploaded tools
uploaded = strings.Replace(uploaded, "%LTS%", config.LatestLtsSeries(), 1)
vers := version.MustParseBinary(uploaded)
r, err := storage.Get(s.Conn.Environ.Storage(), envtools.StorageName(vers))
if !c.Check(err, gc.IsNil) {
continue
}
data, err := ioutil.ReadAll(r)
r.Close()
c.Check(err, gc.IsNil)
expectContent := version.Current
expectContent.Number = agentVersion
checkToolsContent(c, data, "jujud contents "+expectContent.String())
}
}
}
func (s *storageSuite) TestGetNoRetryAllowed(c *gc.C) {
stor := &fakeStorage{}
storage.Get(stor, "foo")
c.Assert(stor.getName, gc.Equals, "foo")
c.Assert(stor.invokeCount, gc.Equals, 1)
}