func (s *filestorageSuite) TestGetRefusesTemp(c *gc.C) {
s.createFile(c, ".tmp/test-file")
_, err := storage.Get(s.reader, ".tmp/test-file")
c.Check(err, gc.NotNil)
c.Check(err, jc.Satisfies, os.IsNotExist)
s.createFile(c, ".tmp/foo/test-file")
_, err = storage.Get(s.reader, ".tmp/foo/test-file")
c.Check(err, gc.NotNil)
c.Check(err, jc.Satisfies, os.IsNotExist)
}
func (s *storageSuite) TestGet(c *gc.C) {
stor, storageDir := s.makeStorage(c)
data := []byte("abc\000def")
err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
c.Assert(err, gc.IsNil)
for _, name := range []string{"b", filepath.Join("a", "b")} {
err = ioutil.WriteFile(filepath.Join(storageDir, name), data, 0644)
c.Assert(err, gc.IsNil)
r, err := storage.Get(stor, name)
c.Assert(err, gc.IsNil)
out, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
c.Assert(out, gc.DeepEquals, data)
}
_, err = storage.Get(stor, "notthere")
c.Assert(err, jc.Satisfies, coreerrors.IsNotFound)
}
// fetchToolsHash fetches the tools from storage and calculates
// its size in bytes and computes a SHA256 hash of its contents.
func fetchToolsHash(stor storage.StorageReader, ver version.Binary) (size int64, sha256hash hash.Hash, err error) {
r, err := storage.Get(stor, StorageName(ver))
if err != nil {
return 0, nil, err
}
defer r.Close()
sha256hash = sha256.New()
size, err = io.Copy(sha256hash, r)
return size, sha256hash, err
}
func (*storageSuite) TestGetReturnsNotFoundIf404(c *gc.C) {
container := "container"
filename := "blobname"
response := makeResponse("not found", http.StatusNotFound)
azStorage, transport := makeFakeStorage(container, "account", "")
transport.AddExchange(response, nil)
_, err := storage.Get(azStorage, filename)
c.Assert(err, gc.NotNil)
c.Check(err, jc.Satisfies, errors.IsNotFound)
}
// LoadState reads state from the given storage.
func LoadState(stor storage.StorageReader) (*BootstrapState, error) {
r, err := storage.Get(stor, StateFile)
if err != nil {
if coreerrors.IsNotFound(err) {
return nil, environs.ErrNotBootstrapped
}
return nil, err
}
return loadState(r)
}
func (s *localServerSuite) TestRemoveAll(c *gc.C) {
env := s.Prepare(c)
stor := env.Storage()
for _, a := range []byte("abcdefghijklmnopqrstuvwxyz") {
content := []byte{a}
name := string(content)
err := stor.Put(name, bytes.NewBuffer(content),
int64(len(content)))
c.Assert(err, gc.IsNil)
}
reader, err := storage.Get(stor, "a")
c.Assert(err, gc.IsNil)
allContent, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Assert(string(allContent), gc.Equals, "a")
err = stor.RemoveAll()
c.Assert(err, gc.IsNil)
_, err = storage.Get(stor, "a")
c.Assert(err, gc.NotNil)
}
func assertMirrors(c *gc.C, stor storage.StorageReader, expectMirrors bool) {
r, err := storage.Get(stor, "tools/"+simplestreams.UnsignedMirror)
if err == nil {
defer r.Close()
}
if expectMirrors {
data, err := ioutil.ReadAll(r)
c.Assert(err, gc.IsNil)
c.Assert(string(data), jc.Contains, `"mirrors":`)
} else {
c.Assert(err, gc.NotNil)
}
}
func (t *LiveTests) TestBootstrapVerifyStorage(c *gc.C) {
// Bootstrap automatically verifies that storage is writable.
t.BootstrapOnce(c)
environ := t.Env
stor := environ.Storage()
reader, err := storage.Get(stor, "bootstrap-verify")
c.Assert(err, gc.IsNil)
defer reader.Close()
contents, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(contents), gc.Equals,
"juju-core storage writing verified: ok\n")
}
func storageCopy(source storage.Storage, sourcePath string, target storage.Storage, targetPath string) error {
rc, err := storage.Get(source, sourcePath)
if err != nil {
return err
}
var buf bytes.Buffer
_, err = io.Copy(&buf, rc)
rc.Close()
if err != nil {
return err
}
return target.Put(targetPath, &buf, int64(buf.Len()))
}
func (s *storageSuite) TestFileContentsAreBinary(c *gc.C) {
const filename = "myfile.bin"
data := []byte{0, 1, 255, 2, 254, 3}
stor := s.makeStorage("binary-test")
err := stor.Put(filename, bytes.NewReader(data), int64(len(data)))
c.Assert(err, gc.IsNil)
file, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
content, err := ioutil.ReadAll(file)
c.Assert(err, gc.IsNil)
c.Check(content, gc.DeepEquals, data)
}
func (s *storageSuite) TestWithExclusiveLocks(c *gc.C) {
stor, storageDir := s.makeStorage(c)
// None of the methods (apart from URL) should be able to do anything
// while an exclusive lock is held.
s.flock(c, flockExclusive, storageDir)
_, err := stor.URL("a")
c.Assert(err, gc.IsNil)
c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
c.Assert(stor.Remove("a"), gc.NotNil)
c.Assert(stor.RemoveAll(), gc.NotNil)
_, err = storage.Get(stor, "a")
c.Assert(err, gc.NotNil)
_, err = storage.List(stor, "")
c.Assert(err, gc.NotNil)
}
func (s *storageSuite) TestRemoveDeletesFile(c *gc.C) {
const filename = "doomed.txt"
stor := NewStorage(s.makeEnviron())
s.fakeStoredFile(stor, filename)
err := stor.Remove(filename)
c.Assert(err, gc.IsNil)
_, err = storage.Get(stor, filename)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
listing, err := storage.List(stor, filename)
c.Assert(err, gc.IsNil)
c.Assert(listing, gc.DeepEquals, []string{})
}
// CheckEnvironment checks if an environment has a bootstrap-verify
// that is written by juju-core commands (as compared to one being
// written by Python juju).
//
// If there is no bootstrap-verify file in the storage, it is still
// considered to be a Juju-core environment since early versions have
// not written it out.
//
// Returns InvalidEnvironmentError on failure, nil otherwise.
func CheckEnvironment(environ Environ) error {
stor := environ.Storage()
reader, err := storage.Get(stor, VerificationFilename)
if errors.IsNotFound(err) {
// When verification file does not exist, this is a juju-core
// environment.
return nil
} else if err != nil {
return err
} else if content, err := ioutil.ReadAll(reader); err != nil {
return err
} else if string(content) != verificationContent {
return InvalidEnvironmentError
}
return nil
}
func (suite *StateSuite) TestCreateStateFileWritesEmptyStateFile(c *gc.C) {
stor := suite.newStorage(c)
url, err := bootstrap.CreateStateFile(stor)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, bootstrap.StateFile)
c.Assert(err, gc.IsNil)
data, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(data), gc.Equals, "")
c.Assert(url, gc.NotNil)
expectedURL, err := stor.URL(bootstrap.StateFile)
c.Assert(err, gc.IsNil)
c.Check(url, gc.Equals, expectedURL)
}
func (s *storageSuite) TestWithSharedLocks(c *gc.C) {
stor, storageDir := s.makeStorage(c)
// Get and List should be able to proceed with a shared lock.
// All other methods should fail.
createFiles(c, storageDir, "a")
s.flock(c, flockShared, storageDir)
_, err := storage.Get(stor, "a")
c.Assert(err, gc.IsNil)
_, err = storage.List(stor, "")
c.Assert(err, gc.IsNil)
c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
c.Assert(stor.Remove("a"), gc.NotNil)
c.Assert(stor.RemoveAll(), gc.NotNil)
}
func (s *storageSuite) TestPutStoresRetrievableFile(c *gc.C) {
const filename = "broken-toaster.jpg"
contents := []byte("Contents here")
length := int64(len(contents))
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(contents), length)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(buf, gc.DeepEquals, contents)
}
func (s *verifyStorageSuite) TestVerifyStorage(c *gc.C) {
defer testing.MakeFakeHome(c, existingEnv, "existing").Restore()
ctx := testing.Context(c)
environ, err := environs.PrepareFromName("test", ctx, configstore.NewMem())
c.Assert(err, gc.IsNil)
stor := environ.Storage()
err = environs.VerifyStorage(stor)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, environs.VerificationFilename)
c.Assert(err, gc.IsNil)
defer reader.Close()
contents, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(contents), gc.Equals,
"juju-core storage writing verified: ok\n")
}
func (suite *StateSuite) TestSaveStateWritesStateFile(c *gc.C) {
stor := suite.newStorage(c)
state := bootstrap.BootstrapState{
StateInstances: []instance.Id{instance.Id("an-instance-id")},
}
marshaledState, err := goyaml.Marshal(state)
c.Assert(err, gc.IsNil)
err = bootstrap.SaveState(stor, &state)
c.Assert(err, gc.IsNil)
loadedState, err := storage.Get(stor, bootstrap.StateFile)
c.Assert(err, gc.IsNil)
content, err := ioutil.ReadAll(loadedState)
c.Assert(err, gc.IsNil)
c.Check(content, gc.DeepEquals, marshaledState)
}
func (s *storageSuite) TestPutStopsAtGivenLength(c *gc.C) {
const filename = "xyzzyz.2.xls"
const length = 5
contents := []byte("abcdefghijklmnopqrstuvwxyz")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(contents), length)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, length)
}
func (s *storageSuite) TestPutOverwritesFile(c *gc.C) {
const filename = "foo.bar"
stor := NewStorage(s.makeEnviron())
s.fakeStoredFile(stor, filename)
newContents := []byte("Overwritten")
err := stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(newContents))
c.Check(buf, gc.DeepEquals, newContents)
}
func (s *storageSuite) TestGetRetrievesFile(c *gc.C) {
const filename = "stored-data"
stor := s.makeStorage("get-retrieves-file")
file := s.fakeStoredFile(stor, filename)
base64Content, err := file.GetField("content")
c.Assert(err, gc.IsNil)
content, err := base64.StdEncoding.DecodeString(base64Content)
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(content))
c.Check(buf, gc.DeepEquals, content)
}
func checkRemoveAll(c *gc.C, stor storage.Storage) {
contents := []byte("File contents.")
aFile := "a-file.txt"
err := stor.Put(aFile, bytes.NewBuffer(contents), int64(len(contents)))
c.Assert(err, gc.IsNil)
err = stor.Put("empty-file", bytes.NewBuffer(nil), 0)
c.Assert(err, gc.IsNil)
err = stor.RemoveAll()
c.Assert(err, gc.IsNil)
files, err := storage.List(stor, "")
c.Assert(err, gc.IsNil)
c.Check(files, gc.HasLen, 0)
_, err = storage.Get(stor, aFile)
c.Assert(err, gc.NotNil)
c.Check(err, gc.ErrorMatches, fmt.Sprintf("file %q not found", aFile))
}
func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte) {
r, err := storage.Get(stor, name)
c.Assert(err, gc.IsNil)
c.Check(r, gc.NotNil)
defer r.Close()
data, err := ioutil.ReadAll(r)
c.Check(err, gc.IsNil)
c.Check(data, gc.DeepEquals, contents)
url, err := stor.URL(name)
c.Assert(err, gc.IsNil)
resp, err := http.Get(url)
c.Assert(err, gc.IsNil)
data, err = ioutil.ReadAll(resp.Body)
c.Assert(err, gc.IsNil)
defer resp.Body.Close()
c.Assert(resp.StatusCode, gc.Equals, http.StatusOK, gc.Commentf("error response: %s", data))
c.Check(data, gc.DeepEquals, contents)
}
func (s *storageSuite) TestPutToExistingFileTruncatesAtGivenLength(c *gc.C) {
const filename = "a-file-which-is-mine"
oldContents := []byte("abcdefghijklmnopqrstuvwxyz")
newContents := []byte("xyz")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(oldContents), int64(len(oldContents)))
c.Assert(err, gc.IsNil)
err = stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
c.Assert(err, gc.IsNil)
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
buf, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(len(buf), gc.Equals, len(newContents))
c.Check(buf, gc.DeepEquals, newContents)
}
func (s *filestorageSuite) TestGet(c *gc.C) {
expectedpath, data := s.createFile(c, "test-file")
_, file := filepath.Split(expectedpath)
rc, err := storage.Get(s.reader, file)
c.Assert(err, gc.IsNil)
defer rc.Close()
c.Assert(err, gc.IsNil)
b, err := ioutil.ReadAll(rc)
c.Assert(err, gc.IsNil)
c.Assert(b, gc.DeepEquals, data)
// Get on a non-existant path returns errors.NotFound
_, err = s.reader.Get("nowhere")
c.Assert(err, jc.Satisfies, coreerrors.IsNotFound)
// Get on a directory returns errors.NotFound
s.createFile(c, "dir/file")
_, err = s.reader.Get("dir")
c.Assert(err, jc.Satisfies, coreerrors.IsNotFound)
}
func (s *storageSuite) TestNamesMayHaveSlashes(c *gc.C) {
const filename = "name/with/slashes"
content := []byte("File contents")
stor := NewStorage(s.makeEnviron())
err := stor.Put(filename, bytes.NewReader(content), int64(len(content)))
c.Assert(err, gc.IsNil)
// There's not much we can say about the anonymous URL, except that
// we get one.
anonURL, err := stor.URL(filename)
c.Assert(err, gc.IsNil)
c.Check(anonURL, gc.Matches, "http[s]*://.*")
reader, err := storage.Get(stor, filename)
c.Assert(err, gc.IsNil)
defer reader.Close()
data, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(data, gc.DeepEquals, content)
}
func (*storageSuite) TestGet(c *gc.C) {
blobContent := "test blob"
container := "container"
filename := "blobname"
response := makeResponse(blobContent, http.StatusOK)
azStorage, transport := makeFakeStorage(container, "account", "")
transport.AddExchange(response, nil)
reader, err := storage.Get(azStorage, filename)
c.Assert(err, gc.IsNil)
c.Assert(reader, gc.NotNil)
defer reader.Close()
context, err := azStorage.getStorageContext()
c.Assert(err, gc.IsNil)
c.Assert(transport.ExchangeCount, gc.Equals, 1)
c.Check(transport.Exchanges[0].Request.URL.String(), gc.Matches, context.GetFileURL(container, filename)+"?.*")
data, err := ioutil.ReadAll(reader)
c.Assert(err, gc.IsNil)
c.Check(string(data), gc.Equals, blobContent)
}
func checkFileDoesNotExist(c *gc.C, stor storage.StorageReader, name string) {
r, err := storage.Get(stor, name)
c.Assert(r, gc.IsNil)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
func (s *storageSuite) TestGetNoRetryAllowed(c *gc.C) {
stor := &fakeStorage{}
storage.Get(stor, "foo")
c.Assert(stor.getName, gc.Equals, "foo")
c.Assert(stor.invokeCount, gc.Equals, 1)
}
func (s *storageSuite) TestGet(c *gc.C) {
stor := &fakeStorage{shouldRetry: true}
storage.Get(stor, "foo")
c.Assert(stor.getName, gc.Equals, "foo")
c.Assert(stor.invokeCount, gc.Equals, 10)
}