func (s *charmsSuite) TestUploadRespectsLocalRevision(c *gc.C) {
// Make a dummy charm dir with revision 123.
dir := coretesting.Charms.ClonedDir(c.MkDir(), "dummy")
dir.SetDiskRevision(123)
// Now bundle the dir.
tempFile, err := ioutil.TempFile(c.MkDir(), "charm")
c.Assert(err, gc.IsNil)
defer tempFile.Close()
defer os.Remove(tempFile.Name())
err = dir.BundleTo(tempFile)
c.Assert(err, gc.IsNil)
// Now try uploading it and ensure the revision persists.
resp, err := s.uploadRequest(c, s.charmsURI(c, "?series=quantal"), true, tempFile.Name())
c.Assert(err, gc.IsNil)
expectedURL := charm.MustParseURL("local:quantal/dummy-123")
s.assertUploadResponse(c, resp, expectedURL.String())
sch, err := s.State.Charm(expectedURL)
c.Assert(err, gc.IsNil)
c.Assert(sch.URL(), gc.DeepEquals, expectedURL)
c.Assert(sch.Revision(), gc.Equals, 123)
c.Assert(sch.IsUploaded(), jc.IsTrue)
// First rewind the reader, which was reset but BundleTo() above.
_, err = tempFile.Seek(0, 0)
c.Assert(err, gc.IsNil)
// Finally, verify the SHA256 and uploaded URL.
expectedSHA256, _, err := utils.ReadSHA256(tempFile)
c.Assert(err, gc.IsNil)
name := charm.Quote(expectedURL.String())
storage, err := environs.GetStorage(s.State)
c.Assert(err, gc.IsNil)
expectedUploadURL, err := storage.URL(name)
c.Assert(err, gc.IsNil)
c.Assert(sch.BundleURL().String(), gc.Equals, expectedUploadURL)
c.Assert(sch.BundleSha256(), gc.Equals, expectedSHA256)
reader, err := storage.Get(name)
c.Assert(err, gc.IsNil)
defer reader.Close()
downloadedSHA256, _, err := utils.ReadSHA256(reader)
c.Assert(err, gc.IsNil)
c.Assert(downloadedSHA256, gc.Equals, expectedSHA256)
}
func (conn *Conn) addCharm(curl *charm.URL, ch charm.Charm) (*state.Charm, error) {
var f *os.File
name := charm.Quote(curl.String())
switch ch := ch.(type) {
case *charm.Dir:
var err error
if f, err = ioutil.TempFile("", name); err != nil {
return nil, err
}
defer os.Remove(f.Name())
defer f.Close()
err = ch.BundleTo(f)
if err != nil {
return nil, fmt.Errorf("cannot bundle charm: %v", err)
}
if _, err := f.Seek(0, 0); err != nil {
return nil, err
}
case *charm.Bundle:
var err error
if f, err = os.Open(ch.Path); err != nil {
return nil, fmt.Errorf("cannot read charm bundle: %v", err)
}
defer f.Close()
default:
return nil, fmt.Errorf("unknown charm type %T", ch)
}
digest, size, err := utils.ReadSHA256(f)
if err != nil {
return nil, err
}
if _, err := f.Seek(0, 0); err != nil {
return nil, err
}
stor := conn.Environ.Storage()
logger.Infof("writing charm to storage [%d bytes]", size)
if err := stor.Put(name, f, size); err != nil {
return nil, fmt.Errorf("cannot put charm: %v", err)
}
ustr, err := stor.URL(name)
if err != nil {
return nil, fmt.Errorf("cannot get storage URL for charm: %v", err)
}
u, err := url.Parse(ustr)
if err != nil {
return nil, fmt.Errorf("cannot parse storage URL: %v", err)
}
logger.Infof("adding charm to state")
sch, err := conn.State.AddCharm(ch, curl, u, digest)
if err != nil {
return nil, fmt.Errorf("cannot add charm: %v", err)
}
return sch, nil
}
func (s *RepoSuite) AssertCharmUploaded(c *gc.C, curl *charm.URL) {
ch, err := s.State.Charm(curl)
c.Assert(err, gc.IsNil)
url := ch.BundleURL()
resp, err := http.Get(url.String())
c.Assert(err, gc.IsNil)
defer resp.Body.Close()
digest, _, err := utils.ReadSHA256(resp.Body)
c.Assert(err, gc.IsNil)
c.Assert(ch.BundleSha256(), gc.Equals, digest)
}
// download fetches the supplied charm and checks that it has the correct sha256
// hash, then copies it into the directory. If a value is received on abort, the
// download will be stopped.
func (d *BundlesDir) download(info BundleInfo, abort <-chan struct{}) (err error) {
archiveURL, disableSSLHostnameVerification, err := info.ArchiveURL()
if err != nil {
return err
}
defer errors.Maskf(&err, "failed to download charm %q from %q", info.URL(), archiveURL)
dir := d.downloadsPath()
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
aurl := archiveURL.String()
logger.Infof("downloading %s from %s", info.URL(), aurl)
if disableSSLHostnameVerification {
logger.Infof("SSL hostname verification disabled")
}
dl := downloader.New(aurl, dir, disableSSLHostnameVerification)
defer dl.Stop()
for {
select {
case <-abort:
logger.Infof("download aborted")
return fmt.Errorf("aborted")
case st := <-dl.Done():
if st.Err != nil {
return st.Err
}
logger.Infof("download complete")
defer st.File.Close()
actualSha256, _, err := utils.ReadSHA256(st.File)
if err != nil {
return err
}
archiveSha256, err := info.ArchiveSha256()
if err != nil {
return err
}
if actualSha256 != archiveSha256 {
return fmt.Errorf(
"expected sha256 %q, got %q", archiveSha256, actualSha256,
)
}
logger.Infof("download verified")
if err := os.MkdirAll(d.path, 0755); err != nil {
return err
}
return os.Rename(st.File.Name(), d.bundlePath(info))
}
}
}
// copyOneToolsPackage copies one tool from the source to the target.
func copyOneToolsPackage(tool *coretools.Tools, dest storage.Storage) error {
toolsName := envtools.StorageName(tool.Version)
logger.Infof("copying %v", toolsName)
resp, err := utils.GetValidatingHTTPClient().Get(tool.URL)
if err != nil {
return err
}
buf := &bytes.Buffer{}
srcFile := resp.Body
defer srcFile.Close()
tool.SHA256, tool.Size, err = utils.ReadSHA256(io.TeeReader(srcFile, buf))
if err != nil {
return err
}
sizeInKB := (tool.Size + 512) / 1024
logger.Infof("downloaded %v (%dkB), uploading", toolsName, sizeInKB)
logger.Infof("download %dkB, uploading", sizeInKB)
return dest.Put(toolsName, buf, tool.Size)
}
// NewMockStore creates a mock charm store containing the specified charms.
func NewMockStore(c *gc.C, charms map[string]int) *MockStore {
s := &MockStore{charms: charms, DefaultSeries: "precise"}
f, err := os.Open(testing.Charms.BundlePath(c.MkDir(), "dummy"))
c.Assert(err, gc.IsNil)
defer f.Close()
buf := &bytes.Buffer{}
s.bundleSha256, _, err = utils.ReadSHA256(io.TeeReader(f, buf))
c.Assert(err, gc.IsNil)
s.bundleBytes = buf.Bytes()
c.Assert(err, gc.IsNil)
s.mux = http.NewServeMux()
s.mux.HandleFunc("/charm-info", s.serveInfo)
s.mux.HandleFunc("/charm-event", s.serveEvent)
s.mux.HandleFunc("/charm/", s.serveCharm)
lis, err := net.Listen("tcp", "127.0.0.1:0")
c.Assert(err, gc.IsNil)
s.listener = lis
go http.Serve(s.listener, s)
return s
}
// AddCharm adds the given charm URL (which must include revision) to
// the environment, if it does not exist yet. Local charms are not
// supported, only charm store URLs. See also AddLocalCharm().
func (c *Client) AddCharm(args params.CharmURL) error {
charmURL, err := charm.ParseURL(args.URL)
if err != nil {
return err
}
if charmURL.Schema != "cs" {
return fmt.Errorf("only charm store charm URLs are supported, with cs: schema")
}
if charmURL.Revision < 0 {
return fmt.Errorf("charm URL must include revision")
}
// First, check if a pending or a real charm exists in state.
stateCharm, err := c.api.state.PrepareStoreCharmUpload(charmURL)
if err == nil && stateCharm.IsUploaded() {
// Charm already in state (it was uploaded already).
return nil
} else if err != nil {
return err
}
// Get the charm and its information from the store.
envConfig, err := c.api.state.EnvironConfig()
if err != nil {
return err
}
store := config.SpecializeCharmRepo(CharmStore, envConfig)
downloadedCharm, err := store.Get(charmURL)
if err != nil {
return errgo.Annotatef(err, "cannot download charm %q", charmURL.String())
}
// Open it and calculate the SHA256 hash.
downloadedBundle, ok := downloadedCharm.(*charm.Bundle)
if !ok {
return errgo.New("expected a charm archive, got %T", downloadedCharm)
}
archive, err := os.Open(downloadedBundle.Path)
if err != nil {
return errgo.Annotate(err, "cannot read downloaded charm")
}
defer archive.Close()
bundleSHA256, size, err := utils.ReadSHA256(archive)
if err != nil {
return errgo.Annotate(err, "cannot calculate SHA256 hash of charm")
}
if _, err := archive.Seek(0, 0); err != nil {
return errgo.Annotate(err, "cannot rewind charm archive")
}
// Get the environment storage and upload the charm.
env, err := environs.New(envConfig)
if err != nil {
return errgo.Annotate(err, "cannot access environment")
}
storage := env.Storage()
archiveName, err := CharmArchiveName(charmURL.Name, charmURL.Revision)
if err != nil {
return errgo.Annotate(err, "cannot generate charm archive name")
}
if err := storage.Put(archiveName, archive, size); err != nil {
return errgo.Annotate(err, "cannot upload charm to provider storage")
}
storageURL, err := storage.URL(archiveName)
if err != nil {
return errgo.Annotate(err, "cannot get storage URL for charm")
}
bundleURL, err := url.Parse(storageURL)
if err != nil {
return errgo.Annotate(err, "cannot parse storage URL")
}
// Finally, update the charm data in state and mark it as no longer pending.
_, err = c.api.state.UpdateUploadedCharm(downloadedCharm, charmURL, bundleURL, bundleSHA256)
if err == state.ErrCharmRevisionAlreadyModified ||
state.IsCharmAlreadyUploadedError(err) {
// This is not an error, it just signifies somebody else
// managed to upload and update the charm in state before
// us. This means we have to delete what we just uploaded
// to storage.
if err := storage.Remove(archiveName); err != nil {
errgo.Annotate(err, "cannot remove duplicated charm from storage")
}
return nil
}
return err
}
func (*utilsSuite) TestCommandString(c *gc.C) {
type test struct {
args []string
expected string
}
tests := []test{
{nil, ""},
{[]string{"a"}, "a"},
{[]string{"a$"}, `"a\$"`},
{[]string{""}, ""},
{[]string{"\\"}, `"\\"`},
{[]string{"a", "'b'"}, "a 'b'"},
{[]string{"a b"}, `"a b"`},
{[]string{"a", `"b"`}, `a "\"b\""`},
{[]string{"a", `"b\"`}, `a "\"b\\\""`},
{[]string{"a\n"}, "\"a\n\""},
}
for i, test := range tests {
c.Logf("test %d: %q", i, test.args)
result := utils.CommandString(test.args...)
c.Assert(result, gc.Equals, test.expected)
}
}
func (*utilsSuite) TestReadSHA256AndReadFileSHA256(c *gc.C) {
sha256Tests := []struct {
content string
sha256 string
}{{
content: "",
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
}, {
content: "some content",
sha256: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
}, {
content: "foo",
sha256: "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
}, {
content: "Foo",
sha256: "1cbec737f863e4922cee63cc2ebbfaafcd1cff8b790d8cfd2e6a5d550b648afa",
}, {
content: "multi\nline\ntext\nhere",
sha256: "c384f11c0294280792a44d9d6abb81f9fd991904cb7eb851a88311b04114231e",
}}
tempDir := c.MkDir()
for i, test := range sha256Tests {
c.Logf("test %d: %q -> %q", i, test.content, test.sha256)
buf := bytes.NewBufferString(test.content)
hash, size, err := utils.ReadSHA256(buf)
c.Check(err, gc.IsNil)
c.Check(hash, gc.Equals, test.sha256)
c.Check(int(size), gc.Equals, len(test.content))
tempFileName := filepath.Join(tempDir, fmt.Sprintf("sha256-%d", i))
err = ioutil.WriteFile(tempFileName, []byte(test.content), 0644)
c.Check(err, gc.IsNil)
fileHash, fileSize, err := utils.ReadFileSHA256(tempFileName)
c.Check(err, gc.IsNil)
c.Check(fileHash, gc.Equals, hash)
c.Check(fileSize, gc.Equals, size)
}
}
