func (s *ImageSuite) TestAddImageExcessiveContention(c *gc.C) {
metadata := []*imagestorage.Metadata{
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path"},
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path"},
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "2", SourceURL: "http://path"},
{ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "3", SourceURL: "http://path"},
}
i := 1
addMetadata := func() {
err := s.storage.AddImage(strings.NewReader(metadata[i].SHA256), metadata[i])
c.Assert(err, gc.IsNil)
i++
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata, addMetadata, addMetadata).Check()
err := s.storage.AddImage(strings.NewReader(metadata[0].SHA256), metadata[0])
c.Assert(err, gc.ErrorMatches, "cannot store image metadata: state changing too quickly; try again soon")
// There should be no blobs apart from the last one added by the before-hook.
for _, metadata := range metadata[:3] {
path := fmt.Sprintf("images/%s-%s-%s:%s", metadata.Kind, metadata.Series, metadata.Arch, metadata.SHA256)
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
_, _, err = managedStorage.GetForBucket("my-uuid", path)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
s.assertImage(c, metadata[3], "3")
}
func (s *ClientSimpleRaceSuite) TestNewClient_WorksDespite_CreateClockRace(c *gc.C) {
config := func(id string) lease.ClientConfig {
return lease.ClientConfig{
Id: id,
Namespace: "ns",
Collection: "leases",
Mongo: NewMongo(s.db),
Clock: lease.SystemClock{},
}
}
sutConfig := config("sut")
sutRunner := sutConfig.Mongo.(*Mongo).runner
// Set up a hook to create the clock doc (and write some important data to
// it) by creating another client before the SUT gets a chance.
defer txntesting.SetBeforeHooks(c, sutRunner, func() {
client, err := lease.NewClient(config("blocker"))
c.Check(err, jc.ErrorIsNil)
err = client.ClaimLease("somewhere", lease.Request{"someone", time.Minute})
c.Check(err, jc.ErrorIsNil)
})()
// Create a client against an apparently-empty namespace.
client, err := lease.NewClient(sutConfig)
c.Check(err, jc.ErrorIsNil)
// Despite the scramble, it's generated with recent lease data and no error.
leases := client.Leases()
info, found := leases["somewhere"]
c.Check(found, jc.IsTrue)
c.Check(info.Holder, gc.Equals, "someone")
}
func (s *ToolsSuite) TestAddToolsExcessiveContention(c *gc.C) {
metadata := []toolstorage.Metadata{
{Version: version.Current, Size: 1, SHA256: "0"},
{Version: version.Current, Size: 1, SHA256: "1"},
{Version: version.Current, Size: 1, SHA256: "2"},
{Version: version.Current, Size: 1, SHA256: "3"},
}
i := 1
addMetadata := func() {
err := s.storage.AddTools(strings.NewReader(metadata[i].SHA256), metadata[i])
c.Assert(err, jc.ErrorIsNil)
i++
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata, addMetadata, addMetadata).Check()
err := s.storage.AddTools(strings.NewReader(metadata[0].SHA256), metadata[0])
c.Assert(err, gc.ErrorMatches, "cannot store tools metadata: state changing too quickly; try again soon")
// There should be no blobs apart from the last one added by the before-hook.
for _, metadata := range metadata[:3] {
path := fmt.Sprintf("tools/%s-%s", metadata.Version, metadata.SHA256)
_, _, err = s.managedStorage.GetForEnvironment("my-uuid", path)
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
s.assertTools(c, metadata[3], "3")
}
func (s *ImageSuite) TestAddImageConcurrent(c *gc.C) {
metadata0 := &imagestorage.Metadata{
ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "0", SourceURL: "http://path",
}
metadata1 := &imagestorage.Metadata{
ModelUUID: "my-uuid", Kind: "lxc", Series: "trusty", Arch: "amd64", Size: 1, SHA256: "1", SourceURL: "http://path",
}
addMetadata := func() {
err := s.storage.AddImage(strings.NewReader("0"), metadata0)
c.Assert(err, gc.IsNil)
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
r, _, err := managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
c.Assert(err, gc.IsNil)
r.Close()
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata).Check()
err := s.storage.AddImage(strings.NewReader("1"), metadata1)
c.Assert(err, gc.IsNil)
// Blob added in before-hook should be removed.
managedStorage := imagestorage.ManagedStorage(s.storage, s.session)
_, _, err = managedStorage.GetForBucket("my-uuid", "images/lxc-trusty-amd64:0")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
s.assertImage(c, metadata1, "1")
}
func (s *txnSuite) TestBeforeHooks(c *gc.C) {
s.insertDoc(c, "1", "Simple")
changeFuncs := []func(){
func() { s.setDocName(c, "1", "FooBar") },
func() { s.setDocName(c, "1", "Foo") },
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, changeFuncs...).Check()
maxAttempt := 0
buildTxn := func(attempt int) ([]txn.Op, error) {
maxAttempt = attempt
ops := []txn.Op{{
C: s.collection.Name,
Id: "1",
Assert: bson.D{{"name", "Foo"}},
Update: bson.D{{"$set", bson.D{{"name", "Bar"}}}},
}}
return ops, nil
}
err := s.txnRunner.Run(buildTxn)
c.Assert(err, gc.IsNil)
var found simpleDoc
err = s.collection.FindId("1").One(&found)
c.Assert(err, gc.IsNil)
c.Assert(maxAttempt, gc.Equals, 1)
doc := simpleDoc{"1", "Bar"}
c.Assert(found, gc.DeepEquals, doc)
}
func (s *resourceCatalogSuite) TestPutDeletedResourceRace(c *gc.C) {
firstId, _ := s.assertPut(c, true, "md5foo", "sha256foo")
err := s.rCatalog.UploadComplete(firstId)
c.Assert(err, gc.IsNil)
beforeFuncs := []func(){
func() {
_, _, err := s.rCatalog.Remove(firstId)
c.Assert(err, gc.IsNil)
},
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
rh := &blobstore.ResourceHash{"md5foo", "sha256foo"}
id, _, isNew, err := s.rCatalog.Put(rh, 200)
c.Assert(err, gc.IsNil)
c.Assert(isNew, jc.IsTrue)
c.Assert(firstId, gc.Equals, id)
err = s.rCatalog.UploadComplete(id)
c.Assert(err, gc.IsNil)
r, err := s.rCatalog.Get(id)
c.Assert(err, gc.IsNil)
s.assertRefCount(c, id, 1)
c.Assert(r.MD5Hash, gc.Equals, "md5foo")
c.Assert(r.SHA256Hash, gc.Equals, "sha256foo")
c.Assert(r.Length, gc.Equals, int64(200))
}
func (s *cloudImageMetadataSuite) assertConcurrentDelete(c *gc.C, imageId0, imageId1 string) {
deleteMetadata := func() {
s.assertDeleteMetadata(c, imageId0)
}
defer txntesting.SetBeforeHooks(c, s.access.runner, deleteMetadata).Check()
s.assertDeleteMetadata(c, imageId1)
s.assertNoMetadata(c)
}
func (s *cloudImageMetadataSuite) assertConcurrentSave(c *gc.C, metadata0, metadata1 cloudimagemetadata.Metadata, expected ...cloudimagemetadata.Metadata) {
addMetadata := func() {
s.assertRecordMetadata(c, metadata0)
}
defer txntesting.SetBeforeHooks(c, s.access.runner, addMetadata).Check()
s.assertRecordMetadata(c, metadata1)
s.assertMetadataRecorded(c, cloudimagemetadata.MetadataAttributes{}, expected...)
}
func (s *managedStorageSuite) TestPutRace(c *gc.C) {
blob := []byte("some resource")
beforeFunc := func() {
s.assertPut(c, "/path/to/blob", blob)
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
anotherblob := []byte("another resource")
s.assertPut(c, "/path/to/blob", anotherblob)
s.assertResourceCatalogCount(c, 1)
}
func (s *resourceCatalogSuite) TestUploadCompleteDeleted(c *gc.C) {
id, _, err := s.rCatalog.Put("sha384foo", 100)
c.Assert(err, gc.IsNil)
remove := func() {
_, _, err := s.rCatalog.Remove(id)
c.Assert(err, gc.IsNil)
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, remove).Check()
err = s.rCatalog.UploadComplete(id, "wherever")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
func (s *managedStorageSuite) TestPutDeleteRace(c *gc.C) {
blob := []byte("some resource")
s.assertPut(c, "/path/to/blob", blob)
beforeFunc := func() {
err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
c.Assert(err, gc.IsNil)
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
anotherblob := []byte("another resource")
s.assertPut(c, "/path/to/blob", anotherblob)
s.assertResourceCatalogCount(c, 1)
}
func (s *managedStorageSuite) TestRemoveRace(c *gc.C) {
blob := []byte("some resource")
s.assertPut(c, "/path/to/blob", blob)
beforeFunc := func() {
err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
c.Assert(err, gc.IsNil)
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
_, _, err = s.managedStorage.GetForEnvironment("env", "/path/to/blob")
c.Assert(err, jc.Satisfies, errors.IsNotFound)
}
func (s *resourceCatalogSuite) TestDeleteResourceRace(c *gc.C) {
id, _ := s.assertPut(c, true, "md5foo", "sha256foo")
s.assertPut(c, false, "md5foo", "sha256foo")
beforeFuncs := []func(){
func() {
_, _, err := s.rCatalog.Remove(id)
c.Assert(err, gc.IsNil)
},
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
_, _, err := s.rCatalog.Remove(id)
c.Assert(err, gc.IsNil)
_, err = s.rCatalog.Get(id)
c.Assert(err, gc.ErrorMatches, `resource with id ".*" not found`)
}
func (s *ClientTrickyRaceSuite) TestExtendLease_BlockedBy_ExpireLease(c *gc.C) {
// Set up a hook to expire the lease before the extend gets a chance.
defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
s.blocker.Clock.Advance(90 * time.Second)
err := s.blocker.Client.ExpireLease("name")
c.Check(err, jc.ErrorIsNil)
})()
// Try to extend; check it aborts.
err := s.sut.Client.ExtendLease("name", lease.Request{"holder", 2 * time.Minute})
c.Check(err, gc.Equals, lease.ErrInvalid)
// The SUT has been refreshed, and you can see why the operation was invalid.
c.Check("name", s.sut.Holder(), "")
}
func (s *resourceCatalogSuite) TestPutNewResourceRace(c *gc.C) {
var firstId string
beforeFuncs := []func(){
func() { firstId, _ = s.assertPut(c, true, "sha384foo") },
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
id, _, err := s.rCatalog.Put("sha384foo", 200)
c.Assert(err, gc.IsNil)
c.Assert(id, gc.Equals, firstId)
err = s.rCatalog.UploadComplete(id, "wherever")
c.Assert(err, gc.IsNil)
r, err := s.rCatalog.Get(id)
c.Assert(err, gc.IsNil)
s.assertRefCount(c, id, 2)
c.Assert(r.SHA384Hash, gc.Equals, "sha384foo")
c.Assert(int(r.Length), gc.Equals, 200)
}
func (s *ClientTrickyRaceSuite) TestExtendLease_WorksDespite_LongerExtendLease(c *gc.C) {
shorterRequest := 90 * time.Second
longerRequest := 120 * time.Second
// Set up hooks to extend the lease by a lot, before the SUT's extend can.
defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
err := s.blocker.Client.ExtendLease("name", lease.Request{"holder", longerRequest})
c.Check(err, jc.ErrorIsNil)
})()
// Extend the lease by a little.
err := s.sut.Client.ExtendLease("name", lease.Request{"holder", shorterRequest})
c.Check(err, jc.ErrorIsNil)
// The SUT was refreshed, and knows that the lease is really valid for longer.
c.Check("name", s.sut.Expiry(), s.sut.Zero.Add(longerRequest))
}
func (s *ClientSimpleRaceSuite) TestClaimLease_BlockedBy_ClaimLease(c *gc.C) {
sut := s.EasyFixture(c)
blocker := s.NewFixture(c, FixtureParams{Id: "blocker"})
// Set up a hook to grab the lease "name" just before the next txn runs.
defer txntesting.SetBeforeHooks(c, sut.Runner, func() {
err := blocker.Client.ClaimLease("name", lease.Request{"ha-haa", time.Minute})
c.Check(err, jc.ErrorIsNil)
})()
// Try to grab the lease "name", and fail.
err := sut.Client.ClaimLease("name", lease.Request{"trying", time.Second})
c.Check(err, gc.Equals, lease.ErrInvalid)
// The client that failed has refreshed state (as it had to, in order
// to discover the reason for the invalidity).
c.Check("name", sut.Holder(), "ha-haa")
c.Check("name", sut.Expiry(), sut.Zero.Add(time.Minute))
}
func (s *ClientTrickyRaceSuite) TestExpireLease_BlockedBy_ExpireThenReclaim(c *gc.C) {
// Set up a hook to expire the lease and then reclaim it.
defer txntesting.SetBeforeHooks(c, s.sut.Runner, func() {
s.blocker.Clock.Advance(90 * time.Second)
err := s.blocker.Client.ExpireLease("name")
c.Check(err, jc.ErrorIsNil)
err = s.blocker.Client.ClaimLease("name", lease.Request{"holder", time.Minute})
c.Check(err, jc.ErrorIsNil)
})()
// Try to expire; check it aborts.
s.sut.Clock.Advance(90 * time.Second)
err := s.sut.Client.ExpireLease("name")
c.Check(err, gc.Equals, lease.ErrInvalid)
// The SUT has been refreshed, and you can see why the operation was invalid.
c.Check("name", s.sut.Expiry(), s.sut.Zero.Add(150*time.Second))
}
func (s *resourceCatalogSuite) TestPutNewResourceRace(c *gc.C) {
var firstId string
beforeFuncs := []func(){
func() { firstId, _ = s.assertPut(c, true, "md5foo", "sha256foo") },
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFuncs...).Check()
rh := &blobstore.ResourceHash{"md5foo", "sha256foo"}
id, _, isNew, err := s.rCatalog.Put(rh, 200)
c.Assert(err, gc.IsNil)
c.Assert(id, gc.Equals, firstId)
c.Assert(isNew, jc.IsFalse)
err = s.rCatalog.UploadComplete(id)
c.Assert(err, gc.IsNil)
r, err := s.rCatalog.Get(id)
c.Assert(err, gc.IsNil)
s.assertRefCount(c, id, 2)
c.Assert(r.MD5Hash, gc.Equals, "md5foo")
c.Assert(r.SHA256Hash, gc.Equals, "sha256foo")
c.Assert(int(r.Length), gc.Equals, 200)
}
func (s *managedStorageSuite) TestPutRaceWhereCatalogEntryRemoved(c *gc.C) {
blob := []byte("some resource")
// Remove the resource catalog entry with the resourceId that we are about
// to write to a managed resource entry.
beforeFunc := []func(){
nil, // resourceCatalog Put()
nil, // managedResource Put()
func() {
// Shamelessly exploit our knowledge of how ids are made.
sha384Hash := calculateCheckSum(c, 0, int64(len(blob)), blob)
_, _, err := blobstore.GetResourceCatalog(s.managedStorage).Remove(sha384Hash)
c.Assert(err, gc.IsNil)
},
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc...).Check()
rdr := bytes.NewReader(blob)
err := s.managedStorage.PutForEnvironment("env", "/path/to/blob", rdr, int64(len(blob)))
c.Assert(err, gc.ErrorMatches, "unexpected deletion .*")
s.assertResourceCatalogCount(c, 0)
}
func (s *ToolsSuite) TestAddToolsConcurrent(c *gc.C) {
metadata0 := toolstorage.Metadata{Version: version.Current, Size: 1, SHA256: "0"}
metadata1 := toolstorage.Metadata{Version: version.Current, Size: 1, SHA256: "1"}
addMetadata := func() {
err := s.storage.AddTools(strings.NewReader("0"), metadata0)
c.Assert(err, jc.ErrorIsNil)
r, _, err := s.managedStorage.GetForEnvironment("my-uuid", fmt.Sprintf("tools/%s-0", version.Current))
c.Assert(err, jc.ErrorIsNil)
r.Close()
}
defer txntesting.SetBeforeHooks(c, s.txnRunner, addMetadata).Check()
err := s.storage.AddTools(strings.NewReader("1"), metadata1)
c.Assert(err, jc.ErrorIsNil)
// Blob added in before-hook should be removed.
_, _, err = s.managedStorage.GetForEnvironment("my-uuid", fmt.Sprintf("tools/%s-0", version.Current))
c.Assert(err, jc.Satisfies, errors.IsNotFound)
s.assertTools(c, metadata1, "1")
}
func SetBeforeHooks(c *gc.C, st *State, fs ...func()) txntesting.TransactionChecker {
return txntesting.SetBeforeHooks(c, st.transactionRunner, fs...)
}
func SetBeforeHooks(c *gc.C, st *State, fs ...func()) txntesting.TransactionChecker {
runner := jujutxn.NewRunner(jujutxn.RunnerParams{Database: st.db})
st.transactionRunner = runner
return txntesting.SetBeforeHooks(c, runner, fs...)
}
func SetBeforeHooks(c *gc.C, st *State, fs ...func()) txntesting.TransactionChecker {
return txntesting.SetBeforeHooks(c, newRunnerForHooks(st), fs...)
}