managedstorage_test.go

// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package blobstore_test

import (
	"bytes"
	"crypto/md5"
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"math/rand"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	jujutxn "github.com/juju/txn"
	txntesting "github.com/juju/txn/testing"
	"github.com/juju/utils"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
	gc "launchpad.net/gocheck"

	"github.com/juju/blobstore"
)

var _ = gc.Suite(&managedStorageSuite{})

type managedStorageSuite struct {
	testing.IsolationSuite
	testing.MgoSuite
	txnRunner       jujutxn.Runner
	managedStorage  blobstore.ManagedStorage
	db              *mgo.Database
	resourceStorage blobstore.ResourceStorage
}

func (s *managedStorageSuite) SetUpSuite(c *gc.C) {
	s.IsolationSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *managedStorageSuite) TearDownSuite(c *gc.C) {
	s.MgoSuite.TearDownSuite(c)
	s.IsolationSuite.TearDownSuite(c)
}

func (s *managedStorageSuite) SetUpTest(c *gc.C) {
	s.IsolationSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
	s.db = s.Session.DB("blobstore")
	s.resourceStorage = blobstore.NewGridFS("storage", "test", s.Session)
	s.managedStorage = blobstore.NewManagedStorage(s.db, s.resourceStorage)

	// For testing, we need to ensure there's a single txnRunner for all operations.
	s.txnRunner = jujutxn.NewRunner(jujutxn.RunnerParams{Database: s.db})
	txnRunnerFunc := func(db *mgo.Database) jujutxn.Runner {
		return s.txnRunner
	}
	s.PatchValue(blobstore.TxnRunner, txnRunnerFunc)
}

func (s *managedStorageSuite) TearDownTest(c *gc.C) {
	s.MgoSuite.TearDownTest(c)
	s.IsolationSuite.TearDownTest(c)
}

func (s *managedStorageSuite) TestResourceStoragePath(c *gc.C) {
	for _, test := range []struct {
		envUUID     string
		user        string
		path        string
		storagePath string
		error       string
	}{
		{
			envUUID:     "",
			user:        "",
			path:        "/path/to/blob",
			storagePath: "global/path/to/blob",
		}, {
			envUUID:     "env",
			user:        "",
			path:        "/path/to/blob",
			storagePath: "environs/env/path/to/blob",
		}, {
			envUUID:     "",
			user:        "user",
			path:        "/path/to/blob",
			storagePath: "users/user/path/to/blob",
		}, {
			envUUID:     "env",
			user:        "user",
			path:        "/path/to/blob",
			storagePath: "environs/env/users/user/path/to/blob",
		}, {
			envUUID: "env/123",
			user:    "user",
			path:    "/path/to/blob",
			error:   `.* cannot contain "/"`,
		}, {
			envUUID: "env",
			user:    "user/123",
			path:    "/path/to/blob",
			error:   `.* cannot contain "/"`,
		},
	} {
		result, err := blobstore.ResourceStoragePath(s.managedStorage, test.envUUID, test.user, test.path)
		if test.error == "" {
			c.Check(err, gc.IsNil)
			c.Check(result, gc.Equals, test.storagePath)
		} else {
			c.Check(err, gc.ErrorMatches, test.error)
		}
	}
}

type managedResourceDocStub struct {
	Path       string
	ResourceId string
}

type resourceDocStub struct {
	Path string
}

func (s *managedStorageSuite) TestPendingUpload(c *gc.C) {
	// Manually set up a scenario where there's a resource recorded
	// but the upload has not occurred.
	rc := blobstore.GetResourceCatalog(s.managedStorage)
	rh := &blobstore.ResourceHash{"foo", "bar"}
	id, _, _, err := rc.Put(rh, 100)
	c.Assert(err, gc.IsNil)
	managedResource := blobstore.ManagedResource{
		EnvUUID: "env",
		User:    "user",
		Path:    "environs/env/path/to/blob",
	}
	_, err = blobstore.PutManagedResource(s.managedStorage, managedResource, id)
	c.Assert(err, gc.IsNil)
	_, _, err = s.managedStorage.GetForEnvironment("env", "/path/to/blob")
	c.Assert(err, gc.Equals, blobstore.ErrUploadPending)
}

func (s *managedStorageSuite) assertPut(c *gc.C, path string, blob []byte) string {
	// Put the data.
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", path, rdr, int64(len(blob)))
	c.Assert(err, gc.IsNil)

	// Load the managed resource record.
	var mrDoc managedResourceDocStub
	err = s.db.C("managedStoredResources").Find(bson.D{{"path", "environs/env" + path}}).One(&mrDoc)
	c.Assert(err, gc.IsNil)

	// Load the corresponding resource catalog record.
	var rd resourceDocStub
	err = s.db.C("storedResources").FindId(mrDoc.ResourceId).One(&rd)
	c.Assert(err, gc.IsNil)

	// Use the resource catalog record to load the underlying data from blobstore.
	r, err := s.resourceStorage.Get(rd.Path)
	c.Assert(err, gc.IsNil)
	data, err := ioutil.ReadAll(r)
	c.Assert(err, gc.IsNil)
	c.Assert(data, gc.DeepEquals, blob)
	return rd.Path
}

func (s *managedStorageSuite) assertResourceCatalogCount(c *gc.C, expected int) {
	num, err := s.db.C("storedResources").Count()
	c.Assert(err, gc.IsNil)
	c.Assert(num, gc.Equals, expected)
}

func (s *managedStorageSuite) TestPut(c *gc.C) {
	s.assertPut(c, "/path/to/blob", []byte("some resource"))
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutSamePathDifferentData(c *gc.C) {
	resPath := s.assertPut(c, "/path/to/blob", []byte("some resource"))
	secondResPath := s.assertPut(c, "/path/to/blob", []byte("another resource"))
	c.Assert(resPath, gc.Not(gc.Equals), secondResPath)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutDifferentPathSameData(c *gc.C) {
	resPath := s.assertPut(c, "/path/to/blob", []byte("some resource"))
	secondResPath := s.assertPut(c, "/anotherpath/to/blob", []byte("some resource"))
	c.Assert(resPath, gc.Equals, secondResPath)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutSamePathDifferentDataMulti(c *gc.C) {
	resPath := s.assertPut(c, "/path/to/blob", []byte("another resource"))
	secondResPath := s.assertPut(c, "/anotherpath/to/blob", []byte("some resource"))
	c.Assert(resPath, gc.Not(gc.Equals), secondResPath)
	s.assertResourceCatalogCount(c, 2)

	thirdResPath := s.assertPut(c, "/path/to/blob", []byte("some resource"))
	c.Assert(resPath, gc.Not(gc.Equals), secondResPath)
	c.Assert(secondResPath, gc.Equals, thirdResPath)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutManagedResourceFail(c *gc.C) {
	var resourcePath string
	s.PatchValue(blobstore.PutResourceTxn, func(
		coll *mgo.Collection, managedResource blobstore.ManagedResource, resourceId string) (string, []txn.Op, error) {
		rc := blobstore.GetResourceCatalog(s.managedStorage)
		r, err := rc.Get(resourceId)
		c.Assert(err, gc.IsNil)
		resourcePath = r.Path
		return "", nil, errors.Errorf("some error")
	})
	// Attempt to put the data.
	blob := []byte("data")
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", "/some/path", rdr, int64(len(blob)))
	c.Assert(err, gc.ErrorMatches, "cannot update managed resource catalog: some error")

	// Now ensure there's no blob data left behind in storage, nor a resource catalog record.
	s.assertResourceCatalogCount(c, 0)
	_, err = s.resourceStorage.Get(resourcePath)
	c.Assert(err, gc.ErrorMatches, ".*not found")
}

func (s *managedStorageSuite) assertGet(c *gc.C, path string, blob []byte) {
	r, length, err := s.managedStorage.GetForEnvironment("env", path)
	c.Assert(err, gc.IsNil)
	defer r.Close()
	data, err := ioutil.ReadAll(r)
	c.Assert(err, gc.IsNil)
	c.Assert(data, gc.DeepEquals, blob)
	c.Assert(int(length), gc.Equals, len(blob))
}

func (s *managedStorageSuite) TestGet(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	s.assertGet(c, "/path/to/blob", blob)
}

func (s *managedStorageSuite) TestGetNonExistent(c *gc.C) {
	_, _, err := s.managedStorage.GetForEnvironment("env", "/path/to/nowhere")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *managedStorageSuite) TestRemove(c *gc.C) {
	blob := []byte("some resource")
	resPath := s.assertPut(c, "/path/to/blob", blob)
	err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
	c.Assert(err, gc.IsNil)

	// Check the data and catalog entry really are removed.
	_, _, err = s.managedStorage.GetForEnvironment("env", "path/to/blob")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
	_, err = s.resourceStorage.Get(resPath)
	c.Assert(err, gc.NotNil)

	s.assertResourceCatalogCount(c, 0)
}

func (s *managedStorageSuite) TestRemoveNonExistent(c *gc.C) {
	err := s.managedStorage.RemoveForEnvironment("env", "/path/to/nowhere")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *managedStorageSuite) TestRemoveDifferentPathKeepsData(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	s.assertPut(c, "/anotherpath/to/blob", blob)
	s.assertResourceCatalogCount(c, 1)
	err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
	c.Assert(err, gc.IsNil)
	s.assertGet(c, "/anotherpath/to/blob", blob)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutRace(c *gc.C) {
	blob := []byte("some resource")
	beforeFunc := func() {
		s.assertPut(c, "/path/to/blob", blob)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	anotherblob := []byte("another resource")
	s.assertPut(c, "/path/to/blob", anotherblob)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutDeleteRace(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	beforeFunc := func() {
		err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
		c.Assert(err, gc.IsNil)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	anotherblob := []byte("another resource")
	s.assertPut(c, "/path/to/blob", anotherblob)
	s.assertResourceCatalogCount(c, 1)
}

func (s *managedStorageSuite) TestPutRaceWhereCatalogEntryRemoved(c *gc.C) {
	blob := []byte("some resource")
	// Remove the resource catalog entry with the resourceId that we are about
	// to write to a managed resource entry.
	beforeFunc := []func(){
		nil, //  resourceCatalog Put()
		nil, //  managedResource Put()
		func() {
			// Shamelessly exploit our knowledge of how ids are made.
			md5hash, sha256hash := calculateCheckSums(c, 0, int64(len(blob)), blob)
			_, _, err := blobstore.GetResourceCatalog(s.managedStorage).Remove(md5hash + sha256hash)
			c.Assert(err, gc.IsNil)
		},
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc...).Check()
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", "/path/to/blob", rdr, int64(len(blob)))
	c.Assert(err, gc.ErrorMatches, "unexpected deletion .*")
	s.assertResourceCatalogCount(c, 0)
}

func (s *managedStorageSuite) TestRemoveRace(c *gc.C) {
	blob := []byte("some resource")
	s.assertPut(c, "/path/to/blob", blob)
	beforeFunc := func() {
		err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
		c.Assert(err, gc.IsNil)
	}
	defer txntesting.SetBeforeHooks(c, s.txnRunner, beforeFunc).Check()
	err := s.managedStorage.RemoveForEnvironment("env", "/path/to/blob")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
	_, _, err = s.managedStorage.GetForEnvironment("env", "/path/to/blob")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *managedStorageSuite) TestPutRequestNotFound(c *gc.C) {
	hash := blobstore.ResourceHash{"md5", "sha256"}
	_, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *managedStorageSuite) putTestBlob(c *gc.C, path string) (blob []byte, md5hashHex, sha256hashHex string) {
	id := bson.NewObjectId().Hex()
	blob = []byte(id)
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", path, rdr, int64(len(blob)))
	c.Assert(err, gc.IsNil)
	s.assertGet(c, path, blob)
	md5hashHex, sha256hashHex = calculateCheckSums(c, 0, int64(len(blob)), blob)
	return blob, md5hashHex, sha256hashHex
}

func calculateCheckSums(c *gc.C, start, length int64, blob []byte) (md5hashHex, sha256hashHex string) {
	data := blob[start : start+length]
	sha256hash := sha256.New()
	_, err := sha256hash.Write(data)
	c.Assert(err, gc.IsNil)
	md5hash := md5.New()
	_, err = md5hash.Write(data)
	c.Assert(err, gc.IsNil)
	md5hashHex = fmt.Sprintf("%x", md5hash.Sum(nil))
	sha256hashHex = fmt.Sprintf("%x", sha256hash.Sum(nil))
	return md5hashHex, sha256hashHex
}

func (s *managedStorageSuite) TestPutRequestResponseMD5Mismatch(c *gc.C) {
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	_, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	response := blobstore.NewPutResponse(reqResp.RequestId, "notmd5", sha256Response)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrResponseMismatch)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) TestPutRequestResponseSHA256Mismatch(c *gc.C) {
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	md5Response, _ := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, "notsha256")
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrResponseMismatch)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) assertPutRequestSingle(c *gc.C, blob []byte, resourceCount int) {
	if blob == nil {
		id := bson.NewObjectId().Hex()
		blob = []byte(id)
	}
	rdr := bytes.NewReader(blob)
	err := s.managedStorage.PutForEnvironment("env", "path/to/blob", rdr, int64(len(blob)))
	c.Assert(err, gc.IsNil)
	md5hash, sha256hash := calculateCheckSums(c, 0, int64(len(blob)), blob)
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, sha256Response)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.IsNil)
	s.assertGet(c, "path/to/blob", blob)
	s.assertResourceCatalogCount(c, resourceCount)
}

var trigger struct{} = struct{}{}

// patchedAfterFunc returns a function like time.AfterFunc, but is triggered on a channel
// select rather than the expiry of a timer interval.
func patchedAfterFunc(ch chan struct{}) func(d time.Duration, f func()) *time.Timer {
	return func(d time.Duration, f func()) *time.Timer {
		go func() {
			select {
			case <-ch:
				f()
				ch <- trigger
			}
		}()
		return nil
	}
}

func (s *managedStorageSuite) TestPutRequestSingle(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	s.assertPutRequestSingle(c, nil, 1)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) TestPutRequestLarge(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	// Use a blob size of 4096 which is greater than max range of put response range length.
	blob := make([]byte, 4096)
	for i := 0; i < 4096; i++ {
		blob[i] = byte(rand.Intn(255))
	}
	s.assertPutRequestSingle(c, blob, 1)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) TestPutRequestMultiSequential(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	s.assertPutRequestSingle(c, nil, 1)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	s.assertPutRequestSingle(c, nil, 1)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) checkPutResponse(c *gc.C, index int, wg *sync.WaitGroup,
	requestId int64, md5hash, sha256hash string, blob []byte) {

	// After a random time, respond to a previously queued put request and check the result.
	go func() {
		delay := rand.Intn(3)
		time.Sleep(time.Duration(delay) * time.Millisecond)
		expectError := index == 2
		if expectError {
			md5hash = "bad"
		}
		response := blobstore.NewPutResponse(requestId, md5hash, sha256hash)
		err := s.managedStorage.ProofOfAccessResponse(response)
		if expectError {
			c.Check(err, gc.NotNil)
		} else {
			c.Check(err, gc.IsNil)
			if err == nil {
				r, length, err := s.managedStorage.GetForEnvironment("env", fmt.Sprintf("path/to/blob%d", index))
				c.Check(err, gc.IsNil)
				if err == nil {
					data, err := ioutil.ReadAll(r)
					c.Check(err, gc.IsNil)
					c.Check(data, gc.DeepEquals, blob)
					c.Check(int(length), gc.DeepEquals, len(blob))
				}
			}
		}
		wg.Done()
	}()
}

func (s *managedStorageSuite) queuePutRequests(c *gc.C, done chan struct{}) {
	var wg sync.WaitGroup
	// One request is allowed to expire so set up wait group for 1 less than number of requests.
	wg.Add(9)
	go func() {
		for i := 0; i < 10; i++ {
			blobPath := fmt.Sprintf("path/to/blob%d", i)
			blob, md5hash, sha256hash := s.putTestBlob(c, blobPath)
			hash := blobstore.ResourceHash{md5hash, sha256hash}
			reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
			c.Assert(err, gc.IsNil)
			// Let one request timeout
			if i == 3 {
				continue
			}
			md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
			s.checkPutResponse(c, i, &wg, reqResp.RequestId, md5Response, sha256Response, blob)
		}
		wg.Wait()
		close(done)
	}()
}

const (
	ShortWait = 50 * time.Millisecond
	LongWait  = 10 * time.Second
)

var LongAttempt = &utils.AttemptStrategy{
	Total: LongWait,
	Delay: ShortWait,
}

func (s *managedStorageSuite) TestPutRequestMultiRandom(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	done := make(chan struct{})
	s.queuePutRequests(c, done)
	select {
	case <-done:
		c.Logf("all done")
	case <-time.After(LongWait):
		c.Fatalf("timed out waiting for put requests to be processed")
	}
	// One request hasn't been processed since we left it to timeout.
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 1)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) TestPutRequestExpired(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	// Trigger the request timeout.
	ch <- trigger
	<-ch
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, sha256Response)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrRequestExpired)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

// Run one simple test with the real time.AfterFunc to ensure it works.
func (s *managedStorageSuite) TestPutRequestExpiredWithRealTimeAfter(c *gc.C) {
	s.PatchValue(blobstore.RequestExpiry, 5*time.Millisecond)
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	// Wait for request timer to trigger.
	time.Sleep(7 * time.Millisecond)
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, sha256Response)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrRequestExpired)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
}

func (s *managedStorageSuite) TestPutRequestExpiredMulti(c *gc.C) {
	ch := make(chan struct{})
	s.PatchValue(blobstore.AfterFunc, patchedAfterFunc(ch))
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	reqResp2, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob2", hash)
	c.Assert(err, gc.IsNil)
	md5Response2, sha256Response2 := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	// Trigger the request timeouts.
	ch <- trigger
	<-ch
	ch <- trigger
	<-ch
	c.Assert(blobstore.RequestQueueLength(s.managedStorage), gc.Equals, 0)
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, sha256Response)
	response2 := blobstore.NewPutResponse(reqResp2.RequestId, md5Response2, sha256Response2)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrRequestExpired)
	err = s.managedStorage.ProofOfAccessResponse(response2)
	c.Assert(err, gc.Equals, blobstore.ErrRequestExpired)
}

func (s *managedStorageSuite) TestPutRequestDeleted(c *gc.C) {
	blob, md5hash, sha256hash := s.putTestBlob(c, "path/to/blob")
	hash := blobstore.ResourceHash{md5hash, sha256hash}
	reqResp, err := s.managedStorage.PutForEnvironmentRequest("env", "path/to/blob", hash)
	c.Assert(err, gc.IsNil)
	err = s.managedStorage.RemoveForEnvironment("env", "path/to/blob")
	c.Assert(err, gc.IsNil)

	md5Response, sha256Response := calculateCheckSums(c, reqResp.RangeStart, reqResp.RangeLength, blob)
	response := blobstore.NewPutResponse(reqResp.RequestId, md5Response, sha256Response)
	err = s.managedStorage.ProofOfAccessResponse(response)
	c.Assert(err, gc.Equals, blobstore.ErrResourceDeleted)
}