Esempio n. 1
0
func (t *ConnTest) BucketsAreSegregatedByName() {
	const objName = "baz"
	var contents []byte
	var err error

	b0, err := t.conn.OpenBucket(t.ctx, "foo")
	AssertEq(nil, err)

	b1, err := t.conn.OpenBucket(t.ctx, "bar")
	AssertEq(nil, err)

	// Add an object with the same name but different contents to each of two
	// buckets.
	_, err = gcsutil.CreateObject(t.ctx, b0, objName, []byte("taco"))
	AssertEq(nil, err)

	_, err = gcsutil.CreateObject(t.ctx, b1, objName, []byte("burrito"))
	AssertEq(nil, err)

	// Each should have stored it independently.
	contents, err = gcsutil.ReadObject(t.ctx, b0, objName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	contents, err = gcsutil.ReadObject(t.ctx, b1, objName)
	AssertEq(nil, err)
	ExpectEq("burrito", string(contents))
}
Esempio n. 2
0
func (t *PrefixBucketTest) CopyObject() {
	var err error
	suffix := "taco"
	name := t.prefix + suffix
	contents := "foobar"

	// Create an object through the back door.
	_, err = gcsutil.CreateObject(t.ctx, t.wrapped, name, []byte(contents))
	AssertEq(nil, err)

	// Copy it to a new name.
	newSuffix := "burrito"
	o, err := t.bucket.CopyObject(
		t.ctx,
		&gcs.CopyObjectRequest{
			SrcName: suffix,
			DstName: newSuffix,
		})

	AssertEq(nil, err)
	ExpectEq(newSuffix, o.Name)

	// Read it through the back door.
	actual, err := gcsutil.ReadObject(t.ctx, t.wrapped, t.prefix+newSuffix)
	AssertEq(nil, err)
	ExpectEq(contents, string(actual))
}
Esempio n. 3
0
func (t *ConnTest) BucketContentsAreStable() {
	const bucketName = "foo"
	const objName = "bar"

	var err error

	// Open the bucket.
	bucket, err := t.conn.OpenBucket(t.ctx, bucketName)
	AssertEq(nil, err)

	// Add an object to a bucket.
	_, err = gcsutil.CreateObject(
		t.ctx,
		bucket,
		objName,
		[]byte("taco"))

	AssertEq(nil, err)

	// Grab the bucket again. It should still be there.
	contents, err := gcsutil.ReadObject(
		t.ctx,
		bucket,
		objName)

	AssertEq(nil, err)
	ExpectEq("taco", string(contents))
}
Esempio n. 4
0
func (t *MountTest) BasicUsage() {
	var err error
	const fileName = "foo"

	// Grab a bucket.
	bucket, err := t.conn.OpenBucket(t.ctx, "some_bucket")
	AssertEq(nil, err)

	// Mount that bucket.
	mfs, err := t.mount(bucket.Name(), t.dir)
	AssertEq(nil, err)

	// Create a file.
	err = ioutil.WriteFile(path.Join(t.dir, fileName), []byte("taco"), 0400)
	AssertEq(nil, err)

	// Read the object from the bucket.
	contents, err := gcsutil.ReadObject(t.ctx, bucket, fileName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Read the file.
	contents, err = ioutil.ReadFile(path.Join(t.dir, fileName))
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Unmount and join.
	err = t.unmount()
	AssertEq(nil, err)

	err = mfs.Join(t.ctx)
	AssertEq(nil, err)
}
Esempio n. 5
0
func (t *IntegrationTest) TruncateThenSync() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Truncate.
	err = t.mc.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Sync should save out the new generation.
	rl, newObj, err := t.sync(o)
	AssertEq(nil, err)

	ExpectNe(o.Generation, newObj.Generation)
	ExpectEq(t.objectGeneration("foo"), newObj.Generation)

	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, "foo")
	AssertEq(nil, err)
	ExpectEq("ta", string(contents))

	// Read via the lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	contents, err = ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq("ta", string(contents))
}
Esempio n. 6
0
func (t *IntegrationTest) TruncateThenSync() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", []byte("taco"))
	AssertEq(nil, err)

	t.create(o)

	// Truncate.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()
	err = t.tf.Truncate(2)
	t.clock.AdvanceTime(time.Second)

	AssertEq(nil, err)

	// Sync should save out the new generation.
	newObj, err := t.sync(o)
	AssertEq(nil, err)

	ExpectNe(o.Generation, newObj.Generation)
	ExpectEq(t.objectGeneration("foo"), newObj.Generation)
	ExpectEq(
		truncateTime.UTC().Format(time.RFC3339Nano),
		newObj.Metadata["gcsfuse_mtime"])

	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, "foo")
	AssertEq(nil, err)
	ExpectEq("ta", string(contents))
}
Esempio n. 7
0
func (t *IntegrationTest) WithinLeaserLimit() {
	AssertLt(len("taco"), fileLeaserLimitBytes)

	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Extend to be up against the leaser limit, then write out to GCS, which
	// should downgrade to a read lease.
	err = t.mc.Truncate(t.ctx, fileLeaserLimitBytes)
	AssertEq(nil, err)

	rl, _, err := t.sync(o)
	AssertEq(nil, err)

	// The backing object should be present and contain the correct contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, o.Name)
	AssertEq(nil, err)
	ExpectEq(fileLeaserLimitBytes, len(contents))

	// Delete the backing object.
	err = t.bucket.DeleteObject(t.ctx, &gcs.DeleteObjectRequest{Name: o.Name})
	AssertEq(nil, err)

	// We should still be able to read the contents, because the read lease
	// should still be valid.
	buf := make([]byte, 4)
	n, err := rl.ReadAt(buf, 0)

	AssertEq(nil, err)
	ExpectEq("taco", string(buf[0:n]))
}
Esempio n. 8
0
func (t *IntegrationTest) LargerThanLeaserLimit() {
	AssertLt(len("taco"), fileLeaserLimitBytes)

	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Extend to be past the leaser limit, then write out to GCS, which should
	// downgrade to a read lease.
	err = t.mc.Truncate(t.ctx, fileLeaserLimitBytes+1)
	AssertEq(nil, err)

	rl, _, err := t.sync(o)
	AssertEq(nil, err)

	// The backing object should be present and contain the correct contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, o.Name)
	AssertEq(nil, err)
	ExpectEq(fileLeaserLimitBytes+1, len(contents))

	// Delete the backing object.
	err = t.bucket.DeleteObject(t.ctx, &gcs.DeleteObjectRequest{Name: o.Name})
	AssertEq(nil, err)

	// The contents should be lost, because the leaser should have revoked the
	// read lease.
	_, err = rl.ReadAt(make([]byte, len(contents)), 0)
	ExpectThat(err, Error(HasSubstr("revoked")))
}
Esempio n. 9
0
func (t *DirTest) CloneToChildFile_DestinationExists() {
	const srcName = "blah/baz"
	dstName := path.Join(dirInodeName, "qux")

	var o *gcs.Object
	var err error

	// Create the source.
	src, err := gcsutil.CreateObject(t.ctx, t.bucket, srcName, []byte("taco"))
	AssertEq(nil, err)

	// And a destination object that will be overwritten.
	_, err = gcsutil.CreateObject(t.ctx, t.bucket, dstName, []byte(""))
	AssertEq(nil, err)

	// Call the inode.
	o, err = t.in.CloneToChildFile(t.ctx, path.Base(dstName), src)
	AssertEq(nil, err)
	AssertNe(nil, o)

	ExpectEq(dstName, o.Name)
	ExpectFalse(inode.IsSymlink(o))
	ExpectEq(len("taco"), o.Size)

	// Check resulting contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, dstName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))
}
Esempio n. 10
0
func (t *ReadOnlyTest) DeleteFile() {
	// Create an object in the bucket.
	_, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	// Attempt to delete it via the file system.
	err = os.Remove(path.Join(t.Dir, "foo"))
	ExpectThat(err, Error(HasSubstr("read-only")))

	// the bucket should not have been modified.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, "foo")

	AssertEq(nil, err)
	ExpectEq("taco", string(contents))
}
Esempio n. 11
0
func (t *FileTest) AppendThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Append some data.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()

	err = t.in.Write(t.ctx, []byte("burrito"), int64(len("taco")))
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration().Object)

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration().Object, o.Generation)
	ExpectEq(t.in.SourceGeneration().Metadata, o.MetaGeneration)
	ExpectEq(len("tacoburrito"), o.Size)
	ExpectEq(
		writeTime.UTC().Format(time.RFC3339Nano),
		o.Metadata["gcsfuse_mtime"])

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("tacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime.UTC()))
}
Esempio n. 12
0
func (t *IntegrationTest) AppendThenSync() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Append some data.
	n, err := t.mc.WriteAt(t.ctx, []byte("burrito"), 4)

	AssertEq(nil, err)
	ExpectEq(len("burrito"), n)

	// Sync should save out the new generation.
	rl, newObj, err := t.sync(o)
	AssertEq(nil, err)

	ExpectNe(o.Generation, newObj.Generation)
	ExpectEq(t.objectGeneration("foo"), newObj.Generation)

	// Read via the bucket.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, "foo")
	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// Read via the lease.
	_, err = rl.Seek(0, 0)
	AssertEq(nil, err)

	contents, err = ioutil.ReadAll(rl)
	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// There should be no junk left over in the bucket besides the object of
	// interest.
	objects, runs, err := gcsutil.ListAll(
		t.ctx,
		t.bucket,
		&gcs.ListObjectsRequest{})

	AssertEq(nil, err)
	AssertEq(1, len(objects))
	AssertEq(0, len(runs))

	ExpectEq("foo", objects[0].Name)
}
Esempio n. 13
0
func (t *DirTest) DeleteChildDir_Exists() {
	const name = "qux"
	objName := path.Join(dirInodeName, name) + "/"

	var err error

	// Create a backing object.
	_, err = gcsutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco"))
	AssertEq(nil, err)

	// Call the inode.
	err = t.in.DeleteChildDir(t.ctx, name)
	AssertEq(nil, err)

	// Check the bucket.
	_, err = gcsutil.ReadObject(t.ctx, t.bucket, objName)
	ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{}))
}
Esempio n. 14
0
func (t *DirTest) DeleteChildFile_ParticularGenerationAndMetaGeneration() {
	const name = "qux"
	objName := path.Join(dirInodeName, name)

	var err error

	// Create a backing object.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco"))
	AssertEq(nil, err)

	// Call the inode.
	err = t.in.DeleteChildFile(t.ctx, name, o.Generation, &o.MetaGeneration)
	AssertEq(nil, err)

	// Check the bucket.
	_, err = gcsutil.ReadObject(t.ctx, t.bucket, objName)
	ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{}))
}
Esempio n. 15
0
func (t *IntegrationTest) AppendThenSync() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", []byte("taco"))
	AssertEq(nil, err)

	t.create(o)

	// Append some data.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()
	n, err := t.tf.WriteAt([]byte("burrito"), 4)
	t.clock.AdvanceTime(time.Second)

	AssertEq(nil, err)
	ExpectEq(len("burrito"), n)

	// Sync should save out the new generation.
	newObj, err := t.sync(o)
	AssertEq(nil, err)

	ExpectNe(o.Generation, newObj.Generation)
	ExpectEq(t.objectGeneration("foo"), newObj.Generation)
	ExpectEq(
		writeTime.UTC().Format(time.RFC3339Nano),
		newObj.Metadata["gcsfuse_mtime"])

	// Read via the bucket.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, "foo")
	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// There should be no junk left over in the bucket besides the object of
	// interest.
	objects, runs, err := gcsutil.ListAll(
		t.ctx,
		t.bucket,
		&gcs.ListObjectsRequest{})

	AssertEq(nil, err)
	AssertEq(1, len(objects))
	AssertEq(0, len(runs))

	ExpectEq("foo", objects[0].Name)
}
Esempio n. 16
0
func (t *IntegrationTest) BackingObjectHasBeenOverwritten_AfterReading() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Fault in the contents.
	_, err = t.mc.ReadAt(t.ctx, []byte{}, 0)
	AssertEq(nil, err)

	// Overwrite the backing object.
	_, err = gcsutil.CreateObject(t.ctx, t.bucket, "foo", "burrito")
	AssertEq(nil, err)

	// Reading and modications should still work.
	_, err = t.mc.ReadAt(t.ctx, []byte{}, 0)
	AssertEq(nil, err)

	_, err = t.mc.WriteAt(t.ctx, []byte("a"), 0)
	AssertEq(nil, err)

	truncateTime := t.clock.Now()
	err = t.mc.Truncate(t.ctx, 3)
	AssertEq(nil, err)
	t.clock.AdvanceTime(time.Second)

	// Stat should see the current state.
	sr, err := t.mc.Stat(t.ctx)
	AssertEq(nil, err)

	ExpectEq(3, sr.Size)
	ExpectEq(0, sr.DirtyThreshold)
	ExpectThat(sr.Mtime, Pointee(timeutil.TimeEq(truncateTime)))

	// Sync should fail with a precondition error.
	_, _, err = t.sync(o)
	ExpectThat(err, HasSameTypeAs(&gcs.PreconditionError{}))

	// The newer version should still be present.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, o.Name)
	AssertEq(nil, err)
	ExpectEq("burrito", string(contents))
}
Esempio n. 17
0
func (t *IntegrationTest) BackingObjectHasBeenDeleted_AfterReading() {
	// Create.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, "foo", "taco")
	AssertEq(nil, err)

	t.create(o)

	// Fault in the contents.
	_, err = t.mc.ReadAt(t.ctx, []byte{}, 0)
	AssertEq(nil, err)

	// Delete the backing object.
	err = t.bucket.DeleteObject(t.ctx, &gcs.DeleteObjectRequest{Name: o.Name})
	AssertEq(nil, err)

	// Reading and modications should still work.
	_, err = t.mc.ReadAt(t.ctx, []byte{}, 0)
	AssertEq(nil, err)

	_, err = t.mc.WriteAt(t.ctx, []byte("a"), 0)
	AssertEq(nil, err)

	truncateTime := t.clock.Now()
	err = t.mc.Truncate(t.ctx, 1)
	AssertEq(nil, err)
	t.clock.AdvanceTime(time.Second)

	// Stat should see the current state.
	sr, err := t.mc.Stat(t.ctx)
	AssertEq(nil, err)

	ExpectEq(1, sr.Size)
	ExpectEq(0, sr.DirtyThreshold)
	ExpectThat(sr.Mtime, Pointee(timeutil.TimeEq(truncateTime)))

	// Sync should fail with a precondition error.
	_, _, err = t.sync(o)
	ExpectThat(err, HasSameTypeAs(&gcs.PreconditionError{}))

	// Nothing should have been created.
	_, err = gcsutil.ReadObject(t.ctx, t.bucket, o.Name)
	ExpectThat(err, HasSameTypeAs(&gcs.NotFoundError{}))
}
Esempio n. 18
0
func (t *DirTest) DeleteChildFile_WrongGeneration() {
	const name = "qux"
	objName := path.Join(dirInodeName, name)

	var err error

	// Create a backing object.
	o, err := gcsutil.CreateObject(t.ctx, t.bucket, objName, []byte("taco"))
	AssertEq(nil, err)

	// Call the inode with the wrong generation. No error should be returned.
	err = t.in.DeleteChildFile(t.ctx, name, o.Generation+1, &o.MetaGeneration)
	AssertEq(nil, err)

	// The original generation should still be there.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, objName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))
}
Esempio n. 19
0
func (t *FileTest) WriteThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(len("paco"), o.Size)

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("paco", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("paco"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}
Esempio n. 20
0
func (t *PrefixBucketTest) ComposeObjects() {
	var err error

	suffix0 := "taco"
	contents0 := "foo"

	suffix1 := "burrito"
	contents1 := "bar"

	// Create two objects through the back door.
	err = gcsutil.CreateObjects(
		t.ctx,
		t.wrapped,
		map[string][]byte{
			t.prefix + suffix0: []byte(contents0),
			t.prefix + suffix1: []byte(contents1),
		})

	AssertEq(nil, err)

	// Compose them.
	newSuffix := "enchilada"
	o, err := t.bucket.ComposeObjects(
		t.ctx,
		&gcs.ComposeObjectsRequest{
			DstName: newSuffix,
			Sources: []gcs.ComposeSource{
				{Name: suffix0},
				{Name: suffix1},
			},
		})

	AssertEq(nil, err)
	ExpectEq(newSuffix, o.Name)

	// Read it through the back door.
	actual, err := gcsutil.ReadObject(t.ctx, t.wrapped, t.prefix+newSuffix)
	AssertEq(nil, err)
	ExpectEq(contents0+contents1, string(actual))
}
Esempio n. 21
0
func (t *PrefixBucketTest) CreateObject() {
	var err error
	suffix := "taco"
	contents := "foobar"

	// Create the object.
	o, err := t.bucket.CreateObject(
		t.ctx,
		&gcs.CreateObjectRequest{
			Name:            suffix,
			ContentLanguage: "en-GB",
			Contents:        strings.NewReader(contents),
		})

	AssertEq(nil, err)
	ExpectEq(suffix, o.Name)
	ExpectEq("en-GB", o.ContentLanguage)

	// Read it through the back door.
	actual, err := gcsutil.ReadObject(t.ctx, t.wrapped, t.prefix+suffix)
	AssertEq(nil, err)
	ExpectEq(contents, string(actual))
}
Esempio n. 22
0
func (t *IntegrationTest) MultipleInteractions() {
	// We will run through the script below for multiple interesting object
	// sizes.
	sizes := []int{
		0,
		1,
		chunkSize - 1,
		chunkSize,
		chunkSize + 1,
		3*chunkSize - 1,
		3 * chunkSize,
		3*chunkSize + 1,
		fileLeaserLimitBytes - 1,
		fileLeaserLimitBytes,
		fileLeaserLimitBytes + 1,
		((fileLeaserLimitBytes / chunkSize) - 1) * chunkSize,
		(fileLeaserLimitBytes / chunkSize) * chunkSize,
		((fileLeaserLimitBytes / chunkSize) + 1) * chunkSize,
	}

	// Generate random contents for the maximum size.
	var maxSize int
	for _, size := range sizes {
		if size > maxSize {
			maxSize = size
		}
	}

	randData := randBytes(maxSize)

	// Transition the mutable object in and out of the dirty state. Make sure
	// everything stays consistent.
	for i, size := range sizes {
		desc := fmt.Sprintf("test case %d (size %d)", i, size)
		name := fmt.Sprintf("obj_%d", i)
		buf := make([]byte, size)

		// Create the backing object with random initial contents.
		expectedContents := make([]byte, size)
		copy(expectedContents, randData)

		o, err := gcsutil.CreateObject(
			t.ctx,
			t.bucket,
			name,
			string(expectedContents))

		AssertEq(nil, err)

		// Create a mutable object around it.
		t.create(o)

		// Read the contents of the mutable object.
		_, err = t.mc.ReadAt(t.ctx, buf, 0)

		AssertThat(err, AnyOf(nil, io.EOF))
		if !bytes.Equal(buf, expectedContents) {
			AddFailure("Contents mismatch for %s", desc)
			AbortTest()
		}

		// Modify some bytes.
		if size > 0 {
			expectedContents[0] = 17
			expectedContents[size/2] = 19
			expectedContents[size-1] = 23

			_, err = t.mc.WriteAt(t.ctx, []byte{17}, 0)
			AssertEq(nil, err)

			_, err = t.mc.WriteAt(t.ctx, []byte{19}, int64(size/2))
			AssertEq(nil, err)

			_, err = t.mc.WriteAt(t.ctx, []byte{23}, int64(size-1))
			AssertEq(nil, err)
		}

		// Compare contents again.
		_, err = t.mc.ReadAt(t.ctx, buf, 0)

		AssertThat(err, AnyOf(nil, io.EOF))
		if !bytes.Equal(buf, expectedContents) {
			AddFailure("Contents mismatch for %s", desc)
			AbortTest()
		}

		// Sync and recreate if necessary.
		_, newObj, err := t.sync(o)
		AssertEq(nil, err)

		if newObj != nil {
			t.create(newObj)
		}

		// Check the new backing object's contents.
		objContents, err := gcsutil.ReadObject(t.ctx, t.bucket, name)
		AssertEq(nil, err)
		if !bytes.Equal(objContents, expectedContents) {
			AddFailure("Contents mismatch for %s", desc)
			AbortTest()
		}

		// Compare contents again.
		_, err = t.mc.ReadAt(t.ctx, buf, 0)

		AssertThat(err, AnyOf(nil, io.EOF))
		if !bytes.Equal(buf, expectedContents) {
			AddFailure("Contents mismatch for %s", desc)
			AbortTest()
		}

		// Dirty again.
		if size > 0 {
			expectedContents[0] = 29

			_, err = t.mc.WriteAt(t.ctx, []byte{29}, 0)
			AssertEq(nil, err)
		}

		// Compare contents again.
		_, err = t.mc.ReadAt(t.ctx, buf, 0)

		AssertThat(err, AnyOf(nil, io.EOF))
		if !bytes.Equal(buf, expectedContents) {
			AddFailure("Contents mismatch for %s", desc)
			AbortTest()
		}
	}
}
Esempio n. 23
0
func verifyObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	verifyAfter time.Duration,
	in <-chan record,
	out chan<- record) (err error) {
	// Set up a worker function.
	worker := func(ctx context.Context) (err error) {
		for r := range in {
			name := fmt.Sprintf("%s%x", objectNamePrefix, r.sha1)

			// Wait until it is time.
			wakeTime := r.creationTime.Add(verifyAfter)
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case <-time.After(wakeTime.Sub(time.Now())):
			}

			// Attempt to read the object.
			var contents []byte
			contents, err = gcsutil.ReadObject(ctx, bucket, name)
			if err != nil {
				err = fmt.Errorf("ReadObject(%q): %v", name, err)
				return
			}

			// Check the contents.
			actual := sha1.Sum(contents)
			if actual != r.sha1 {
				err = fmt.Errorf(
					"SHA1 mismatch for %q: %x vs. %x",
					name,
					actual,
					r.sha1)

				return
			}

			log.Printf("Verified object %q.", name)

			// Pass on the record if we've been asked to.
			if out != nil {
				select {
				case <-ctx.Done():
					err = ctx.Err()
					return

				case out <- r:
				}
			}
		}

		return
	}

	// Run a bunch of workers.
	b := syncutil.NewBundle(ctx)
	for i := 0; i < perStageParallelism; i++ {
		b.Add(worker)
	}

	err = b.Join()
	return
}