// Create an object with the supplied contents in the given bucket with the
// given name.
func CreateObject(
ctx context.Context,
bucket gcs.Bucket,
name string,
contents string) (*gcs.Object, error) {
req := &gcs.CreateObjectRequest{
Name: name,
Contents: strings.NewReader(contents),
}
return bucket.CreateObject(ctx, req)
}
func createObjects(
ctx context.Context,
bucket gcs.Bucket,
out chan<- record) (err error) {
throttle := time.Tick(time.Second / createRateHz)
// Set up a worker function.
worker := func(ctx context.Context) (err error) {
randSrc := rand.New(rand.NewSource(makeSeed()))
for {
// Choose a random size (every once in awhile making sure we see the
// max), and generate content of that size.
const maxSize = 1 << 24
var size int
if randSrc.Int31n(100) == 0 {
size = maxSize
} else {
size = int(randSrc.Int31n(maxSize + 1))
}
content := randBytes(randSrc, size)
// Compute hashes and checksums.
sha1 := sha1.Sum(content)
crc32c := *gcsutil.CRC32C(content)
// Wait for permission to proceed.
select {
case <-ctx.Done():
err = ctx.Err()
return
case <-throttle:
}
// Create an object.
req := &gcs.CreateObjectRequest{
Name: fmt.Sprintf("%s%x", objectNamePrefix, sha1),
Contents: bytes.NewReader(content),
CRC32C: &crc32c,
Metadata: map[string]string{
"expected_sha1": fmt.Sprintf("%x", sha1),
"expected_crc32c": fmt.Sprintf("%#08x", crc32c),
},
}
var o *gcs.Object
o, err = bucket.CreateObject(ctx, req)
if err != nil {
err = fmt.Errorf("CreateObject(%q): %v", req.Name, err)
return
}
log.Printf("Created object %q.", req.Name)
// Check the object.
if o.Name != req.Name {
err = fmt.Errorf(
"Name mismatch: %q vs. %q",
o.Name,
req.Name)
return
}
if o.CRC32C != crc32c {
err = fmt.Errorf(
"Object %q CRC mismatch: %#08x vs. %#08x",
o.Name,
o.CRC32C,
crc32c)
return
}
// Write out the record.
r := record{
sha1: sha1,
creationTime: time.Now(),
}
select {
case <-ctx.Done():
err = ctx.Err()
return
case out <- r:
}
}
}
// Run a bunch of workers.
b := syncutil.NewBundle(ctx)
for i := 0; i < perStageParallelism; i++ {
b.Add(worker)
}
err = b.Join()
return
}