Пример #1
0
// Create an object with the supplied contents in the given bucket with the
// given name.
func CreateObject(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	contents string) (*gcs.Object, error) {
	req := &gcs.CreateObjectRequest{
		Name:     name,
		Contents: strings.NewReader(contents),
	}

	return bucket.CreateObject(ctx, req)
}
Пример #2
0
func createObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	out chan<- record) (err error) {
	throttle := time.Tick(time.Second / createRateHz)

	// Set up a worker function.
	worker := func(ctx context.Context) (err error) {
		randSrc := rand.New(rand.NewSource(makeSeed()))
		for {
			// Choose a random size (every once in awhile making sure we see the
			// max), and generate content of that size.
			const maxSize = 1 << 24
			var size int
			if randSrc.Int31n(100) == 0 {
				size = maxSize
			} else {
				size = int(randSrc.Int31n(maxSize + 1))
			}

			content := randBytes(randSrc, size)

			// Compute hashes and checksums.
			sha1 := sha1.Sum(content)
			crc32c := *gcsutil.CRC32C(content)

			// Wait for permission to proceed.
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case <-throttle:
			}

			// Create an object.
			req := &gcs.CreateObjectRequest{
				Name:     fmt.Sprintf("%s%x", objectNamePrefix, sha1),
				Contents: bytes.NewReader(content),
				CRC32C:   &crc32c,

				Metadata: map[string]string{
					"expected_sha1":   fmt.Sprintf("%x", sha1),
					"expected_crc32c": fmt.Sprintf("%#08x", crc32c),
				},
			}

			var o *gcs.Object
			o, err = bucket.CreateObject(ctx, req)
			if err != nil {
				err = fmt.Errorf("CreateObject(%q): %v", req.Name, err)
				return
			}

			log.Printf("Created object %q.", req.Name)

			// Check the object.
			if o.Name != req.Name {
				err = fmt.Errorf(
					"Name mismatch: %q vs. %q",
					o.Name,
					req.Name)

				return
			}

			if o.CRC32C != crc32c {
				err = fmt.Errorf(
					"Object %q CRC mismatch: %#08x vs. %#08x",
					o.Name,
					o.CRC32C,
					crc32c)

				return
			}

			// Write out the record.
			r := record{
				sha1:         sha1,
				creationTime: time.Now(),
			}

			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case out <- r:
			}
		}
	}

	// Run a bunch of workers.
	b := syncutil.NewBundle(ctx)
	for i := 0; i < perStageParallelism; i++ {
		b.Add(worker)
	}

	err = b.Join()
	return
}