func readSequential(
r io.ReadSeeker,
readSize int,
desiredDuration time.Duration) (err error) {
buf := make([]byte, readSize)
start := time.Now()
var readCount int64
var bytesRead int64
for time.Since(start) < desiredDuration {
var n int
n, err = r.Read(buf)
switch {
case err == io.EOF:
_, err = r.Seek(0, 0)
if err != nil {
err = fmt.Errorf("Seek: %v", err)
return
}
case err != nil:
err = fmt.Errorf("Read: %v", err)
}
bytesRead += int64(n)
readCount++
}
d := time.Since(start)
// Report.
seconds := float64(d) / float64(time.Second)
readsPerSec := float64(readCount) / seconds
bytesPerSec := float64(bytesRead) / seconds
fmt.Printf(
"Read %d times (%s) in %v (%.1f Hz, %s/s)\n",
readCount,
format.Bytes(float64(bytesRead)),
d,
readsPerSec,
format.Bytes(bytesPerSec))
fmt.Println()
return
}
func readRandom(
r io.ReaderAt,
fileSize int64,
readSize int,
desiredDuration time.Duration) (err error) {
// Make sure the logic below for choosing offsets works.
if fileSize < int64(readSize) {
err = fmt.Errorf(
"File size of %d bytes not large enough for reads of %d bytes",
fileSize,
readSize)
return
}
buf := make([]byte, readSize)
start := time.Now()
var readCount int64
var bytesRead int64
for time.Since(start) < desiredDuration {
// Choose a random offset at which to read.
off := rand.Int63n(fileSize - int64(readSize))
// Read, ignoring io.EOF which io.ReaderAt is allowed to return for reads
// that abut the end of the file.
var n int
n, err = r.ReadAt(buf, off)
switch {
case err == io.EOF && n == readSize:
err = nil
case err != nil:
err = fmt.Errorf("ReadAt: %v", err)
return
}
readCount++
bytesRead += int64(n)
}
d := time.Since(start)
// Report.
seconds := float64(d) / float64(time.Second)
readsPerSec := float64(readCount) / seconds
fmt.Printf(
"Read %d times (%s) in %v (%.1f Hz)\n",
readCount,
format.Bytes(float64(bytesRead)),
d,
readsPerSec)
fmt.Println()
return
}
func run() (err error) {
if *fFile == "" {
err = errors.New("You must set --file.")
return
}
// Open the file for reading.
f, err := os.Open(*fFile)
if err != nil {
return
}
// Find its size.
size, err := f.Seek(0, 2)
if err != nil {
err = fmt.Errorf("Seek: %v", err)
return
}
log.Printf("%s has size %s.", f.Name(), format.Bytes(float64(size)))
// Perform reads.
if *fRandom {
err = readRandom(f, size, *fReadSize, *fDuration)
if err != nil {
err = fmt.Errorf("readRandom: %v", err)
return
}
} else {
err = readSequential(f, *fReadSize, *fDuration)
if err != nil {
err = fmt.Errorf("readSequential: %v", err)
return
}
}
return
}
func run() (err error) {
if *fDir == "" {
err = errors.New("You must set --dir.")
return
}
// Create a temporary file.
log.Printf("Creating a temporary file in %s.", *fDir)
f, err := ioutil.TempFile(*fDir, "write_locally")
if err != nil {
err = fmt.Errorf("TempFile: %v", err)
return
}
path := f.Name()
// Make sure we clean it up later.
defer func() {
log.Printf("Truncating and closing %s.", path)
f.Truncate(0)
f.Close()
log.Printf("Deleting %s.", path)
os.Remove(path)
}()
// Extend to the initial size.
log.Printf("Truncating to %d bytes.", *fFileSize)
err = f.Truncate(*fFileSize)
if err != nil {
err = fmt.Errorf("Truncate: %v", err)
return
}
// Repeatedly overwrite the file with zeroes.
log.Println("Measuring...")
var bytesWritten int64
var writeCount int64
buf := make([]byte, *fWriteSize)
start := time.Now()
for time.Since(start) < *fDuration {
// Seek to the beginning.
_, err = f.Seek(0, 0)
if err != nil {
err = fmt.Errorf("Seek: %v", err)
return
}
// Overwrite.
var n int64
for n < *fFileSize && time.Since(start) < *fDuration {
toWrite := *fFileSize - n
if toWrite > *fWriteSize {
toWrite = *fWriteSize
}
var tmp int
tmp, err = f.Write(buf)
if err != nil {
err = fmt.Errorf("Write: %v", err)
return
}
n += int64(tmp)
bytesWritten += int64(tmp)
writeCount++
}
}
d := time.Since(start)
// Report.
seconds := float64(d) / float64(time.Second)
writesPerSec := float64(writeCount) / seconds
bytesPerSec := float64(bytesWritten) / seconds
fmt.Printf(
"Wrote %d times (%s) in %v (%.1f Hz, %s/s)\n",
writeCount,
format.Bytes(float64(bytesWritten)),
d,
writesPerSec,
format.Bytes(bytesPerSec))
fmt.Println()
return
}
func run() (err error) {
if *fDir == "" {
err = errors.New("You must set --dir.")
return
}
// Create a temporary file.
log.Printf("Creating a temporary file in %s.", *fDir)
f, err := ioutil.TempFile(*fDir, "write_to_gcs")
if err != nil {
err = fmt.Errorf("TempFile: %v", err)
return
}
path := f.Name()
// Make sure we clean it up later.
defer func() {
log.Printf("Deleting %s.", path)
os.Remove(path)
}()
// Write the configured number of zeroes to the file, measuing the time
// taken.
log.Println("Writing...")
buf := make([]byte, *fWriteSize)
var bytesWritten int64
start := time.Now()
for bytesWritten < *fFileSize {
// Decide how many bytes to write.
toWrite := *fFileSize - bytesWritten
if toWrite > *fWriteSize {
toWrite = *fWriteSize
}
// Write them.
_, err = f.Write(buf)
if err != nil {
err = fmt.Errorf("Write: %v", err)
return
}
bytesWritten += toWrite
}
writeDuration := time.Since(start)
// Close the file, measuring the time taken.
log.Println("Flushing...")
start = time.Now()
err = f.Close()
closeDuration := time.Since(start)
if err != nil {
err = fmt.Errorf("Close: %v", err)
return
}
// Report.
{
seconds := float64(writeDuration) / float64(time.Second)
bytesPerSec := float64(bytesWritten) / seconds
fmt.Printf(
"Wrote %s in %v (%s/s)\n",
format.Bytes(float64(bytesWritten)),
writeDuration,
format.Bytes(bytesPerSec))
}
{
seconds := float64(closeDuration) / float64(time.Second)
bytesPerSec := float64(bytesWritten) / seconds
fmt.Printf(
"Flushed %s in %v (%s/s)\n",
format.Bytes(float64(bytesWritten)),
closeDuration,
format.Bytes(bytesPerSec))
}
fmt.Println()
return
}
func run() (err error) {
if *fDir == "" {
err = errors.New("You must set --dir.")
return
}
// Create a temporary file.
log.Printf("Creating a temporary file in %s.", *fDir)
f, err := ioutil.TempFile(*fDir, "sequential_read")
if err != nil {
err = fmt.Errorf("TempFile: %v", err)
return
}
path := f.Name()
// Make sure we clean it up later.
defer func() {
log.Printf("Deleting %s.", path)
os.Remove(path)
}()
// Fill it with random content.
log.Printf("Writing %d random bytes.", *fFileSize)
_, err = io.Copy(f, io.LimitReader(rand.Reader, *fFileSize))
if err != nil {
err = fmt.Errorf("Copying random bytes: %v", err)
return
}
// Finish off the file.
err = f.Close()
if err != nil {
err = fmt.Errorf("Closing file: %v", err)
return
}
// Run several iterations.
log.Printf("Measuring for %v...", *fDuration)
var fullFileRead percentile.DurationSlice
var singleReadCall percentile.DurationSlice
buf := make([]byte, *fReadSize)
overallStartTime := time.Now()
for len(fullFileRead) == 0 || time.Since(overallStartTime) < *fDuration {
// Open the file for reading.
f, err = os.Open(path)
if err != nil {
err = fmt.Errorf("Opening file: %v", err)
return
}
// Read the whole thing.
fileStartTime := time.Now()
for err == nil {
readStartTime := time.Now()
_, err = f.Read(buf)
singleReadCall = append(singleReadCall, time.Since(readStartTime))
}
fullFileRead = append(fullFileRead, time.Since(fileStartTime))
switch {
case err == io.EOF:
err = nil
case err != nil:
err = fmt.Errorf("Reading: %v", err)
return
}
// Close the file.
err = f.Close()
if err != nil {
err = fmt.Errorf("Closing file after reading: %v", err)
return
}
}
sort.Sort(fullFileRead)
sort.Sort(singleReadCall)
log.Printf(
"Read the file %d times, using %d calls to read(2).",
len(fullFileRead),
len(singleReadCall))
// Report.
ptiles := []int{50, 90, 98}
reportSlice := func(
name string,
bytesPerObservation int64,
observations percentile.DurationSlice) {
fmt.Printf("\n%s:\n", name)
for _, ptile := range ptiles {
d := percentile.Duration(observations, ptile)
seconds := float64(d) / float64(time.Second)
bandwidthBytesPerSec := float64(bytesPerObservation) / seconds
fmt.Printf(
" %02dth ptile: %10v (%s/s)\n",
ptile,
d,
format.Bytes(bandwidthBytesPerSec))
}
}
reportSlice("Full-file read times", *fFileSize, fullFileRead)
reportSlice("read(2) latencies", *fReadSize, singleReadCall)
fmt.Println()
return
}