func main() {
log.SetPrefix("pv: ")
log.Printf("streaming!")
start := time.Now()
mrw := iocontrol.NewMeasuredReader(os.Stdin)
pw, pr, sample := iocontrol.Profile(os.Stdout, mrw)
go func() {
for range time.NewTicker(5 * time.Second).C {
profile := sample()
log.Printf("%s at %sps (read=%v write=%v total=%v)",
humanize.IBytes(uint64(mrw.Total())),
humanize.IBytes(mrw.BytesPerSec()),
profile.WaitRead,
profile.WaitWrite,
profile.Total,
)
}
}()
n, err := io.Copy(pw, spark.ReaderOut(pr, os.Stderr))
if err != nil {
log.Fatalf("error after %s: %v", humanize.IBytes(uint64(n)), err)
}
log.Printf("done after %s in %v", humanize.IBytes(uint64(n)), time.Since(start))
}
func TestParse(t *testing.T) {
testfile := os.Getenv("TESTFILE")
if testfile == "" {
t.Skip("No testfile specified. Please set `TESTFILE` environment variable with the file path.")
}
f, err := os.Open(testfile)
ensure.Nil(t, err)
defer f.Close()
mr := iocontrol.NewMeasuredReader(f)
rdr := newMockOSMReader()
d := NewDecoder(mr)
err = d.Parse(rdr)
fmt.Printf("Speed: %v/s, total read: %v\n", humanize.Bytes(mr.BytesPerSec()), humanize.Bytes(uint64(mr.Total())))
fmt.Printf("Read %v nodes, %v ways, %v relations\n", atomic.LoadUint64(rdr.Nodes), atomic.LoadUint64(rdr.Ways), atomic.LoadUint64(rdr.Relations))
ensure.Nil(t, err)
}
func Example_ThrottledReader() {
totalSize := 10 * iocontrol.KiB
readPerSec := 100 * iocontrol.KiB
maxBurst := 10 * time.Millisecond
input := randBytes(totalSize)
src := bytes.NewReader(input)
measured := iocontrol.NewMeasuredReader(src)
throttled := iocontrol.ThrottledReader(measured, readPerSec, maxBurst)
done := make(chan []byte)
go func() {
start := time.Now()
output, err := ioutil.ReadAll(throttled)
fmt.Printf("done in %.1fs", time.Since(start).Seconds())
if err != nil {
log.Fatalf("error reading: %v", err)
}
done <- output
}()
for {
select {
case <-time.Tick(time.Millisecond * 10):
log.Printf("reading at %s/s", humanize.IBytes(measured.BytesPerSec()))
case output := <-done:
if !bytes.Equal(input, output) {
log.Print("==== input ====\n", hex.Dump(input))
log.Print("==== output ====\n", hex.Dump(output))
log.Fatalf("mismatch between input and output")
}
return
}
}
// Output:
// done in 0.1s
}