func TestCache(t *testing.T) {
// Under these conditions we cannot guarantee that the order of
// blocks returned by nextBlock work will not result in additional
// cache puts.
if *conc != 1 {
return
}
const (
infix = "payload"
blocks = 10
)
// Each pattern is a series of seek-and-read (when the element >= 0)
// or read (when the element < 0). Each read is exactly one block
// worth of data.
type opPair struct{ seekBlock, blockID int }
patterns := []struct {
ops []opPair
// One for each cache case below. If new caches are added to the
// test list, stats must be added here.
expectedStats []cache.Stats
}{
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +0, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: -1, blockID: 3},
{seekBlock: -1, blockID: 4},
},
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache: LRU(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(5)
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(10)
{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(5)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
},
},
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: -1, blockID: 3},
{seekBlock: -1, blockID: 4},
{seekBlock: -1, blockID: 5},
},
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache.
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // LRU(1)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(5)
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(10)
{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // Random(1)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(5)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
},
},
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +2, blockID: 2},
{seekBlock: -1, blockID: 3},
{seekBlock: -1, blockID: 4},
{seekBlock: -1, blockID: 5},
{seekBlock: -1, blockID: 6},
},
// Re-reading the same block avoids a cache look-up.
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache.
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // LRU(1)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // LRU(5)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(10)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // FIFO(1)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // FIFO(5)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(10)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // Random(1)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // Random(5)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(10)
{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(11)
},
},
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +3, blockID: 3},
{seekBlock: -1, blockID: 4},
{seekBlock: -1, blockID: 5},
{seekBlock: -1, blockID: 6},
{seekBlock: -1, blockID: 7},
},
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache.
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
},
},
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +4, blockID: 4},
{seekBlock: -1, blockID: 5},
{seekBlock: -1, blockID: 6},
{seekBlock: -1, blockID: 7},
{seekBlock: -1, blockID: 8},
},
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache.
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
},
},
{
ops: []opPair{
{seekBlock: -1, blockID: 0},
{seekBlock: -1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +1, blockID: 1},
{seekBlock: +2, blockID: 2},
{seekBlock: +1, blockID: 1},
{seekBlock: +1, blockID: 1},
{seekBlock: -1, blockID: 2},
{seekBlock: +7, blockID: 7},
{seekBlock: -1, blockID: 8},
{seekBlock: -1, blockID: 9},
},
expectedStats: []cache.Stats{
{}, // nil cache.
{}, // nil cache.
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // LRU(1)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(5)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(10)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(11)
{}, // nil cache: FIFO(0)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 8}, // FIFO(1)
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(5)
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(10)
{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(11)
{}, // nil cache: Random(0)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // Random(1)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(5)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(10)
{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(11)
},
},
}
for k, pat := range patterns {
// Use different caches.
for j, s := range []Cache{
nil, // Explicitly nil.
cache.NewLRU(0), // Functionally nil.
cache.NewLRU(1),
cache.NewLRU(blocks / 2),
cache.NewLRU(blocks),
cache.NewLRU(blocks + 1),
cache.NewFIFO(0), // Functionally nil.
cache.NewFIFO(1),
cache.NewFIFO(blocks / 2),
cache.NewFIFO(blocks),
cache.NewFIFO(blocks + 1),
cache.NewRandom(0), // Functionally nil.
cache.NewRandom(1),
cache.NewRandom(blocks / 2),
cache.NewRandom(blocks),
cache.NewRandom(blocks + 1),
} {
var (
buf bytes.Buffer
offsets = []int{0}
)
w := NewWriter(&buf, 1)
for i := 0; i < blocks; i++ {
if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil {
t.Fatalf("Write: %v", err)
}
if err := w.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if err := w.Wait(); err != nil {
t.Fatalf("Wait: %v", err)
}
offsets = append(offsets, buf.Len())
}
w.Close()
offsets = offsets[:len(offsets)-1]
br := bytes.NewReader(buf.Bytes())
// Insert a HasEOF to ensure it does not corrupt subsequent reads.
HasEOF(br)
r, err := NewReader(br, *conc)
if err != nil {
t.Fatalf("NewReader: %v", err)
}
var stats *cache.StatsRecorder
if s != nil {
stats = &cache.StatsRecorder{Cache: s}
s = stats
}
r.SetCache(s)
p := make([]byte, len(infix)+2)
for _, op := range pat.ops {
if op.seekBlock >= 0 {
err := r.Seek(Offset{File: int64(offsets[op.seekBlock])})
if err != nil {
t.Fatalf("Seek: %v", err)
}
}
n, err := r.Read(p)
if n != len(p) {
t.Errorf("Unexpected read length: got:%d want:%d", n, len(p))
}
if err != nil {
t.Fatalf("Read: %v", err)
}
got := string(p)
want := fmt.Sprintf("%d%[2]s%[1]d", op.blockID, infix)
if got != want {
t.Errorf("Unexpected result: got:%q want:%q", got, want)
}
}
if stats != nil && stats.Stats() != pat.expectedStats[j] {
t.Errorf("Unexpected result for cache %d pattern %d: got:%+v want:%+v", j, k, stats.Stats(), pat.expectedStats[j])
}
r.Close()
}
}
}
func TestSeekFast(t *testing.T) {
// Under these conditions we cannot guarantee that a worker
// will not read bytes after a Seek call has been made.
if *conc != 1 && runtime.GOMAXPROCS(0) > 1 {
return
}
const (
infix = "payload"
blocks = 10
)
// Use different caches.
for _, cache := range []Cache{
nil, // Explicitly nil.
cache.NewLRU(0), // Functionally nil.
cache.NewLRU(1),
cache.NewLRU(blocks / 2),
cache.NewLRU(blocks),
cache.NewLRU(blocks + 1),
cache.NewRandom(0), // Functionally nil.
cache.NewRandom(1),
cache.NewRandom(blocks / 2),
cache.NewRandom(blocks),
cache.NewRandom(blocks + 1),
} {
var (
buf bytes.Buffer
offsets = []int{0}
)
w := NewWriter(&buf, 1)
for i := 0; i < blocks; i++ {
if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil {
t.Fatalf("Write: %v", err)
}
if err := w.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if err := w.Wait(); err != nil {
t.Fatalf("Wait: %v", err)
}
offsets = append(offsets, buf.Len())
}
w.Close()
offsets = offsets[:len(offsets)-1]
c := &countReadSeeker{r: bytes.NewReader(buf.Bytes())}
// Insert a HasEOF to ensure it does not corrupt subsequent reads.
HasEOF(bytes.NewReader(buf.Bytes()))
r, err := NewReader(c, *conc)
if err != nil {
t.Fatalf("NewReader: %v", err)
}
r.SetCache(cache)
p := make([]byte, len(infix)+2)
func() {
defer func() {
r := recover()
if r != nil {
t.Fatalf("Seek on unread reader panicked: %v", r)
}
}()
err := r.Seek(Offset{})
if err != nil {
t.Fatalf("Seek: %v", err)
}
}()
// Standard read through of the data.
for i := range offsets {
n, err := r.Read(p)
if n != len(p) {
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
}
if err != nil {
t.Fatalf("Read: %v", err)
}
got := string(p)
want := fmt.Sprintf("%d%[2]s%[1]d", i, infix)
if got != want {
t.Errorf("Unexpected result: got:%q want:%q", got, want)
}
}
// Seek to each block in turn
for i, o := range offsets {
err := r.Seek(Offset{File: int64(o)})
if err != nil {
t.Fatalf("Seek: %v", err)
}
n, err := r.Read(p)
if n != len(p) {
t.Errorf("Unexpected read length: got:%d want:%d", n, len(p))
}
if err != nil {
t.Fatalf("Read: %v", err)
}
got := string(p)
want := fmt.Sprintf("%d%[2]s%[1]d", i, infix)
if got != want {
t.Errorf("Unexpected result: got:%q want:%q", got, want)
}
}
// Seek to each block in turn, but read the infix and then the first 2 bytes.
for i, o := range offsets {
if err := r.Seek(Offset{File: int64(o), Block: 1}); err != nil {
t.Fatalf("Seek: %v", err)
}
p = p[:len(infix)]
n, err := r.Read(p)
if n != len(p) {
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
}
if err != nil {
t.Fatalf("Read: %v", err)
}
got := string(p)
want := infix
if got != want {
t.Fatalf("Unexpected result: got:%q want:%q", got, want)
}
// Check whether the underlying reader was seeked or read.
hasRead := c.offset()
if err = r.Seek(Offset{File: int64(o), Block: 0}); err != nil {
t.Fatalf("Seek: %v", err)
}
if b := c.offset() - hasRead; b != 0 {
t.Errorf("Seek performed unexpected read: %d bytes", b)
}
if c.didSeek() {
t.Error("Seek caused underlying Seek.")
}
p = p[:2]
n, err = r.Read(p)
if n != len(p) {
t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
}
if err != nil {
t.Fatalf("Read: %v", err)
}
got = string(p)
want = fmt.Sprintf("%dp", i)
if got != want {
t.Fatalf("Unexpected result: got:%q want:%q", got, want)
}
}
r.Close()
}
}