Ejemplo n.º 1
0
func TestCache(t *testing.T) {
	// Under these conditions we cannot guarantee that the order of
	// blocks returned by nextBlock work will not result in additional
	// cache puts.
	if *conc != 1 {
		return
	}
	const (
		infix  = "payload"
		blocks = 10
	)

	// Each pattern is a series of seek-and-read (when the element >= 0)
	// or read (when the element < 0). Each read is exactly one block
	// worth of data.
	type opPair struct{ seekBlock, blockID int }
	patterns := []struct {
		ops []opPair

		// One for each cache case below. If new caches are added to the
		// test list, stats must be added here.
		expectedStats []cache.Stats
	}{
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +0, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: -1, blockID: 3},
				{seekBlock: -1, blockID: 4},
			},
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache: LRU(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(5)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(10)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 4, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(5)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
				{Gets: 7, Misses: 4, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
			},
		},
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: -1, blockID: 3},
				{seekBlock: -1, blockID: 4},
				{seekBlock: -1, blockID: 5},
			},
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache.
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // LRU(1)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(5)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(5)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(10)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 5, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 4}, // Random(1)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(5)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
				{Gets: 7, Misses: 5, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
			},
		},
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +2, blockID: 2},
				{seekBlock: -1, blockID: 3},
				{seekBlock: -1, blockID: 4},
				{seekBlock: -1, blockID: 5},
				{seekBlock: -1, blockID: 6},
			},
			// Re-reading the same block avoids a cache look-up.
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache.
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // LRU(1)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // LRU(5)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(10)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // FIFO(1)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // FIFO(5)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(10)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 5}, // Random(1)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 1}, // Random(5)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(10)
				{Gets: 6, Misses: 6, Puts: 6, Retains: 6, Evictions: 0}, // Random(11)
			},
		},
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +3, blockID: 3},
				{seekBlock: -1, blockID: 4},
				{seekBlock: -1, blockID: 5},
				{seekBlock: -1, blockID: 6},
				{seekBlock: -1, blockID: 7},
			},
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache.
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
			},
		},
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +4, blockID: 4},
				{seekBlock: -1, blockID: 5},
				{seekBlock: -1, blockID: 6},
				{seekBlock: -1, blockID: 7},
				{seekBlock: -1, blockID: 8},
			},
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache.
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // LRU(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // LRU(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // FIFO(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // FIFO(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 6}, // Random(1)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 2}, // Random(5)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(10)
				{Gets: 7, Misses: 7, Puts: 7, Retains: 7, Evictions: 0}, // Random(11)
			},
		},
		{
			ops: []opPair{
				{seekBlock: -1, blockID: 0},
				{seekBlock: -1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +1, blockID: 1},
				{seekBlock: +2, blockID: 2},
				{seekBlock: +1, blockID: 1},
				{seekBlock: +1, blockID: 1},
				{seekBlock: -1, blockID: 2},
				{seekBlock: +7, blockID: 7},
				{seekBlock: -1, blockID: 8},
				{seekBlock: -1, blockID: 9},
			},
			expectedStats: []cache.Stats{
				{}, // nil cache.
				{}, // nil cache.
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // LRU(1)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(5)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(10)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // LRU(11)
				{}, // nil cache: FIFO(0)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 8}, // FIFO(1)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(5)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(10)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 5, Evictions: 0}, // FIFO(11)
				{}, // nil cache: Random(0)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 4}, // Random(1)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(5)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(10)
				{Gets: 9, Misses: 5, Puts: 9, Retains: 9, Evictions: 0}, // Random(11)
			},
		},
	}

	for k, pat := range patterns {
		// Use different caches.
		for j, s := range []Cache{
			nil, // Explicitly nil.

			cache.NewLRU(0), // Functionally nil.
			cache.NewLRU(1),
			cache.NewLRU(blocks / 2),
			cache.NewLRU(blocks),
			cache.NewLRU(blocks + 1),

			cache.NewFIFO(0), // Functionally nil.
			cache.NewFIFO(1),
			cache.NewFIFO(blocks / 2),
			cache.NewFIFO(blocks),
			cache.NewFIFO(blocks + 1),

			cache.NewRandom(0), // Functionally nil.
			cache.NewRandom(1),
			cache.NewRandom(blocks / 2),
			cache.NewRandom(blocks),
			cache.NewRandom(blocks + 1),
		} {
			var (
				buf     bytes.Buffer
				offsets = []int{0}
			)
			w := NewWriter(&buf, 1)
			for i := 0; i < blocks; i++ {
				if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil {
					t.Fatalf("Write: %v", err)
				}
				if err := w.Flush(); err != nil {
					t.Fatalf("Flush: %v", err)
				}
				if err := w.Wait(); err != nil {
					t.Fatalf("Wait: %v", err)
				}
				offsets = append(offsets, buf.Len())
			}
			w.Close()
			offsets = offsets[:len(offsets)-1]

			br := bytes.NewReader(buf.Bytes())
			// Insert a HasEOF to ensure it does not corrupt subsequent reads.
			HasEOF(br)

			r, err := NewReader(br, *conc)
			if err != nil {
				t.Fatalf("NewReader: %v", err)
			}
			var stats *cache.StatsRecorder
			if s != nil {
				stats = &cache.StatsRecorder{Cache: s}
				s = stats
			}
			r.SetCache(s)
			p := make([]byte, len(infix)+2)

			for _, op := range pat.ops {
				if op.seekBlock >= 0 {
					err := r.Seek(Offset{File: int64(offsets[op.seekBlock])})
					if err != nil {
						t.Fatalf("Seek: %v", err)
					}
				}
				n, err := r.Read(p)
				if n != len(p) {
					t.Errorf("Unexpected read length: got:%d want:%d", n, len(p))
				}
				if err != nil {
					t.Fatalf("Read: %v", err)
				}
				got := string(p)
				want := fmt.Sprintf("%d%[2]s%[1]d", op.blockID, infix)
				if got != want {
					t.Errorf("Unexpected result: got:%q want:%q", got, want)
				}
			}
			if stats != nil && stats.Stats() != pat.expectedStats[j] {
				t.Errorf("Unexpected result for cache %d pattern %d: got:%+v want:%+v", j, k, stats.Stats(), pat.expectedStats[j])
			}
			r.Close()
		}
	}
}
Ejemplo n.º 2
0
func TestSeekFast(t *testing.T) {
	// Under these conditions we cannot guarantee that a worker
	// will not read bytes after a Seek call has been made.
	if *conc != 1 && runtime.GOMAXPROCS(0) > 1 {
		return
	}
	const (
		infix  = "payload"
		blocks = 10
	)

	// Use different caches.
	for _, cache := range []Cache{
		nil, // Explicitly nil.

		cache.NewLRU(0), // Functionally nil.
		cache.NewLRU(1),
		cache.NewLRU(blocks / 2),
		cache.NewLRU(blocks),
		cache.NewLRU(blocks + 1),

		cache.NewRandom(0), // Functionally nil.
		cache.NewRandom(1),
		cache.NewRandom(blocks / 2),
		cache.NewRandom(blocks),
		cache.NewRandom(blocks + 1),
	} {
		var (
			buf     bytes.Buffer
			offsets = []int{0}
		)
		w := NewWriter(&buf, 1)
		for i := 0; i < blocks; i++ {
			if _, err := fmt.Fprintf(w, "%d%[2]s%[1]d", i, infix); err != nil {
				t.Fatalf("Write: %v", err)
			}
			if err := w.Flush(); err != nil {
				t.Fatalf("Flush: %v", err)
			}
			if err := w.Wait(); err != nil {
				t.Fatalf("Wait: %v", err)
			}
			offsets = append(offsets, buf.Len())
		}
		w.Close()
		offsets = offsets[:len(offsets)-1]

		c := &countReadSeeker{r: bytes.NewReader(buf.Bytes())}

		// Insert a HasEOF to ensure it does not corrupt subsequent reads.
		HasEOF(bytes.NewReader(buf.Bytes()))

		r, err := NewReader(c, *conc)
		if err != nil {
			t.Fatalf("NewReader: %v", err)
		}

		r.SetCache(cache)
		p := make([]byte, len(infix)+2)

		func() {
			defer func() {
				r := recover()
				if r != nil {
					t.Fatalf("Seek on unread reader panicked: %v", r)
				}
			}()
			err := r.Seek(Offset{})
			if err != nil {
				t.Fatalf("Seek: %v", err)
			}
		}()

		// Standard read through of the data.
		for i := range offsets {
			n, err := r.Read(p)
			if n != len(p) {
				t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
			}
			if err != nil {
				t.Fatalf("Read: %v", err)
			}
			got := string(p)
			want := fmt.Sprintf("%d%[2]s%[1]d", i, infix)
			if got != want {
				t.Errorf("Unexpected result: got:%q want:%q", got, want)
			}
		}

		// Seek to each block in turn
		for i, o := range offsets {
			err := r.Seek(Offset{File: int64(o)})
			if err != nil {
				t.Fatalf("Seek: %v", err)
			}
			n, err := r.Read(p)
			if n != len(p) {
				t.Errorf("Unexpected read length: got:%d want:%d", n, len(p))
			}
			if err != nil {
				t.Fatalf("Read: %v", err)
			}
			got := string(p)
			want := fmt.Sprintf("%d%[2]s%[1]d", i, infix)
			if got != want {
				t.Errorf("Unexpected result: got:%q want:%q", got, want)
			}
		}

		// Seek to each block in turn, but read the infix and then the first 2 bytes.
		for i, o := range offsets {
			if err := r.Seek(Offset{File: int64(o), Block: 1}); err != nil {
				t.Fatalf("Seek: %v", err)
			}
			p = p[:len(infix)]
			n, err := r.Read(p)
			if n != len(p) {
				t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
			}
			if err != nil {
				t.Fatalf("Read: %v", err)
			}
			got := string(p)
			want := infix
			if got != want {
				t.Fatalf("Unexpected result: got:%q want:%q", got, want)
			}

			// Check whether the underlying reader was seeked or read.
			hasRead := c.offset()
			if err = r.Seek(Offset{File: int64(o), Block: 0}); err != nil {
				t.Fatalf("Seek: %v", err)
			}
			if b := c.offset() - hasRead; b != 0 {
				t.Errorf("Seek performed unexpected read: %d bytes", b)
			}
			if c.didSeek() {
				t.Error("Seek caused underlying Seek.")
			}

			p = p[:2]
			n, err = r.Read(p)
			if n != len(p) {
				t.Fatalf("Unexpected read length: got:%d want:%d", n, len(p))
			}
			if err != nil {
				t.Fatalf("Read: %v", err)
			}
			got = string(p)
			want = fmt.Sprintf("%dp", i)
			if got != want {
				t.Fatalf("Unexpected result: got:%q want:%q", got, want)
			}
		}
		r.Close()
	}
}