// Should wrap four times
func TestWrapPut(t *testing.T) {
// Simple log
blocks := uint32(16)
testcachefile := tests.Tempfile()
err := tests.CreateFile(testcachefile, 16*4096)
tests.Assert(t, nil == err)
defer os.Remove(testcachefile)
l, logblocks, err := NewLog(testcachefile, 4096, 2, 4096*2, false)
tests.Assert(t, err == nil)
tests.Assert(t, l != nil)
tests.Assert(t, blocks == logblocks)
l.Start()
here := make(chan *message.Message)
wraps := uint32(4)
// Write enough blocks to wrap around the log
// as many times as determined by the value in 'wraps'
for io := uint32(0); io < (blocks * wraps); io++ {
buf := make([]byte, 4096)
buf[0] = byte(io)
msg := message.NewMsgPut()
msg.RetChan = here
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = io % blocks
l.Msgchan <- msg
<-here
}
// Close will also empty all the channels
l.Close()
// Check that we have wrapped the correct number of times
tests.Assert(t, l.Stats().Wraps == uint64(wraps))
}
func TestSpcOpen(t *testing.T) {
// initialize
var cache *cache.CacheMap
usedirectio := false
blocksize := 4 * KB
s := NewSpcInfo(cache, usedirectio, blocksize)
// Get a test file
tmpfile := tests.Tempfile()
// No file exists
err := s.Open(1, tmpfile)
tests.Assert(t, err != nil)
// Create the file and open it
err = tests.CreateFile(tmpfile, 16*4*KB)
tests.Assert(t, err == nil)
defer os.Remove(tmpfile)
// Now open, and it should work
err = s.Open(1, tmpfile)
tests.Assert(t, err == nil)
}
func TestLogConcurrency(t *testing.T) {
// Simple log
blocks := uint32(240)
bs := uint32(4096)
blocks_per_segment := uint32(2)
buffercache := uint32(4096 * 24)
testcachefile := tests.Tempfile()
tests.Assert(t, nil == tests.CreateFile(testcachefile, int64(blocks*4096)))
defer os.Remove(testcachefile)
l, logblocks, err := NewLog(testcachefile,
bs,
blocks_per_segment,
buffercache,
false)
tests.Assert(t, err == nil)
tests.Assert(t, l != nil)
tests.Assert(t, blocks == logblocks)
l.Start()
here := make(chan *message.Message)
// Fill the log
for io := uint32(0); io < blocks; io++ {
buf := make([]byte, 4096)
buf[0] = byte(io)
msg := message.NewMsgPut()
msg.RetChan = here
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = io
l.Msgchan <- msg
<-here
}
var wgIo, wgRet sync.WaitGroup
// Start up response server
returnch := make(chan *message.Message, 100)
quit := make(chan struct{})
wgRet.Add(1)
go logtest_response_handler(t, &wgRet, quit, returnch)
// Create 100 readers
for i := 0; i < 100; i++ {
wgIo.Add(1)
go func() {
defer wgIo.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
// Each client to send 1k IOs
for io := 0; io < 1000; io++ {
msg := message.NewMsgGet()
iopkt := msg.IoPkt()
iopkt.Buffer = make([]byte, bs)
// Maximum "disk" size is 10 times bigger than cache
iopkt.LogBlock = uint32(r.Int31n(int32(blocks)))
msg.RetChan = returnch
// Send request
msg.TimeStart()
l.Msgchan <- msg
// Simulate waiting for more work by sleeping
// anywhere from 100usecs to 10ms
time.Sleep(time.Microsecond * time.Duration((r.Intn(10000) + 100)))
}
}()
}
// Write to the log while the readers are reading
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for wrap := 0; wrap < 30; wrap++ {
for io := uint32(0); io < blocks; io++ {
buf := make([]byte, 4096)
buf[0] = byte(io)
msg := message.NewMsgPut()
msg.RetChan = returnch
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = io
msg.TimeStart()
l.Msgchan <- msg
time.Sleep(time.Microsecond * time.Duration((r.Intn(1000) + 100)))
}
}
// Wait for all clients to finish
wgIo.Wait()
// Send receiver a message that all clients have shut down
close(quit)
// Wait for receiver to finish emptying its channel
wgRet.Wait()
// Cleanup
fmt.Print(l)
l.Close()
os.Remove(testcachefile)
}
func TestReadCorrectness(t *testing.T) {
// Simple log
blocks := uint32(240)
bs := uint32(4096)
blocks_per_segment := uint32(2)
buffercache := uint32(4096 * 10)
testcachefile := tests.Tempfile()
tests.Assert(t, nil == tests.CreateFile(testcachefile, int64(blocks*4096)))
defer os.Remove(testcachefile)
l, logblocks, err := NewLog(testcachefile,
bs,
blocks_per_segment,
buffercache,
false)
tests.Assert(t, err == nil)
tests.Assert(t, l != nil)
tests.Assert(t, blocks == logblocks)
l.Start()
here := make(chan *message.Message)
// Write enough blocks in the log to reach
// the end.
for io := uint32(0); io < blocks; io++ {
buf := make([]byte, 4096)
// Save the block number in the buffer
// so that we can check it later. For simplicity
// we have made sure the block number is only
// one byte.
buf[0] = byte(io)
msg := message.NewMsgPut()
msg.RetChan = here
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = io
l.Msgchan <- msg
<-here
}
buf := make([]byte, 4096)
msg := message.NewMsgGet()
msg.RetChan = here
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = blocks - 1
l.Msgchan <- msg
<-here
tests.Assert(t, buf[0] == uint8(blocks-1))
for io := uint32(0); io < blocks; io++ {
buf := make([]byte, 4096)
msg := message.NewMsgGet()
msg.RetChan = here
iopkt := msg.IoPkt()
iopkt.Buffer = buf
iopkt.LogBlock = io
l.Msgchan <- msg
// Wait here for the response
<-here
// Check the block number is correct
tests.Assert(t, buf[0] == uint8(io))
}
l.Close()
}
// This test will check that the cache tries to place
// as many contigous blocks as possible. We will initialize
// the 8 slot cache with four slots, then remove slot 1 and 2
// to leave the following: [X__X____]
// When we put a message with 6 blocks the cache should be
// populated as follows: [X45X01234]
//
// At the end, check multiblock Get()
//
func TestCacheMapMultiblock(t *testing.T) {
// This service will run in its own goroutine
// and send to mocklog any messages
mocklog := make(chan *message.Message)
pipe := message.NewNullPipeline(mocklog)
pipe.Start()
defer pipe.Close()
c := NewCacheMap(8, 4096, pipe.In)
tests.Assert(t, c != nil)
here := make(chan *message.Message)
buffer := make([]byte, 4096)
// Initialize data in cache
for i := uint64(0); i < 4; i++ {
m := message.NewMsgPut()
m.RetChan = here
io := m.IoPkt()
io.Buffer = buffer
io.Address = i
// First Put
err := c.Put(m)
tests.Assert(t, err == nil)
retmsg := <-mocklog
retmsg.Done()
<-here
}
c.Invalidate(&message.IoPkt{Address: 1, Blocks: 2})
tests.Assert(t, c.stats.insertions == 4)
tests.Assert(t, c.stats.invalidatehits == 2)
tests.Assert(t, c.bda.bds[0].used == true)
tests.Assert(t, c.bda.bds[0].key == 0)
tests.Assert(t, c.bda.bds[1].used == false)
tests.Assert(t, c.bda.bds[2].used == false)
tests.Assert(t, c.bda.bds[3].used == true)
tests.Assert(t, c.bda.bds[3].key == 3)
// Set the clock so they do not get erased
c.bda.bds[0].clock_set = true
c.bda.bds[3].clock_set = true
// Insert multiblock
largebuffer := make([]byte, 6*4096)
m := message.NewMsgPut()
m.RetChan = here
io := m.IoPkt()
io.Buffer = largebuffer
io.Address = 10
io.Blocks = 6
// First Put
err := c.Put(m)
tests.Assert(t, err == nil)
for i := uint32(0); i < io.Blocks; i++ {
// Put send a message for each block
retmsg := <-mocklog
retmsg.Done()
}
<-here
tests.Assert(t, c.stats.insertions == 10)
tests.Assert(t, c.stats.invalidatehits == 2)
// Check the two blocks left from before
tests.Assert(t, c.bda.bds[0].used == true)
tests.Assert(t, c.bda.bds[0].key == 0)
tests.Assert(t, c.bda.bds[0].clock_set == false)
tests.Assert(t, c.bda.bds[3].used == true)
tests.Assert(t, c.bda.bds[3].key == 3)
tests.Assert(t, c.bda.bds[3].clock_set == true)
// Now check the blocks we inserted
tests.Assert(t, c.bda.bds[4].used == true)
tests.Assert(t, c.bda.bds[4].key == 10)
tests.Assert(t, c.bda.bds[4].clock_set == false)
tests.Assert(t, c.bda.bds[5].used == true)
tests.Assert(t, c.bda.bds[5].key == 11)
tests.Assert(t, c.bda.bds[5].clock_set == false)
tests.Assert(t, c.bda.bds[6].used == true)
tests.Assert(t, c.bda.bds[6].key == 12)
tests.Assert(t, c.bda.bds[6].clock_set == false)
tests.Assert(t, c.bda.bds[7].used == true)
tests.Assert(t, c.bda.bds[7].key == 13)
tests.Assert(t, c.bda.bds[7].clock_set == false)
tests.Assert(t, c.bda.bds[1].used == true)
tests.Assert(t, c.bda.bds[1].key == 14)
tests.Assert(t, c.bda.bds[1].clock_set == false)
tests.Assert(t, c.bda.bds[2].used == true)
tests.Assert(t, c.bda.bds[2].key == 15)
tests.Assert(t, c.bda.bds[2].clock_set == false)
// Check for a block not in the cache
m = message.NewMsgGet()
m.RetChan = here
io = m.IoPkt()
io.Buffer = buffer
io.Address = 20
io.Blocks = 1
hitmap, err := c.Get(m)
tests.Assert(t, err == ErrNotFound)
tests.Assert(t, hitmap == nil)
tests.Assert(t, len(here) == 0)
// Get offset 0, 4 blocks. It should return
// a bit map of [1001]
buffer4 := make([]byte, 4*4096)
m = message.NewMsgGet()
m.RetChan = here
io = m.IoPkt()
io.Buffer = buffer4
io.Address = 0
io.Blocks = 4
hitmap, err = c.Get(m)
tests.Assert(t, err == nil)
tests.Assert(t, hitmap.Hits == 2)
tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks))
tests.Assert(t, hitmap.Hitmap[0] == true)
tests.Assert(t, hitmap.Hitmap[1] == false)
tests.Assert(t, hitmap.Hitmap[2] == false)
tests.Assert(t, hitmap.Hitmap[3] == true)
for i := 0; i < 2; i++ {
// Get sends a get for each contiguous blocks
retmsg := <-mocklog
retmsg.Done()
}
<-here
// Get the 6 blocks we inserted previously. This
// should show that there are two sets of continguous
// blocks
m = message.NewMsgGet()
m.RetChan = here
io = m.IoPkt()
io.Buffer = largebuffer
io.Address = 10
io.Blocks = 6
hitmap, err = c.Get(m)
tests.Assert(t, err == nil)
tests.Assert(t, hitmap.Hits == 6)
tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks))
tests.Assert(t, hitmap.Hitmap[0] == true)
tests.Assert(t, hitmap.Hitmap[1] == true)
tests.Assert(t, hitmap.Hitmap[2] == true)
tests.Assert(t, hitmap.Hitmap[3] == true)
tests.Assert(t, hitmap.Hitmap[4] == true)
tests.Assert(t, hitmap.Hitmap[5] == true)
// The first message to the log
retmsg := <-mocklog
retio := retmsg.IoPkt()
tests.Assert(t, retmsg.RetChan == nil)
tests.Assert(t, retio.Address == 10)
tests.Assert(t, retio.LogBlock == 4)
tests.Assert(t, retio.Blocks == 4)
retmsg.Done()
// Second message will have the rest of the contigous block
retmsg = <-mocklog
retio = retmsg.IoPkt()
tests.Assert(t, retmsg.RetChan == nil)
tests.Assert(t, retio.Address == 14)
tests.Assert(t, retio.LogBlock == 1)
tests.Assert(t, retio.Blocks == 2)
retmsg.Done()
<-here
// Save the cache metadata
save := tests.Tempfile()
defer os.Remove(save)
err = c.Save(save, nil)
tests.Assert(t, err == nil)
c.Close()
c = NewCacheMap(8, 4096, pipe.In)
tests.Assert(t, c != nil)
err = c.Load(save, nil)
tests.Assert(t, err == nil)
// Get data again.
m = message.NewMsgGet()
m.RetChan = here
io = m.IoPkt()
io.Buffer = largebuffer
io.Address = 10
io.Blocks = 6
hitmap, err = c.Get(m)
tests.Assert(t, err == nil)
tests.Assert(t, hitmap.Hits == 6)
tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks))
tests.Assert(t, hitmap.Hitmap[0] == true)
tests.Assert(t, hitmap.Hitmap[1] == true)
tests.Assert(t, hitmap.Hitmap[2] == true)
tests.Assert(t, hitmap.Hitmap[3] == true)
tests.Assert(t, hitmap.Hitmap[4] == true)
tests.Assert(t, hitmap.Hitmap[5] == true)
// The first message to the log
retmsg = <-mocklog
retio = retmsg.IoPkt()
tests.Assert(t, retmsg.RetChan == nil)
tests.Assert(t, retio.Address == 10)
tests.Assert(t, retio.LogBlock == 4)
tests.Assert(t, retio.Blocks == 4)
retmsg.Done()
// Second message will have the rest of the contigous block
retmsg = <-mocklog
retio = retmsg.IoPkt()
tests.Assert(t, retmsg.RetChan == nil)
tests.Assert(t, retio.Address == 14)
tests.Assert(t, retio.LogBlock == 1)
tests.Assert(t, retio.Blocks == 2)
retmsg.Done()
<-here
c.Close()
}
