// Should wrap four times func TestWrapPut(t *testing.T) { // Simple log blocks := uint32(16) testcachefile := tests.Tempfile() err := tests.CreateFile(testcachefile, 16*4096) tests.Assert(t, nil == err) defer os.Remove(testcachefile) l, logblocks, err := NewLog(testcachefile, 4096, 2, 4096*2, false) tests.Assert(t, err == nil) tests.Assert(t, l != nil) tests.Assert(t, blocks == logblocks) l.Start() here := make(chan *message.Message) wraps := uint32(4) // Write enough blocks to wrap around the log // as many times as determined by the value in 'wraps' for io := uint32(0); io < (blocks * wraps); io++ { buf := make([]byte, 4096) buf[0] = byte(io) msg := message.NewMsgPut() msg.RetChan = here iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = io % blocks l.Msgchan <- msg <-here } // Close will also empty all the channels l.Close() // Check that we have wrapped the correct number of times tests.Assert(t, l.Stats().Wraps == uint64(wraps)) }
func TestSpcOpen(t *testing.T) { // initialize var cache *cache.CacheMap usedirectio := false blocksize := 4 * KB s := NewSpcInfo(cache, usedirectio, blocksize) // Get a test file tmpfile := tests.Tempfile() // No file exists err := s.Open(1, tmpfile) tests.Assert(t, err != nil) // Create the file and open it err = tests.CreateFile(tmpfile, 16*4*KB) tests.Assert(t, err == nil) defer os.Remove(tmpfile) // Now open, and it should work err = s.Open(1, tmpfile) tests.Assert(t, err == nil) }
func TestLogConcurrency(t *testing.T) { // Simple log blocks := uint32(240) bs := uint32(4096) blocks_per_segment := uint32(2) buffercache := uint32(4096 * 24) testcachefile := tests.Tempfile() tests.Assert(t, nil == tests.CreateFile(testcachefile, int64(blocks*4096))) defer os.Remove(testcachefile) l, logblocks, err := NewLog(testcachefile, bs, blocks_per_segment, buffercache, false) tests.Assert(t, err == nil) tests.Assert(t, l != nil) tests.Assert(t, blocks == logblocks) l.Start() here := make(chan *message.Message) // Fill the log for io := uint32(0); io < blocks; io++ { buf := make([]byte, 4096) buf[0] = byte(io) msg := message.NewMsgPut() msg.RetChan = here iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = io l.Msgchan <- msg <-here } var wgIo, wgRet sync.WaitGroup // Start up response server returnch := make(chan *message.Message, 100) quit := make(chan struct{}) wgRet.Add(1) go logtest_response_handler(t, &wgRet, quit, returnch) // Create 100 readers for i := 0; i < 100; i++ { wgIo.Add(1) go func() { defer wgIo.Done() r := rand.New(rand.NewSource(time.Now().UnixNano())) // Each client to send 1k IOs for io := 0; io < 1000; io++ { msg := message.NewMsgGet() iopkt := msg.IoPkt() iopkt.Buffer = make([]byte, bs) // Maximum "disk" size is 10 times bigger than cache iopkt.LogBlock = uint32(r.Int31n(int32(blocks))) msg.RetChan = returnch // Send request msg.TimeStart() l.Msgchan <- msg // Simulate waiting for more work by sleeping // anywhere from 100usecs to 10ms time.Sleep(time.Microsecond * time.Duration((r.Intn(10000) + 100))) } }() } // Write to the log while the readers are reading r := rand.New(rand.NewSource(time.Now().UnixNano())) for wrap := 0; wrap < 30; wrap++ { for io := uint32(0); io < blocks; io++ { buf := make([]byte, 4096) buf[0] = byte(io) msg := message.NewMsgPut() msg.RetChan = returnch iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = io msg.TimeStart() l.Msgchan <- msg time.Sleep(time.Microsecond * time.Duration((r.Intn(1000) + 100))) } } // Wait for all clients to finish wgIo.Wait() // Send receiver a message that all clients have shut down close(quit) // Wait for receiver to finish emptying its channel wgRet.Wait() // Cleanup fmt.Print(l) l.Close() os.Remove(testcachefile) }
func TestReadCorrectness(t *testing.T) { // Simple log blocks := uint32(240) bs := uint32(4096) blocks_per_segment := uint32(2) buffercache := uint32(4096 * 10) testcachefile := tests.Tempfile() tests.Assert(t, nil == tests.CreateFile(testcachefile, int64(blocks*4096))) defer os.Remove(testcachefile) l, logblocks, err := NewLog(testcachefile, bs, blocks_per_segment, buffercache, false) tests.Assert(t, err == nil) tests.Assert(t, l != nil) tests.Assert(t, blocks == logblocks) l.Start() here := make(chan *message.Message) // Write enough blocks in the log to reach // the end. for io := uint32(0); io < blocks; io++ { buf := make([]byte, 4096) // Save the block number in the buffer // so that we can check it later. For simplicity // we have made sure the block number is only // one byte. buf[0] = byte(io) msg := message.NewMsgPut() msg.RetChan = here iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = io l.Msgchan <- msg <-here } buf := make([]byte, 4096) msg := message.NewMsgGet() msg.RetChan = here iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = blocks - 1 l.Msgchan <- msg <-here tests.Assert(t, buf[0] == uint8(blocks-1)) for io := uint32(0); io < blocks; io++ { buf := make([]byte, 4096) msg := message.NewMsgGet() msg.RetChan = here iopkt := msg.IoPkt() iopkt.Buffer = buf iopkt.LogBlock = io l.Msgchan <- msg // Wait here for the response <-here // Check the block number is correct tests.Assert(t, buf[0] == uint8(io)) } l.Close() }
// This test will check that the cache tries to place // as many contigous blocks as possible. We will initialize // the 8 slot cache with four slots, then remove slot 1 and 2 // to leave the following: [X__X____] // When we put a message with 6 blocks the cache should be // populated as follows: [X45X01234] // // At the end, check multiblock Get() // func TestCacheMapMultiblock(t *testing.T) { // This service will run in its own goroutine // and send to mocklog any messages mocklog := make(chan *message.Message) pipe := message.NewNullPipeline(mocklog) pipe.Start() defer pipe.Close() c := NewCacheMap(8, 4096, pipe.In) tests.Assert(t, c != nil) here := make(chan *message.Message) buffer := make([]byte, 4096) // Initialize data in cache for i := uint64(0); i < 4; i++ { m := message.NewMsgPut() m.RetChan = here io := m.IoPkt() io.Buffer = buffer io.Address = i // First Put err := c.Put(m) tests.Assert(t, err == nil) retmsg := <-mocklog retmsg.Done() <-here } c.Invalidate(&message.IoPkt{Address: 1, Blocks: 2}) tests.Assert(t, c.stats.insertions == 4) tests.Assert(t, c.stats.invalidatehits == 2) tests.Assert(t, c.bda.bds[0].used == true) tests.Assert(t, c.bda.bds[0].key == 0) tests.Assert(t, c.bda.bds[1].used == false) tests.Assert(t, c.bda.bds[2].used == false) tests.Assert(t, c.bda.bds[3].used == true) tests.Assert(t, c.bda.bds[3].key == 3) // Set the clock so they do not get erased c.bda.bds[0].clock_set = true c.bda.bds[3].clock_set = true // Insert multiblock largebuffer := make([]byte, 6*4096) m := message.NewMsgPut() m.RetChan = here io := m.IoPkt() io.Buffer = largebuffer io.Address = 10 io.Blocks = 6 // First Put err := c.Put(m) tests.Assert(t, err == nil) for i := uint32(0); i < io.Blocks; i++ { // Put send a message for each block retmsg := <-mocklog retmsg.Done() } <-here tests.Assert(t, c.stats.insertions == 10) tests.Assert(t, c.stats.invalidatehits == 2) // Check the two blocks left from before tests.Assert(t, c.bda.bds[0].used == true) tests.Assert(t, c.bda.bds[0].key == 0) tests.Assert(t, c.bda.bds[0].clock_set == false) tests.Assert(t, c.bda.bds[3].used == true) tests.Assert(t, c.bda.bds[3].key == 3) tests.Assert(t, c.bda.bds[3].clock_set == true) // Now check the blocks we inserted tests.Assert(t, c.bda.bds[4].used == true) tests.Assert(t, c.bda.bds[4].key == 10) tests.Assert(t, c.bda.bds[4].clock_set == false) tests.Assert(t, c.bda.bds[5].used == true) tests.Assert(t, c.bda.bds[5].key == 11) tests.Assert(t, c.bda.bds[5].clock_set == false) tests.Assert(t, c.bda.bds[6].used == true) tests.Assert(t, c.bda.bds[6].key == 12) tests.Assert(t, c.bda.bds[6].clock_set == false) tests.Assert(t, c.bda.bds[7].used == true) tests.Assert(t, c.bda.bds[7].key == 13) tests.Assert(t, c.bda.bds[7].clock_set == false) tests.Assert(t, c.bda.bds[1].used == true) tests.Assert(t, c.bda.bds[1].key == 14) tests.Assert(t, c.bda.bds[1].clock_set == false) tests.Assert(t, c.bda.bds[2].used == true) tests.Assert(t, c.bda.bds[2].key == 15) tests.Assert(t, c.bda.bds[2].clock_set == false) // Check for a block not in the cache m = message.NewMsgGet() m.RetChan = here io = m.IoPkt() io.Buffer = buffer io.Address = 20 io.Blocks = 1 hitmap, err := c.Get(m) tests.Assert(t, err == ErrNotFound) tests.Assert(t, hitmap == nil) tests.Assert(t, len(here) == 0) // Get offset 0, 4 blocks. It should return // a bit map of [1001] buffer4 := make([]byte, 4*4096) m = message.NewMsgGet() m.RetChan = here io = m.IoPkt() io.Buffer = buffer4 io.Address = 0 io.Blocks = 4 hitmap, err = c.Get(m) tests.Assert(t, err == nil) tests.Assert(t, hitmap.Hits == 2) tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks)) tests.Assert(t, hitmap.Hitmap[0] == true) tests.Assert(t, hitmap.Hitmap[1] == false) tests.Assert(t, hitmap.Hitmap[2] == false) tests.Assert(t, hitmap.Hitmap[3] == true) for i := 0; i < 2; i++ { // Get sends a get for each contiguous blocks retmsg := <-mocklog retmsg.Done() } <-here // Get the 6 blocks we inserted previously. This // should show that there are two sets of continguous // blocks m = message.NewMsgGet() m.RetChan = here io = m.IoPkt() io.Buffer = largebuffer io.Address = 10 io.Blocks = 6 hitmap, err = c.Get(m) tests.Assert(t, err == nil) tests.Assert(t, hitmap.Hits == 6) tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks)) tests.Assert(t, hitmap.Hitmap[0] == true) tests.Assert(t, hitmap.Hitmap[1] == true) tests.Assert(t, hitmap.Hitmap[2] == true) tests.Assert(t, hitmap.Hitmap[3] == true) tests.Assert(t, hitmap.Hitmap[4] == true) tests.Assert(t, hitmap.Hitmap[5] == true) // The first message to the log retmsg := <-mocklog retio := retmsg.IoPkt() tests.Assert(t, retmsg.RetChan == nil) tests.Assert(t, retio.Address == 10) tests.Assert(t, retio.LogBlock == 4) tests.Assert(t, retio.Blocks == 4) retmsg.Done() // Second message will have the rest of the contigous block retmsg = <-mocklog retio = retmsg.IoPkt() tests.Assert(t, retmsg.RetChan == nil) tests.Assert(t, retio.Address == 14) tests.Assert(t, retio.LogBlock == 1) tests.Assert(t, retio.Blocks == 2) retmsg.Done() <-here // Save the cache metadata save := tests.Tempfile() defer os.Remove(save) err = c.Save(save, nil) tests.Assert(t, err == nil) c.Close() c = NewCacheMap(8, 4096, pipe.In) tests.Assert(t, c != nil) err = c.Load(save, nil) tests.Assert(t, err == nil) // Get data again. m = message.NewMsgGet() m.RetChan = here io = m.IoPkt() io.Buffer = largebuffer io.Address = 10 io.Blocks = 6 hitmap, err = c.Get(m) tests.Assert(t, err == nil) tests.Assert(t, hitmap.Hits == 6) tests.Assert(t, len(hitmap.Hitmap) == int(io.Blocks)) tests.Assert(t, hitmap.Hitmap[0] == true) tests.Assert(t, hitmap.Hitmap[1] == true) tests.Assert(t, hitmap.Hitmap[2] == true) tests.Assert(t, hitmap.Hitmap[3] == true) tests.Assert(t, hitmap.Hitmap[4] == true) tests.Assert(t, hitmap.Hitmap[5] == true) // The first message to the log retmsg = <-mocklog retio = retmsg.IoPkt() tests.Assert(t, retmsg.RetChan == nil) tests.Assert(t, retio.Address == 10) tests.Assert(t, retio.LogBlock == 4) tests.Assert(t, retio.Blocks == 4) retmsg.Done() // Second message will have the rest of the contigous block retmsg = <-mocklog retio = retmsg.IoPkt() tests.Assert(t, retmsg.RetChan == nil) tests.Assert(t, retio.Address == 14) tests.Assert(t, retio.LogBlock == 1) tests.Assert(t, retio.Blocks == 2) retmsg.Done() <-here c.Close() }