func (mngr *mMapBufferManager) Acquire(offset, length int64) ([]byte, error) {
	if int64(pageSize)%length != 0 || offset%length != 0 {
		return nil, UnalignedBuffer{offset, length}
	}

	page, err := pageForSection(offset, length)
	if err != nil {
		return nil, err
	}

	relOffset := offset % int64(pageSize)

	mmap, err := gommap.MapAt(0, mngr.fd.Fd(), int64(page)*int64(pageSize), int64(pageSize), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
	return []byte(mmap)[relOffset : relOffset+length], err
}
func NewMappedBloomSHA(m, k uint, backingFile string) (
	mb3 *MappedBloomSHA, err error) {

	var (
		f           *os.File
		filterBits  uint
		filterWords uint
		size        int64
		Filter      []uint64
		inCore      gm.MMap
		b3          *BloomSHA
	)
	if m < MIN_M || m > MAX_M {
		err = MOutOfRange
	} else {
		filterBits = uint(1) << m
		filterWords = filterBits / 64
		size = int64(filterBits / 8) // bytes
	}
	if err == nil && (k < MIN_K || (k*m > MAX_MK_PRODUCT)) {
		err = TooManyHashFunctions
	}
	if err == nil {
		var found bool
		found, err = xf.PathExists(backingFile)
		if err == nil {
			if found {
				f, err = os.OpenFile(backingFile,
					os.O_RDWR, 0640)
				if err == nil {
					var fi os.FileInfo
					fi, err = f.Stat()
					if err == nil {
						// XXX could be huge ...
						fileSize := int64(fi.Size())
						if fileSize < size {
							err = MappingFileTooSmall
						}
					}
				}
			} else {
				// ! found {
				f, err = os.OpenFile(backingFile,
					os.O_CREATE|os.O_RDWR, 0640)

				// XXX should write blocks in a loop
				zeroes := make([]byte, size)
				err = ioutil.WriteFile(backingFile, zeroes, 0640)
			}
		}
	}
	if err == nil {
		// 2013-11-01: was getting EPERM errors here because I used
		// os.Open to open the backing file.  Switched to OpenFile
		// with explicit permissions and everything worked.
		inCore, err = gm.MapAt(0, f.Fd(), 0, size,
			gm.PROT_READ|gm.PROT_WRITE,
			gm.MAP_SHARED)
	}
	if err == nil {
		ih := (*reflect.SliceHeader)(unsafe.Pointer(&inCore))
		fh := (*reflect.SliceHeader)(unsafe.Pointer(&Filter))
		fh.Data = ih.Data               // Filter slice points at same data
		fh.Len = ih.Len / SIZEOF_UINT64 // length suitably modified
		fh.Cap = ih.Cap / SIZEOF_UINT64 // likewise for capacity

		b3 = &BloomSHA{
			m:      m,
			k:      k,
			Filter: Filter,

			// comments say these are convenience variables but they
			// are actually used
			filterBits:  filterBits,
			filterWords: filterWords,
		}
		b3.doClear() // no lock
	}
	if err == nil {
		mb3 = &MappedBloomSHA{
			backingFile: backingFile,
			f:           f,
			inCore:      inCore,
			BloomSHA:    *b3,
		}
	}
	return
}
Beispiel #3
0
func (s *XLSuite) TestMmap(c *C) {
	if VERBOSITY > 0 {
		fmt.Println("\nTEST_MMAP")
	}

	rng := xr.MakeSimpleRNG()
	pathToFile := s.scratchFileName(c, rng, "tmp")

	// XXX take care: this is ONE block
	data := make([]byte, BLOCK_SIZE)
	rng.NextBytes(data)
	err := ioutil.WriteFile(pathToFile, data, 0644)
	c.Assert(err, IsNil)

	f, err := os.OpenFile(pathToFile, os.O_CREATE|os.O_RDWR, 0644)
	c.Assert(err, IsNil)

	// XXX Changing this from gm.MAP_PRIVATE to gm.MAP_SHARED made
	// the tests at the bottom succeed.  That is, changes made to
	// memory were written to disk by the Sync.
	inCore, err := gm.MapAt(0, f.Fd(), 0, 2*BLOCK_SIZE,
		gm.PROT_READ|gm.PROT_WRITE, gm.MAP_SHARED)
	c.Assert(err, IsNil)
	c.Assert(inCore, Not(IsNil))
	// The next succeeds, so it has grabbed that much memory ...
	c.Assert(len(inCore), Equals, 2*BLOCK_SIZE)

	// these are per-block flags
	boolz, err := inCore.IsResident()
	c.Assert(err, IsNil)
	c.Assert(boolz[0], Equals, true)

	// HACKING ABOUT ////////////////////////////////////////////////
	//for i := 0; i < BLOCK_SIZE; i++ {
	//	inCore[i] = byte(0)
	//}

	// inCore is an MMap
	mh := (*reflect.SliceHeader)(unsafe.Pointer(&inCore))

	const SIZEOF_UINT64 = 8 // bytes

	type MMap64 []uint64
	var inCore64 MMap64
	ih := (*reflect.SliceHeader)(unsafe.Pointer(&inCore64))

	ih.Data = mh.Data
	ih.Len = mh.Len / SIZEOF_UINT64
	ih.Cap = mh.Cap / SIZEOF_UINT64

	inCore[0] = 0x7f
	// END HACKING //////////////////////////////////////////////////

	// This succeeds, so the mapping from disk succeeded.
	c.Assert(bytes.Equal(inCore[1:BLOCK_SIZE], data[1:]), Equals, true)

	const (
		ASCII_A = byte(64)
	)
	inCore[BLOCK_SIZE-1] = ASCII_A
	inCore.Sync(gm.MS_SYNC) // should block

	// With the change to gm.MAP_SHARED, this does not seem to be
	// necessary:
	//
	// if the Sync didn't flush the ASCII_A to disk, this should do it.
	//err = inCore.UnsafeUnmap()
	// c.Assert(err, IsNil)

	f.Close()

	data2, err := ioutil.ReadFile(pathToFile)
	c.Assert(err, IsNil)

	// if this succeeds, then the flush to disk succeeded
	c.Assert(data2[BLOCK_SIZE-1], Equals, ASCII_A)
}