Example #1
0
File: reader.go Project: rboyer/wal
func newReader(fn string) (*reader, error) {
	f, err := os.OpenFile(fn, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	var data gommap.MMap
	var zero bool
	if fi.Size() <= 0 {
		zero = true
	} else {
		data, err = gommap.Map(f.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
		if err != nil {
			return nil, err
		}
	}

	r := &reader{
		cs:   crc32.NewIEEE(),
		data: data,
		zero: zero,
		done: zero, // noop reader if zero-byte file
	}
	r.bv.Data = data

	return r, nil
}
Example #2
0
func (s *S) TestIsResidentTwoPages(c *C) {
	testPath := path.Join(c.MkDir(), "test.txt")
	file, err := os.Create(testPath)
	c.Assert(err, IsNil)
	defer file.Close()

	file.Seek(int64(os.Getpagesize()*2-1), 0)
	file.Write([]byte{'x'})

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()

	// Not entirely a stable test, but should usually work.

	mmap[len(mmap)-1] = 'x'

	mapped, err := mmap.IsResident()
	c.Assert(err, IsNil)
	c.Assert(mapped, DeepEquals, []bool{false, true})

	mmap[0] = 'x'

	mapped, err = mmap.IsResident()
	c.Assert(err, IsNil)
	c.Assert(mapped, DeepEquals, []bool{true, true})
}
Example #3
0
func (s *S) TestIsResidentUnderOnePage(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()

	mapped, err := mmap.IsResident()
	c.Assert(err, IsNil)
	c.Assert(mapped, DeepEquals, []bool{true})
}
Example #4
0
func (s *S) TestProtFlagsAndErr(c *C) {
	testPath := s.file.Name()
	s.file.Close()
	file, err := os.Open(testPath)
	c.Assert(err, IsNil)
	s.file = file
	_, err = gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
	// For this to happen, both the error and the protection flag must work.
	c.Assert(err, Equals, syscall.EACCES)
}
Example #5
0
func fixFile() {
	file, err := os.Open("./data/0000.txt")
	check(err)

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	check(err)

	numLines, err := countLines(bytes.NewReader(mmap))
	check(err)

	lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)

	lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")

	// dear lord, fix this
	sub := [][][]byte{
		lines[:(numLines / 4)],
		lines[(numLines / 4):(numLines / 2)],
		lines[(numLines / 2) : (numLines/2)+(numLines/4)],
		lines[(numLines/2)+(numLines/4) : numLines],
	}

	jobs := make(chan []byte)
	results := make(chan string)

	wg := new(sync.WaitGroup)
	for w := 0; w <= 3; w++ {
		wg.Add(1)
		go normalizeLines(jobs, results, wg)
	}

	go func() {
		for i := 0; i <= 3; i++ {
			jobs <- bytes.Join(sub[i], []byte{'\n'})
		}
		close(jobs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	for v := range results {
		if strings.HasPrefix(v, "trimmed:") {
			//fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else if strings.HasPrefix(v, "bad:") {
			fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
		} else {
			//fmt.Fprintf(os.Stdout, "%s\n", v)
		}
	}

}
Example #6
0
func BenchmarkMmapReader(b *testing.B) {
	file := createDBFile()
	defer os.Remove(file.Name())
	defer file.Close()
	m, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
	if err != nil {
		b.Fatal(err)
	}
	defer m.UnsafeUnmap()

	benchReader(b, New(bytes.NewReader(m)))
}
Example #7
0
func (s *S) TestAdvise(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()

	// A bit tricky to blackbox-test these.
	err = mmap.Advise(gommap.MADV_RANDOM)
	c.Assert(err, IsNil)

	err = mmap.Advise(9999)
	c.Assert(err, ErrorMatches, "invalid argument")
}
Example #8
0
func (s *S) TestProtect(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()
	c.Assert([]byte(mmap), DeepEquals, testData)

	err = mmap.Protect(gommap.PROT_READ | gommap.PROT_WRITE)
	c.Assert(err, IsNil)

	// If this operation doesn't blow up tests, the call above worked.
	mmap[9] = 'X'
}
Example #9
0
func (s *S) TestFlags(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()

	mmap[9] = 'X'
	mmap.Sync(gommap.MS_SYNC)

	fileData, err := ioutil.ReadFile(s.file.Name())
	c.Assert(err, IsNil)
	// Shouldn't have written, since the map is private.
	c.Assert(fileData, DeepEquals, []byte("0123456789ABCDEF"))
}
Example #10
0
func (s *S) TestSliceMethods(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()
	c.Assert([]byte(mmap), DeepEquals, testData)

	mmap[9] = 'X'
	mmap[7:10].Sync(gommap.MS_SYNC)

	fileData, err := ioutil.ReadFile(s.file.Name())
	c.Assert(err, IsNil)
	c.Assert(fileData, DeepEquals, []byte("012345678XABCDEF"))
}
Example #11
0
func (s *S) TestLock(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
	c.Assert(err, IsNil)
	defer mmap.UnsafeUnmap()

	// A bit tricky to blackbox-test these.
	err = mmap.Lock()
	c.Assert(err, IsNil)

	err = mmap.Lock()
	c.Assert(err, IsNil)

	err = mmap.Unlock()
	c.Assert(err, IsNil)

	err = mmap.Unlock()
	c.Assert(err, IsNil)
}
Example #12
0
func GetFileHash(filepath string) string {

	fi, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer fi.Close()

	running_hash := md5.New()
	//io.Copy(running_hash, fi)
	mmap, err := gommap.Map(fi.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	if err != nil {
		panic(err)
	}
	//mmapReader := bytes.NewReader(mmap)
	//io.Copy(running_hash, mmapReader)
	buf := bytes.NewBuffer(mmap)
	io.Copy(running_hash, buf)

	return fmt.Sprintf("%x", running_hash.Sum(nil))
}
Example #13
0
func MMapTyped(f *File, val interface{}) (interface{}, gommap.MMap, error) {
	stat, err := f.Stat()
	if err != nil {
		return nil, nil, err
	}

	typ := reflect.TypeOf(val)
	typSize := int64(typ.Size())
	if stat.Size()%typSize != 0 {
		return nil, nil, errors.New("Shm region size not a multiple of value type")
	}

	buf, err := gommap.Map(f.File.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
	if err != nil {
		return nil, nil, err
	}

	header := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
	header.Len /= int(typSize)
	header.Cap /= int(typSize)

	ns := reflect.NewAt(reflect.SliceOf(typ), unsafe.Pointer(&header))
	return reflect.Indirect(ns).Interface(), buf, nil
}
Example #14
0
func (c *clock) createBuffers() error {
	g := c.shmGlobal
	c.shm = c.wlc.NewShm(c)
	if err := c.registry.Bind(g.Name, g.Interface, g.Version, c.shm.Id()); err != nil {
		return errgo.Trace(err)
	}

	// collect shm formats
	if err := c.sync(); err != nil {
		return errgo.Trace(err)
	}

	// allocate pool for 2 buffers
	poolSize := int32(len(c.buffers)) * c.bufSize

	shmO, err := shm.Open("clock", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
	if err != nil {
		return errgo.Trace(err)
	}
	defer shmO.Close()
	if err := shmO.Truncate(int64(poolSize)); err != nil {
		return errgo.Trace(err)
	}

	if c.bufsMap, err = gommap.Map(shmO.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED); err != nil {
		return errgo.Trace(err)
	}

	shmPool := c.wlc.NewShmPool(c)
	if err := c.shm.CreatePool(shmPool.Id(), shmO.Fd(), poolSize); err != nil {
		return errgo.Trace(err)
	}

	for i := range c.buffers {
		buf := new(buffer)
		buf.img.Rect = image.Rect(0, 0, int(c.w), int(c.h))
		buf.img.Stride = int(c.w) * 4
		buf.img.Pix = c.bufsMap[i*int(c.bufSize) : (i+1)*int(c.bufSize)]

		buf.ClientBuffer = c.wlc.NewBuffer(buf)
		if err := shmPool.CreateBuffer(
			buf.Id(),           // Id
			int32(i)*c.bufSize, // Offset
			c.w,                // Width
			c.h,                // Height
			c.stride,           // Stride
			0,                  // Format
		); err != nil {
			return errgo.Trace(err)
		}
		c.buffers[i] = buf
	}

	//if err := shmPool.Destroy(); err != nil {
	//    return errgo.Trace(err)
	//}

	if err := c.sync(); err != nil {
		return errgo.Trace(err)
	}

	return nil
}
Example #15
0
func main() {
	var (
		fileOpt,
		dataDirOpt,
		databaseName,
		profileFile,
		metadataStr string
		metadata                    = index.Metadata{}
		helpOpt, newIndex, debugOpt bool
		err                         error
		index                       *index.Index
		batchSize                   int
	)

	optarg.Header("General options")
	optarg.Add("f", "file", "Read NeoSearch JSON database from file. (Required)", "")
	optarg.Add("c", "create", "Create new index database", false)
	optarg.Add("b", "batch-size", "Batch size", 1000)
	optarg.Add("n", "name", "Name of index database", "")
	optarg.Add("d", "data-dir", "Data directory", "")
	optarg.Add("t", "trace-debug", "Enable trace for debug", false)
	optarg.Add("h", "help", "Display this help", false)
	optarg.Add("p", "cpuprofile", "write cpu profile to file", "")
	optarg.Add("m", "metadata", "metadata of documents", "")

	for opt := range optarg.Parse() {
		switch opt.ShortName {
		case "f":
			fileOpt = opt.String()
		case "b":
			batchSize = opt.Int()
		case "d":
			dataDirOpt = opt.String()
		case "n":
			databaseName = opt.String()
		case "c":
			newIndex = true
		case "t":
			debugOpt = true
		case "p":
			profileFile = opt.String()
		case "m":
			metadataStr = opt.String()
		case "h":
			helpOpt = true
		}
	}

	if helpOpt {
		optarg.Usage()
		os.Exit(0)
	}

	if dataDirOpt == "" {
		dataDirOpt, _ = os.Getwd()
	}

	if fileOpt == "" {
		optarg.Usage()
		os.Exit(1)
	}

	if profileFile != "" {
		f, err := os.Create(profileFile)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println("Profiling to file: ", profileFile)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if metadataStr != "" {
		err = json.Unmarshal([]byte(metadataStr), &metadata)

		if err != nil {
			log.Fatal(err)
		}
	}

	cfg := neosearch.NewConfig()

	cfg.Option(neosearch.DataDir(dataDirOpt))
	cfg.Option(neosearch.Debug(debugOpt))
	cfg.Option(neosearch.KVCacheSize(1 << 15))

	neo := neosearch.New(cfg)

	if newIndex {
		log.Printf("Creating index %s\n", databaseName)
		index, err = neo.CreateIndex(databaseName)
	} else {
		log.Printf("Opening index %s ...\n", databaseName)
		index, err = neo.OpenIndex(databaseName)
	}

	if err != nil {
		log.Fatalf("Failed to open database '%s': %v", err)
		return
	}

	file, err := os.OpenFile(fileOpt, os.O_RDONLY, 0)

	if err != nil {
		log.Fatalf("Unable to open file: %s", fileOpt)
		return
	}

	jsonBytes, err := gommap.Map(file.Fd(), gommap.PROT_READ,
		gommap.MAP_PRIVATE)

	if err != nil {
		panic(err)
	}

	data := make([]map[string]interface{}, 0)

	err = json.Unmarshal(jsonBytes, &data)

	if err != nil {
		panic(err)
	}

	jsonBytes = nil

	startTime := time.Now()

	index.Batch()
	var count int
	totalResults := len(data)

	runtime.GC()

	cleanup := func() {
		neo.Close()
		file.Close()
		if profileFile != "" {
			fmt.Println("stopping profile: ", profileFile)
			pprof.StopCPUProfile()
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		cleanup()
		os.Exit(1)
	}()

	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered from panic", r)
			cleanup()
			os.Exit(1)
		}

		cleanup()
	}()

	fmt.Println("Importing ", len(data), " records")

	for idx := range data {
		dataEntry := data[idx]

		if dataEntry["_id"] == nil {
			dataEntry["_id"] = idx
		}

		entryJSON, err := json.Marshal(&dataEntry)
		if err != nil {
			log.Println(err)
			return
		}

		err = index.Add(uint64(idx), entryJSON, metadata)
		if err != nil {
			panic(err)
		}

		if count == batchSize {
			count = 0

			fmt.Println("Flushing batch: ", idx, " from ", totalResults)
			index.FlushBatch()
			if idx != (totalResults - 1) {
				index.Batch()
			}

			runtime.GC()
		} else {
			count = count + 1
		}

		data[idx] = nil
	}

	index.FlushBatch()
	index.Close()
	neo.Close()

	elapsed := time.Since(startTime)

	log.Printf("Database indexed in %v\n", elapsed)
}
Example #16
0
func (s *S) TestUnsafeUnmap(c *C) {
	mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
	c.Assert(err, IsNil)
	c.Assert(mmap.UnsafeUnmap(), IsNil)
}