func newReader(fn string) (*reader, error) { f, err := os.OpenFile(fn, os.O_RDONLY, 0) if err != nil { return nil, err } defer f.Close() fi, err := f.Stat() if err != nil { return nil, err } var data gommap.MMap var zero bool if fi.Size() <= 0 { zero = true } else { data, err = gommap.Map(f.Fd(), gommap.PROT_READ, gommap.MAP_SHARED) if err != nil { return nil, err } } r := &reader{ cs: crc32.NewIEEE(), data: data, zero: zero, done: zero, // noop reader if zero-byte file } r.bv.Data = data return r, nil }
func (s *S) TestIsResidentTwoPages(c *C) { testPath := path.Join(c.MkDir(), "test.txt") file, err := os.Create(testPath) c.Assert(err, IsNil) defer file.Close() file.Seek(int64(os.Getpagesize()*2-1), 0) file.Write([]byte{'x'}) mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() // Not entirely a stable test, but should usually work. mmap[len(mmap)-1] = 'x' mapped, err := mmap.IsResident() c.Assert(err, IsNil) c.Assert(mapped, DeepEquals, []bool{false, true}) mmap[0] = 'x' mapped, err = mmap.IsResident() c.Assert(err, IsNil) c.Assert(mapped, DeepEquals, []bool{true, true}) }
func (s *S) TestIsResidentUnderOnePage(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() mapped, err := mmap.IsResident() c.Assert(err, IsNil) c.Assert(mapped, DeepEquals, []bool{true}) }
func (s *S) TestProtFlagsAndErr(c *C) { testPath := s.file.Name() s.file.Close() file, err := os.Open(testPath) c.Assert(err, IsNil) s.file = file _, err = gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED) // For this to happen, both the error and the protection flag must work. c.Assert(err, Equals, syscall.EACCES) }
func fixFile() { file, err := os.Open("./data/0000.txt") check(err) mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE) check(err) numLines, err := countLines(bytes.NewReader(mmap)) check(err) lines := bytes.SplitN(mmap, []byte{'\n'}, numLines) lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n") // dear lord, fix this sub := [][][]byte{ lines[:(numLines / 4)], lines[(numLines / 4):(numLines / 2)], lines[(numLines / 2) : (numLines/2)+(numLines/4)], lines[(numLines/2)+(numLines/4) : numLines], } jobs := make(chan []byte) results := make(chan string) wg := new(sync.WaitGroup) for w := 0; w <= 3; w++ { wg.Add(1) go normalizeLines(jobs, results, wg) } go func() { for i := 0; i <= 3; i++ { jobs <- bytes.Join(sub[i], []byte{'\n'}) } close(jobs) }() go func() { wg.Wait() close(results) }() for v := range results { if strings.HasPrefix(v, "trimmed:") { //fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:]) } else if strings.HasPrefix(v, "bad:") { fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:]) } else { //fmt.Fprintf(os.Stdout, "%s\n", v) } } }
func BenchmarkMmapReader(b *testing.B) { file := createDBFile() defer os.Remove(file.Name()) defer file.Close() m, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED) if err != nil { b.Fatal(err) } defer m.UnsafeUnmap() benchReader(b, New(bytes.NewReader(m))) }
func (s *S) TestAdvise(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() // A bit tricky to blackbox-test these. err = mmap.Advise(gommap.MADV_RANDOM) c.Assert(err, IsNil) err = mmap.Advise(9999) c.Assert(err, ErrorMatches, "invalid argument") }
func (s *S) TestProtect(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() c.Assert([]byte(mmap), DeepEquals, testData) err = mmap.Protect(gommap.PROT_READ | gommap.PROT_WRITE) c.Assert(err, IsNil) // If this operation doesn't blow up tests, the call above worked. mmap[9] = 'X' }
func (s *S) TestFlags(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() mmap[9] = 'X' mmap.Sync(gommap.MS_SYNC) fileData, err := ioutil.ReadFile(s.file.Name()) c.Assert(err, IsNil) // Shouldn't have written, since the map is private. c.Assert(fileData, DeepEquals, []byte("0123456789ABCDEF")) }
func (s *S) TestSliceMethods(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() c.Assert([]byte(mmap), DeepEquals, testData) mmap[9] = 'X' mmap[7:10].Sync(gommap.MS_SYNC) fileData, err := ioutil.ReadFile(s.file.Name()) c.Assert(err, IsNil) c.Assert(fileData, DeepEquals, []byte("012345678XABCDEF")) }
func (s *S) TestLock(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE) c.Assert(err, IsNil) defer mmap.UnsafeUnmap() // A bit tricky to blackbox-test these. err = mmap.Lock() c.Assert(err, IsNil) err = mmap.Lock() c.Assert(err, IsNil) err = mmap.Unlock() c.Assert(err, IsNil) err = mmap.Unlock() c.Assert(err, IsNil) }
func GetFileHash(filepath string) string { fi, err := os.Open(filepath) if err != nil { panic(err) } defer fi.Close() running_hash := md5.New() //io.Copy(running_hash, fi) mmap, err := gommap.Map(fi.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE) if err != nil { panic(err) } //mmapReader := bytes.NewReader(mmap) //io.Copy(running_hash, mmapReader) buf := bytes.NewBuffer(mmap) io.Copy(running_hash, buf) return fmt.Sprintf("%x", running_hash.Sum(nil)) }
func MMapTyped(f *File, val interface{}) (interface{}, gommap.MMap, error) { stat, err := f.Stat() if err != nil { return nil, nil, err } typ := reflect.TypeOf(val) typSize := int64(typ.Size()) if stat.Size()%typSize != 0 { return nil, nil, errors.New("Shm region size not a multiple of value type") } buf, err := gommap.Map(f.File.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED) if err != nil { return nil, nil, err } header := *(*reflect.SliceHeader)(unsafe.Pointer(&buf)) header.Len /= int(typSize) header.Cap /= int(typSize) ns := reflect.NewAt(reflect.SliceOf(typ), unsafe.Pointer(&header)) return reflect.Indirect(ns).Interface(), buf, nil }
func (c *clock) createBuffers() error { g := c.shmGlobal c.shm = c.wlc.NewShm(c) if err := c.registry.Bind(g.Name, g.Interface, g.Version, c.shm.Id()); err != nil { return errgo.Trace(err) } // collect shm formats if err := c.sync(); err != nil { return errgo.Trace(err) } // allocate pool for 2 buffers poolSize := int32(len(c.buffers)) * c.bufSize shmO, err := shm.Open("clock", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { return errgo.Trace(err) } defer shmO.Close() if err := shmO.Truncate(int64(poolSize)); err != nil { return errgo.Trace(err) } if c.bufsMap, err = gommap.Map(shmO.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED); err != nil { return errgo.Trace(err) } shmPool := c.wlc.NewShmPool(c) if err := c.shm.CreatePool(shmPool.Id(), shmO.Fd(), poolSize); err != nil { return errgo.Trace(err) } for i := range c.buffers { buf := new(buffer) buf.img.Rect = image.Rect(0, 0, int(c.w), int(c.h)) buf.img.Stride = int(c.w) * 4 buf.img.Pix = c.bufsMap[i*int(c.bufSize) : (i+1)*int(c.bufSize)] buf.ClientBuffer = c.wlc.NewBuffer(buf) if err := shmPool.CreateBuffer( buf.Id(), // Id int32(i)*c.bufSize, // Offset c.w, // Width c.h, // Height c.stride, // Stride 0, // Format ); err != nil { return errgo.Trace(err) } c.buffers[i] = buf } //if err := shmPool.Destroy(); err != nil { // return errgo.Trace(err) //} if err := c.sync(); err != nil { return errgo.Trace(err) } return nil }
func main() { var ( fileOpt, dataDirOpt, databaseName, profileFile, metadataStr string metadata = index.Metadata{} helpOpt, newIndex, debugOpt bool err error index *index.Index batchSize int ) optarg.Header("General options") optarg.Add("f", "file", "Read NeoSearch JSON database from file. (Required)", "") optarg.Add("c", "create", "Create new index database", false) optarg.Add("b", "batch-size", "Batch size", 1000) optarg.Add("n", "name", "Name of index database", "") optarg.Add("d", "data-dir", "Data directory", "") optarg.Add("t", "trace-debug", "Enable trace for debug", false) optarg.Add("h", "help", "Display this help", false) optarg.Add("p", "cpuprofile", "write cpu profile to file", "") optarg.Add("m", "metadata", "metadata of documents", "") for opt := range optarg.Parse() { switch opt.ShortName { case "f": fileOpt = opt.String() case "b": batchSize = opt.Int() case "d": dataDirOpt = opt.String() case "n": databaseName = opt.String() case "c": newIndex = true case "t": debugOpt = true case "p": profileFile = opt.String() case "m": metadataStr = opt.String() case "h": helpOpt = true } } if helpOpt { optarg.Usage() os.Exit(0) } if dataDirOpt == "" { dataDirOpt, _ = os.Getwd() } if fileOpt == "" { optarg.Usage() os.Exit(1) } if profileFile != "" { f, err := os.Create(profileFile) if err != nil { log.Fatal(err) } fmt.Println("Profiling to file: ", profileFile) pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } if metadataStr != "" { err = json.Unmarshal([]byte(metadataStr), &metadata) if err != nil { log.Fatal(err) } } cfg := neosearch.NewConfig() cfg.Option(neosearch.DataDir(dataDirOpt)) cfg.Option(neosearch.Debug(debugOpt)) cfg.Option(neosearch.KVCacheSize(1 << 15)) neo := neosearch.New(cfg) if newIndex { log.Printf("Creating index %s\n", databaseName) index, err = neo.CreateIndex(databaseName) } else { log.Printf("Opening index %s ...\n", databaseName) index, err = neo.OpenIndex(databaseName) } if err != nil { log.Fatalf("Failed to open database '%s': %v", err) return } file, err := os.OpenFile(fileOpt, os.O_RDONLY, 0) if err != nil { log.Fatalf("Unable to open file: %s", fileOpt) return } jsonBytes, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE) if err != nil { panic(err) } data := make([]map[string]interface{}, 0) err = json.Unmarshal(jsonBytes, &data) if err != nil { panic(err) } jsonBytes = nil startTime := time.Now() index.Batch() var count int totalResults := len(data) runtime.GC() cleanup := func() { neo.Close() file.Close() if profileFile != "" { fmt.Println("stopping profile: ", profileFile) pprof.StopCPUProfile() } } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { <-c cleanup() os.Exit(1) }() defer func() { if r := recover(); r != nil { fmt.Println("Recovered from panic", r) cleanup() os.Exit(1) } cleanup() }() fmt.Println("Importing ", len(data), " records") for idx := range data { dataEntry := data[idx] if dataEntry["_id"] == nil { dataEntry["_id"] = idx } entryJSON, err := json.Marshal(&dataEntry) if err != nil { log.Println(err) return } err = index.Add(uint64(idx), entryJSON, metadata) if err != nil { panic(err) } if count == batchSize { count = 0 fmt.Println("Flushing batch: ", idx, " from ", totalResults) index.FlushBatch() if idx != (totalResults - 1) { index.Batch() } runtime.GC() } else { count = count + 1 } data[idx] = nil } index.FlushBatch() index.Close() neo.Close() elapsed := time.Since(startTime) log.Printf("Database indexed in %v\n", elapsed) }
func (s *S) TestUnsafeUnmap(c *C) { mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED) c.Assert(err, IsNil) c.Assert(mmap.UnsafeUnmap(), IsNil) }