func newReader(fn string) (*reader, error) {
f, err := os.OpenFile(fn, os.O_RDONLY, 0)
if err != nil {
return nil, err
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return nil, err
}
var data gommap.MMap
var zero bool
if fi.Size() <= 0 {
zero = true
} else {
data, err = gommap.Map(f.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
if err != nil {
return nil, err
}
}
r := &reader{
cs: crc32.NewIEEE(),
data: data,
zero: zero,
done: zero, // noop reader if zero-byte file
}
r.bv.Data = data
return r, nil
}
func (s *S) TestIsResidentTwoPages(c *C) {
testPath := path.Join(c.MkDir(), "test.txt")
file, err := os.Create(testPath)
c.Assert(err, IsNil)
defer file.Close()
file.Seek(int64(os.Getpagesize()*2-1), 0)
file.Write([]byte{'x'})
mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
// Not entirely a stable test, but should usually work.
mmap[len(mmap)-1] = 'x'
mapped, err := mmap.IsResident()
c.Assert(err, IsNil)
c.Assert(mapped, DeepEquals, []bool{false, true})
mmap[0] = 'x'
mapped, err = mmap.IsResident()
c.Assert(err, IsNil)
c.Assert(mapped, DeepEquals, []bool{true, true})
}
func (s *S) TestIsResidentUnderOnePage(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
mapped, err := mmap.IsResident()
c.Assert(err, IsNil)
c.Assert(mapped, DeepEquals, []bool{true})
}
func (s *S) TestProtFlagsAndErr(c *C) {
testPath := s.file.Name()
s.file.Close()
file, err := os.Open(testPath)
c.Assert(err, IsNil)
s.file = file
_, err = gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
// For this to happen, both the error and the protection flag must work.
c.Assert(err, Equals, syscall.EACCES)
}
func fixFile() {
file, err := os.Open("./data/0000.txt")
check(err)
mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
check(err)
numLines, err := countLines(bytes.NewReader(mmap))
check(err)
lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)
lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")
// dear lord, fix this
sub := [][][]byte{
lines[:(numLines / 4)],
lines[(numLines / 4):(numLines / 2)],
lines[(numLines / 2) : (numLines/2)+(numLines/4)],
lines[(numLines/2)+(numLines/4) : numLines],
}
jobs := make(chan []byte)
results := make(chan string)
wg := new(sync.WaitGroup)
for w := 0; w <= 3; w++ {
wg.Add(1)
go normalizeLines(jobs, results, wg)
}
go func() {
for i := 0; i <= 3; i++ {
jobs <- bytes.Join(sub[i], []byte{'\n'})
}
close(jobs)
}()
go func() {
wg.Wait()
close(results)
}()
for v := range results {
if strings.HasPrefix(v, "trimmed:") {
//fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
} else if strings.HasPrefix(v, "bad:") {
fmt.Fprintf(os.Stdout, "%s\n", v[strings.IndexAny(v, ":")+1:])
} else {
//fmt.Fprintf(os.Stdout, "%s\n", v)
}
}
}
func BenchmarkMmapReader(b *testing.B) {
file := createDBFile()
defer os.Remove(file.Name())
defer file.Close()
m, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
if err != nil {
b.Fatal(err)
}
defer m.UnsafeUnmap()
benchReader(b, New(bytes.NewReader(m)))
}
func (s *S) TestAdvise(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
// A bit tricky to blackbox-test these.
err = mmap.Advise(gommap.MADV_RANDOM)
c.Assert(err, IsNil)
err = mmap.Advise(9999)
c.Assert(err, ErrorMatches, "invalid argument")
}
func (s *S) TestProtect(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
c.Assert([]byte(mmap), DeepEquals, testData)
err = mmap.Protect(gommap.PROT_READ | gommap.PROT_WRITE)
c.Assert(err, IsNil)
// If this operation doesn't blow up tests, the call above worked.
mmap[9] = 'X'
}
func (s *S) TestFlags(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
mmap[9] = 'X'
mmap.Sync(gommap.MS_SYNC)
fileData, err := ioutil.ReadFile(s.file.Name())
c.Assert(err, IsNil)
// Shouldn't have written, since the map is private.
c.Assert(fileData, DeepEquals, []byte("0123456789ABCDEF"))
}
func (s *S) TestSliceMethods(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
c.Assert([]byte(mmap), DeepEquals, testData)
mmap[9] = 'X'
mmap[7:10].Sync(gommap.MS_SYNC)
fileData, err := ioutil.ReadFile(s.file.Name())
c.Assert(err, IsNil)
c.Assert(fileData, DeepEquals, []byte("012345678XABCDEF"))
}
func (s *S) TestLock(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_PRIVATE)
c.Assert(err, IsNil)
defer mmap.UnsafeUnmap()
// A bit tricky to blackbox-test these.
err = mmap.Lock()
c.Assert(err, IsNil)
err = mmap.Lock()
c.Assert(err, IsNil)
err = mmap.Unlock()
c.Assert(err, IsNil)
err = mmap.Unlock()
c.Assert(err, IsNil)
}
func GetFileHash(filepath string) string {
fi, err := os.Open(filepath)
if err != nil {
panic(err)
}
defer fi.Close()
running_hash := md5.New()
//io.Copy(running_hash, fi)
mmap, err := gommap.Map(fi.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
if err != nil {
panic(err)
}
//mmapReader := bytes.NewReader(mmap)
//io.Copy(running_hash, mmapReader)
buf := bytes.NewBuffer(mmap)
io.Copy(running_hash, buf)
return fmt.Sprintf("%x", running_hash.Sum(nil))
}
func MMapTyped(f *File, val interface{}) (interface{}, gommap.MMap, error) {
stat, err := f.Stat()
if err != nil {
return nil, nil, err
}
typ := reflect.TypeOf(val)
typSize := int64(typ.Size())
if stat.Size()%typSize != 0 {
return nil, nil, errors.New("Shm region size not a multiple of value type")
}
buf, err := gommap.Map(f.File.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
if err != nil {
return nil, nil, err
}
header := *(*reflect.SliceHeader)(unsafe.Pointer(&buf))
header.Len /= int(typSize)
header.Cap /= int(typSize)
ns := reflect.NewAt(reflect.SliceOf(typ), unsafe.Pointer(&header))
return reflect.Indirect(ns).Interface(), buf, nil
}
func (c *clock) createBuffers() error {
g := c.shmGlobal
c.shm = c.wlc.NewShm(c)
if err := c.registry.Bind(g.Name, g.Interface, g.Version, c.shm.Id()); err != nil {
return errgo.Trace(err)
}
// collect shm formats
if err := c.sync(); err != nil {
return errgo.Trace(err)
}
// allocate pool for 2 buffers
poolSize := int32(len(c.buffers)) * c.bufSize
shmO, err := shm.Open("clock", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil {
return errgo.Trace(err)
}
defer shmO.Close()
if err := shmO.Truncate(int64(poolSize)); err != nil {
return errgo.Trace(err)
}
if c.bufsMap, err = gommap.Map(shmO.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED); err != nil {
return errgo.Trace(err)
}
shmPool := c.wlc.NewShmPool(c)
if err := c.shm.CreatePool(shmPool.Id(), shmO.Fd(), poolSize); err != nil {
return errgo.Trace(err)
}
for i := range c.buffers {
buf := new(buffer)
buf.img.Rect = image.Rect(0, 0, int(c.w), int(c.h))
buf.img.Stride = int(c.w) * 4
buf.img.Pix = c.bufsMap[i*int(c.bufSize) : (i+1)*int(c.bufSize)]
buf.ClientBuffer = c.wlc.NewBuffer(buf)
if err := shmPool.CreateBuffer(
buf.Id(), // Id
int32(i)*c.bufSize, // Offset
c.w, // Width
c.h, // Height
c.stride, // Stride
0, // Format
); err != nil {
return errgo.Trace(err)
}
c.buffers[i] = buf
}
//if err := shmPool.Destroy(); err != nil {
// return errgo.Trace(err)
//}
if err := c.sync(); err != nil {
return errgo.Trace(err)
}
return nil
}
func main() {
var (
fileOpt,
dataDirOpt,
databaseName,
profileFile,
metadataStr string
metadata = index.Metadata{}
helpOpt, newIndex, debugOpt bool
err error
index *index.Index
batchSize int
)
optarg.Header("General options")
optarg.Add("f", "file", "Read NeoSearch JSON database from file. (Required)", "")
optarg.Add("c", "create", "Create new index database", false)
optarg.Add("b", "batch-size", "Batch size", 1000)
optarg.Add("n", "name", "Name of index database", "")
optarg.Add("d", "data-dir", "Data directory", "")
optarg.Add("t", "trace-debug", "Enable trace for debug", false)
optarg.Add("h", "help", "Display this help", false)
optarg.Add("p", "cpuprofile", "write cpu profile to file", "")
optarg.Add("m", "metadata", "metadata of documents", "")
for opt := range optarg.Parse() {
switch opt.ShortName {
case "f":
fileOpt = opt.String()
case "b":
batchSize = opt.Int()
case "d":
dataDirOpt = opt.String()
case "n":
databaseName = opt.String()
case "c":
newIndex = true
case "t":
debugOpt = true
case "p":
profileFile = opt.String()
case "m":
metadataStr = opt.String()
case "h":
helpOpt = true
}
}
if helpOpt {
optarg.Usage()
os.Exit(0)
}
if dataDirOpt == "" {
dataDirOpt, _ = os.Getwd()
}
if fileOpt == "" {
optarg.Usage()
os.Exit(1)
}
if profileFile != "" {
f, err := os.Create(profileFile)
if err != nil {
log.Fatal(err)
}
fmt.Println("Profiling to file: ", profileFile)
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if metadataStr != "" {
err = json.Unmarshal([]byte(metadataStr), &metadata)
if err != nil {
log.Fatal(err)
}
}
cfg := neosearch.NewConfig()
cfg.Option(neosearch.DataDir(dataDirOpt))
cfg.Option(neosearch.Debug(debugOpt))
cfg.Option(neosearch.KVCacheSize(1 << 15))
neo := neosearch.New(cfg)
if newIndex {
log.Printf("Creating index %s\n", databaseName)
index, err = neo.CreateIndex(databaseName)
} else {
log.Printf("Opening index %s ...\n", databaseName)
index, err = neo.OpenIndex(databaseName)
}
if err != nil {
log.Fatalf("Failed to open database '%s': %v", err)
return
}
file, err := os.OpenFile(fileOpt, os.O_RDONLY, 0)
if err != nil {
log.Fatalf("Unable to open file: %s", fileOpt)
return
}
jsonBytes, err := gommap.Map(file.Fd(), gommap.PROT_READ,
gommap.MAP_PRIVATE)
if err != nil {
panic(err)
}
data := make([]map[string]interface{}, 0)
err = json.Unmarshal(jsonBytes, &data)
if err != nil {
panic(err)
}
jsonBytes = nil
startTime := time.Now()
index.Batch()
var count int
totalResults := len(data)
runtime.GC()
cleanup := func() {
neo.Close()
file.Close()
if profileFile != "" {
fmt.Println("stopping profile: ", profileFile)
pprof.StopCPUProfile()
}
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
<-c
cleanup()
os.Exit(1)
}()
defer func() {
if r := recover(); r != nil {
fmt.Println("Recovered from panic", r)
cleanup()
os.Exit(1)
}
cleanup()
}()
fmt.Println("Importing ", len(data), " records")
for idx := range data {
dataEntry := data[idx]
if dataEntry["_id"] == nil {
dataEntry["_id"] = idx
}
entryJSON, err := json.Marshal(&dataEntry)
if err != nil {
log.Println(err)
return
}
err = index.Add(uint64(idx), entryJSON, metadata)
if err != nil {
panic(err)
}
if count == batchSize {
count = 0
fmt.Println("Flushing batch: ", idx, " from ", totalResults)
index.FlushBatch()
if idx != (totalResults - 1) {
index.Batch()
}
runtime.GC()
} else {
count = count + 1
}
data[idx] = nil
}
index.FlushBatch()
index.Close()
neo.Close()
elapsed := time.Since(startTime)
log.Printf("Database indexed in %v\n", elapsed)
}
func (s *S) TestUnsafeUnmap(c *C) {
mmap, err := gommap.Map(s.file.Fd(), gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
c.Assert(err, IsNil)
c.Assert(mmap.UnsafeUnmap(), IsNil)
}