func Example() {
dataDir, err := ioutil.TempDir("", "neosearchExample")
defer os.RemoveAll(dataDir)
OnErrorPanic(err)
cfg := neosearch.NewConfig()
cfg.Option(neosearch.DataDir(dataDir))
cfg.Option(neosearch.Debug(false))
neo := neosearch.New(cfg)
defer neo.Close()
index, err := neo.CreateIndex("test")
OnErrorPanic(err)
err = index.Add(1, []byte(`{"id": 1, "name": "Neoway Business Solution"}`), nil)
OnErrorPanic(err)
err = index.Add(2, []byte(`{"id": 2, "name": "Google Inc."}`), nil)
OnErrorPanic(err)
err = index.Add(3, []byte(`{"id": 3, "name": "Facebook Company"}`), nil)
OnErrorPanic(err)
err = index.Add(4, []byte(`{"id": 4, "name": "Neoway Teste"}`), nil)
OnErrorPanic(err)
data, err := index.Get(1)
OnErrorPanic(err)
fmt.Println(string(data))
// Output:
// {"id": 1, "name": "Neoway Business Solution"}
}
func ExampleMatchPrefix() {
dataDir, err := ioutil.TempDir("", "neosearchExample")
defer os.RemoveAll(dataDir)
OnErrorPanic(err)
cfg := neosearch.NewConfig()
cfg.Option(neosearch.DataDir(dataDir))
cfg.Option(neosearch.Debug(false))
neo := neosearch.New(cfg)
defer neo.Close()
index, err := neo.CreateIndex("test")
OnErrorPanic(err)
err = index.Add(1, []byte(`{"id": 1, "name": "Neoway Business Solution"}`), nil)
OnErrorPanic(err)
err = index.Add(2, []byte(`{"id": 2, "name": "Google Inc."}`), nil)
OnErrorPanic(err)
err = index.Add(3, []byte(`{"id": 3, "name": "Facebook Company"}`), nil)
OnErrorPanic(err)
err = index.Add(4, []byte(`{"id": 4, "name": "Neoway Teste"}`), nil)
OnErrorPanic(err)
values, err := index.MatchPrefix([]byte("name"), []byte("neoway"))
OnErrorPanic(err)
for _, value := range values {
fmt.Println(value)
}
// Output:
// {"id": 1, "name": "Neoway Business Solution"}
// {"id": 4, "name": "Neoway Teste"}
}
func main() {
var (
configOpt, dataDirOpt, hostOpt string
goProcsOpt uint64
portOpt uint16
helpOpt, debugOpt bool
err error
cfg *neosearch.Config
cfgServer *server.ServerConfig
)
cfg = neosearch.NewConfig()
cfgServer = server.NewConfig()
optarg.Header("General options")
optarg.Add("c", "config", "Configurations file", "")
optarg.Add("d", "data-dir", "Data directory", "")
optarg.Add("g", "maximum-concurrence", "Set the maximum number of concurrent go routines", 0)
optarg.Add("t", "trace-debug", "Enable debug traces", false)
optarg.Add("s", "server-address", "Server host and port", "0.0.0.0:9500")
optarg.Add("h", "help", "Display this help", false)
for opt := range optarg.Parse() {
switch opt.ShortName {
case "c":
configOpt = opt.String()
case "d":
dataDirOpt = opt.String()
case "s":
address := opt.String()
addrParts := strings.Split(address, ":")
if len(addrParts) > 1 {
hostOpt = addrParts[0]
port := addrParts[1]
portInt, err := strconv.Atoi(port)
if err == nil {
portOpt = uint16(portInt)
} else {
log.Fatalf("Invalid port number: %s (%s)", port, err)
return
}
} else {
hostOpt = addrParts[0]
portOpt = DefaultPort
}
case "t":
debugOpt = opt.Bool()
case "g":
goprocsStr := opt.String()
goProcsInt, err := strconv.Atoi(goprocsStr)
if err != nil || goProcsInt <= 0 {
log.Fatal("Invalid -g option. Should be a unsigned integer value greater than zero.")
return
}
goProcsOpt = uint64(goProcsInt)
case "h":
helpOpt = true
}
}
if helpOpt {
optarg.Usage()
os.Exit(0)
}
if dataDirOpt == "" {
dataDirOpt, _ = os.Getwd()
}
if configOpt == "" {
log.Println("No configuration file supplied. Using defaults...")
cfg.Debug = false
cfg.DataDir = "/data"
cfg.EnableCache = true
} else {
cfg, err = neosearch.ConfigFromFile(configOpt)
if err != nil {
log.Fatalf("Failed to read configuration file: %s", err.Error())
return
}
}
if hostOpt == "" {
hostOpt = DefaultHost
}
if portOpt == 0 {
portOpt = DefaultPort
}
// override config options by argument options
cfg.Option(neosearch.DataDir(dataDirOpt))
cfg.Option(neosearch.Debug(debugOpt))
cfgServer.Host = hostOpt
cfgServer.Port = portOpt
search := neosearch.New(cfg)
defer func() {
search.Close()
}()
httpServer, err := server.New(search, cfgServer)
_ = goProcsOpt
if err != nil {
log.Fatal(err.Error())
return
}
// Wait for a SIGINT (perhaps triggered by user with CTRL-C)
// Run cleanup when signal is received
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, os.Interrupt)
go func() {
for _ = range signalChan {
fmt.Println("\nReceived an interrupt, closing indexes...\n")
search.Close()
os.Exit(0)
}
}()
err = httpServer.Start()
if err != nil {
log.Fatalf("Failed to start http server: %s", err.Error())
}
}
func main() {
var (
fileOpt,
dataDirOpt,
databaseName,
profileFile,
metadataStr string
metadata = index.Metadata{}
helpOpt, newIndex, debugOpt bool
err error
index *index.Index
batchSize int
)
optarg.Header("General options")
optarg.Add("f", "file", "Read NeoSearch JSON database from file. (Required)", "")
optarg.Add("c", "create", "Create new index database", false)
optarg.Add("b", "batch-size", "Batch size", 1000)
optarg.Add("n", "name", "Name of index database", "")
optarg.Add("d", "data-dir", "Data directory", "")
optarg.Add("t", "trace-debug", "Enable trace for debug", false)
optarg.Add("h", "help", "Display this help", false)
optarg.Add("p", "cpuprofile", "write cpu profile to file", "")
optarg.Add("m", "metadata", "metadata of documents", "")
for opt := range optarg.Parse() {
switch opt.ShortName {
case "f":
fileOpt = opt.String()
case "b":
batchSize = opt.Int()
case "d":
dataDirOpt = opt.String()
case "n":
databaseName = opt.String()
case "c":
newIndex = true
case "t":
debugOpt = true
case "p":
profileFile = opt.String()
case "m":
metadataStr = opt.String()
case "h":
helpOpt = true
}
}
if helpOpt {
optarg.Usage()
os.Exit(0)
}
if dataDirOpt == "" {
dataDirOpt, _ = os.Getwd()
}
if fileOpt == "" {
optarg.Usage()
os.Exit(1)
}
if profileFile != "" {
f, err := os.Create(profileFile)
if err != nil {
log.Fatal(err)
}
fmt.Println("Profiling to file: ", profileFile)
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if metadataStr != "" {
err = json.Unmarshal([]byte(metadataStr), &metadata)
if err != nil {
log.Fatal(err)
}
}
cfg := neosearch.NewConfig()
cfg.Option(neosearch.DataDir(dataDirOpt))
cfg.Option(neosearch.Debug(debugOpt))
cfg.Option(neosearch.KVCacheSize(1 << 15))
neo := neosearch.New(cfg)
if newIndex {
log.Printf("Creating index %s\n", databaseName)
index, err = neo.CreateIndex(databaseName)
} else {
log.Printf("Opening index %s ...\n", databaseName)
index, err = neo.OpenIndex(databaseName)
}
if err != nil {
log.Fatalf("Failed to open database '%s': %v", err)
return
}
file, err := os.OpenFile(fileOpt, os.O_RDONLY, 0)
if err != nil {
log.Fatalf("Unable to open file: %s", fileOpt)
return
}
jsonBytes, err := gommap.Map(file.Fd(), gommap.PROT_READ,
gommap.MAP_PRIVATE)
if err != nil {
panic(err)
}
data := make([]map[string]interface{}, 0)
err = json.Unmarshal(jsonBytes, &data)
if err != nil {
panic(err)
}
jsonBytes = nil
startTime := time.Now()
index.Batch()
var count int
totalResults := len(data)
runtime.GC()
cleanup := func() {
neo.Close()
file.Close()
if profileFile != "" {
fmt.Println("stopping profile: ", profileFile)
pprof.StopCPUProfile()
}
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
<-c
cleanup()
os.Exit(1)
}()
defer func() {
if r := recover(); r != nil {
fmt.Println("Recovered from panic", r)
cleanup()
os.Exit(1)
}
cleanup()
}()
fmt.Println("Importing ", len(data), " records")
for idx := range data {
dataEntry := data[idx]
if dataEntry["_id"] == nil {
dataEntry["_id"] = idx
}
entryJSON, err := json.Marshal(&dataEntry)
if err != nil {
log.Println(err)
return
}
err = index.Add(uint64(idx), entryJSON, metadata)
if err != nil {
panic(err)
}
if count == batchSize {
count = 0
fmt.Println("Flushing batch: ", idx, " from ", totalResults)
index.FlushBatch()
if idx != (totalResults - 1) {
index.Batch()
}
runtime.GC()
} else {
count = count + 1
}
data[idx] = nil
}
index.FlushBatch()
index.Close()
neo.Close()
elapsed := time.Since(startTime)
log.Printf("Database indexed in %v\n", elapsed)
}