Example #1
0
func Example() {
	dataDir, err := ioutil.TempDir("", "neosearchExample")
	defer os.RemoveAll(dataDir)

	OnErrorPanic(err)

	cfg := neosearch.NewConfig()
	cfg.Option(neosearch.DataDir(dataDir))
	cfg.Option(neosearch.Debug(false))

	neo := neosearch.New(cfg)
	defer neo.Close()

	index, err := neo.CreateIndex("test")
	OnErrorPanic(err)

	err = index.Add(1, []byte(`{"id": 1, "name": "Neoway Business Solution"}`), nil)
	OnErrorPanic(err)

	err = index.Add(2, []byte(`{"id": 2, "name": "Google Inc."}`), nil)
	OnErrorPanic(err)

	err = index.Add(3, []byte(`{"id": 3, "name": "Facebook Company"}`), nil)
	OnErrorPanic(err)

	err = index.Add(4, []byte(`{"id": 4, "name": "Neoway Teste"}`), nil)
	OnErrorPanic(err)

	data, err := index.Get(1)
	OnErrorPanic(err)

	fmt.Println(string(data))
	// Output:
	// {"id": 1, "name": "Neoway Business Solution"}
}
Example #2
0
func ExampleMatchPrefix() {
	dataDir, err := ioutil.TempDir("", "neosearchExample")
	defer os.RemoveAll(dataDir)

	OnErrorPanic(err)

	cfg := neosearch.NewConfig()
	cfg.Option(neosearch.DataDir(dataDir))
	cfg.Option(neosearch.Debug(false))

	neo := neosearch.New(cfg)
	defer neo.Close()

	index, err := neo.CreateIndex("test")
	OnErrorPanic(err)

	err = index.Add(1, []byte(`{"id": 1, "name": "Neoway Business Solution"}`), nil)
	OnErrorPanic(err)

	err = index.Add(2, []byte(`{"id": 2, "name": "Google Inc."}`), nil)
	OnErrorPanic(err)

	err = index.Add(3, []byte(`{"id": 3, "name": "Facebook Company"}`), nil)
	OnErrorPanic(err)

	err = index.Add(4, []byte(`{"id": 4, "name": "Neoway Teste"}`), nil)
	OnErrorPanic(err)

	values, err := index.MatchPrefix([]byte("name"), []byte("neoway"))
	OnErrorPanic(err)

	for _, value := range values {
		fmt.Println(value)
	}

	// Output:
	// {"id": 1, "name": "Neoway Business Solution"}
	// {"id": 4, "name": "Neoway Teste"}
}
Example #3
0
func main() {
	var (
		configOpt, dataDirOpt, hostOpt string
		goProcsOpt                     uint64
		portOpt                        uint16
		helpOpt, debugOpt              bool
		err                            error
		cfg                            *neosearch.Config
		cfgServer                      *server.ServerConfig
	)

	cfg = neosearch.NewConfig()
	cfgServer = server.NewConfig()

	optarg.Header("General options")
	optarg.Add("c", "config", "Configurations file", "")
	optarg.Add("d", "data-dir", "Data directory", "")
	optarg.Add("g", "maximum-concurrence", "Set the maximum number of concurrent go routines", 0)
	optarg.Add("t", "trace-debug", "Enable debug traces", false)
	optarg.Add("s", "server-address", "Server host and port", "0.0.0.0:9500")
	optarg.Add("h", "help", "Display this help", false)

	for opt := range optarg.Parse() {
		switch opt.ShortName {
		case "c":
			configOpt = opt.String()
		case "d":
			dataDirOpt = opt.String()
		case "s":
			address := opt.String()
			addrParts := strings.Split(address, ":")

			if len(addrParts) > 1 {
				hostOpt = addrParts[0]
				port := addrParts[1]

				portInt, err := strconv.Atoi(port)

				if err == nil {
					portOpt = uint16(portInt)
				} else {
					log.Fatalf("Invalid port number: %s (%s)", port, err)
					return
				}
			} else {
				hostOpt = addrParts[0]
				portOpt = DefaultPort
			}
		case "t":
			debugOpt = opt.Bool()
		case "g":
			goprocsStr := opt.String()
			goProcsInt, err := strconv.Atoi(goprocsStr)

			if err != nil || goProcsInt <= 0 {
				log.Fatal("Invalid -g option. Should be a unsigned integer value greater than zero.")
				return
			}

			goProcsOpt = uint64(goProcsInt)
		case "h":
			helpOpt = true
		}
	}

	if helpOpt {
		optarg.Usage()
		os.Exit(0)
	}

	if dataDirOpt == "" {
		dataDirOpt, _ = os.Getwd()
	}

	if configOpt == "" {
		log.Println("No configuration file supplied. Using defaults...")
		cfg.Debug = false
		cfg.DataDir = "/data"
		cfg.EnableCache = true
	} else {
		cfg, err = neosearch.ConfigFromFile(configOpt)

		if err != nil {
			log.Fatalf("Failed to read configuration file: %s", err.Error())
			return
		}
	}

	if hostOpt == "" {
		hostOpt = DefaultHost
	}

	if portOpt == 0 {
		portOpt = DefaultPort
	}

	// override config options by argument options
	cfg.Option(neosearch.DataDir(dataDirOpt))
	cfg.Option(neosearch.Debug(debugOpt))

	cfgServer.Host = hostOpt
	cfgServer.Port = portOpt

	search := neosearch.New(cfg)

	defer func() {
		search.Close()
	}()

	httpServer, err := server.New(search, cfgServer)

	_ = goProcsOpt

	if err != nil {
		log.Fatal(err.Error())
		return
	}

	// Wait for a SIGINT (perhaps triggered by user with CTRL-C)
	// Run cleanup when signal is received
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go func() {
		for _ = range signalChan {
			fmt.Println("\nReceived an interrupt, closing indexes...\n")
			search.Close()
			os.Exit(0)
		}
	}()

	err = httpServer.Start()

	if err != nil {
		log.Fatalf("Failed to start http server: %s", err.Error())
	}
}
Example #4
0
func main() {
	var (
		fileOpt,
		dataDirOpt,
		databaseName,
		profileFile,
		metadataStr string
		metadata                    = index.Metadata{}
		helpOpt, newIndex, debugOpt bool
		err                         error
		index                       *index.Index
		batchSize                   int
	)

	optarg.Header("General options")
	optarg.Add("f", "file", "Read NeoSearch JSON database from file. (Required)", "")
	optarg.Add("c", "create", "Create new index database", false)
	optarg.Add("b", "batch-size", "Batch size", 1000)
	optarg.Add("n", "name", "Name of index database", "")
	optarg.Add("d", "data-dir", "Data directory", "")
	optarg.Add("t", "trace-debug", "Enable trace for debug", false)
	optarg.Add("h", "help", "Display this help", false)
	optarg.Add("p", "cpuprofile", "write cpu profile to file", "")
	optarg.Add("m", "metadata", "metadata of documents", "")

	for opt := range optarg.Parse() {
		switch opt.ShortName {
		case "f":
			fileOpt = opt.String()
		case "b":
			batchSize = opt.Int()
		case "d":
			dataDirOpt = opt.String()
		case "n":
			databaseName = opt.String()
		case "c":
			newIndex = true
		case "t":
			debugOpt = true
		case "p":
			profileFile = opt.String()
		case "m":
			metadataStr = opt.String()
		case "h":
			helpOpt = true
		}
	}

	if helpOpt {
		optarg.Usage()
		os.Exit(0)
	}

	if dataDirOpt == "" {
		dataDirOpt, _ = os.Getwd()
	}

	if fileOpt == "" {
		optarg.Usage()
		os.Exit(1)
	}

	if profileFile != "" {
		f, err := os.Create(profileFile)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println("Profiling to file: ", profileFile)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if metadataStr != "" {
		err = json.Unmarshal([]byte(metadataStr), &metadata)

		if err != nil {
			log.Fatal(err)
		}
	}

	cfg := neosearch.NewConfig()

	cfg.Option(neosearch.DataDir(dataDirOpt))
	cfg.Option(neosearch.Debug(debugOpt))
	cfg.Option(neosearch.KVCacheSize(1 << 15))

	neo := neosearch.New(cfg)

	if newIndex {
		log.Printf("Creating index %s\n", databaseName)
		index, err = neo.CreateIndex(databaseName)
	} else {
		log.Printf("Opening index %s ...\n", databaseName)
		index, err = neo.OpenIndex(databaseName)
	}

	if err != nil {
		log.Fatalf("Failed to open database '%s': %v", err)
		return
	}

	file, err := os.OpenFile(fileOpt, os.O_RDONLY, 0)

	if err != nil {
		log.Fatalf("Unable to open file: %s", fileOpt)
		return
	}

	jsonBytes, err := gommap.Map(file.Fd(), gommap.PROT_READ,
		gommap.MAP_PRIVATE)

	if err != nil {
		panic(err)
	}

	data := make([]map[string]interface{}, 0)

	err = json.Unmarshal(jsonBytes, &data)

	if err != nil {
		panic(err)
	}

	jsonBytes = nil

	startTime := time.Now()

	index.Batch()
	var count int
	totalResults := len(data)

	runtime.GC()

	cleanup := func() {
		neo.Close()
		file.Close()
		if profileFile != "" {
			fmt.Println("stopping profile: ", profileFile)
			pprof.StopCPUProfile()
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		cleanup()
		os.Exit(1)
	}()

	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered from panic", r)
			cleanup()
			os.Exit(1)
		}

		cleanup()
	}()

	fmt.Println("Importing ", len(data), " records")

	for idx := range data {
		dataEntry := data[idx]

		if dataEntry["_id"] == nil {
			dataEntry["_id"] = idx
		}

		entryJSON, err := json.Marshal(&dataEntry)
		if err != nil {
			log.Println(err)
			return
		}

		err = index.Add(uint64(idx), entryJSON, metadata)
		if err != nil {
			panic(err)
		}

		if count == batchSize {
			count = 0

			fmt.Println("Flushing batch: ", idx, " from ", totalResults)
			index.FlushBatch()
			if idx != (totalResults - 1) {
				index.Batch()
			}

			runtime.GC()
		} else {
			count = count + 1
		}

		data[idx] = nil
	}

	index.FlushBatch()
	index.Close()
	neo.Close()

	elapsed := time.Since(startTime)

	log.Printf("Database indexed in %v\n", elapsed)
}