Exemplo n.º 1
0
func main() {

	var cores int
	var mport int
	var lport int
	var master int
	var localip string
	var masterip string
	flag.IntVar(&cores, "core", runtime.NumCPU(), "CPU 核心数量")
	flag.IntVar(&lport, "p", 9991, "启动端口,默认9991")
	flag.IntVar(&master, "m", 0, "启动master,默认启动的为searcher")
	flag.StringVar(&localip, "lip", "127.0.0.1", "本机ip地址,默认127.0.0.1")
	flag.StringVar(&masterip, "mip", "127.0.0.1", "主节点ip地址,默认127.0.0.1")
	flag.IntVar(&mport, "mp", 9990, "主节点端口,默认9990")
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	//debug.SetGCPercent(0)
	fmt.Printf("CORES:%v\n", runtime.NumCPU())
	//启动日志系统
	logger, err := utils.New("FalconSearcher")
	if err != nil {
		fmt.Printf("[ERROR] Create logger Error: %v\n", err)
		return
	}

	//初始化Manager
	var engine utils.Engine
	if master == 0 {
		//初始化分词器
		logger.Info("[INFO] Loading Segmenter ...")
		utils.GSegmenter = utils.NewSegmenter("./data/dictionary.txt")
		//utils.GOptions = utils.NewPy(utils.STYLE_NORMAL, utils.NO_SEGMENT)
		logger.Info("[INFO] Init Search Engine ...")
		engine = fe.NewDefaultEngine(localip, masterip, lport, mport, logger)
	} else {
		logger.Info("[INFO] Init Dispatcher ...")
		engine = fd.NewDispatcher(localip, logger)
	}

	//启动性能监控
	//go func() {
	//	log.Println(http.ListenAndServe(":6060", nil))
	//}()

	utils.GetDocIDsChan, utils.GiveDocIDsChan = utils.DocIdsMaker()

	//启动全局缓存
	// utils.Cache,err = cache.NewCache("memory", `{"interval":60}`)
	// if err != nil {
	//     fmt.Printf("[ERROR] Create Cache Error: %v\n", err)
	//		return
	// }

	logger.Info("[INFO] Starting FalconEngine Service.....")
	http := fs.NewHttpService(engine, lport, logger)

	http.Start()

}
Exemplo n.º 2
0
func main() {

	fmt.Printf("Start DB ....")
	logger, _ := utils.New("logname")

	logger.Info("info...")
	db := dl.NewSDatabase("testdb", "./", logger)
	db.AddData(nil)

}
Exemplo n.º 3
0
func TestNewDB(t *testing.T) {

	logger, _ := utils.New("test_db")

	db := NewSDatabase("testdb", "./testdata", logger)

	var feilds []FieldMeta
	err := json.Unmarshal([]byte(tbinfo), feilds)
	if err != nil {
		fmt.Printf("error\n")
	}

	fields := make([]FieldMeta, 0)
	fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "id", FieldType: 0, MkIdx: true})
	fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "name", FieldType: 0, MkIdx: true})
	fields = append(fields, FieldMeta{FieldLen: 10, Fieldname: "age", FieldType: 0, MkIdx: false})

	err = db.CreateTable("biao", fields)
	if err != nil {
		fmt.Printf("biao err\n")
	}

	content := make(map[string]string)

	content["id"] = "10"
	content["name"] = "abc"
	content["age"] = "20"

	for i := 0; i < 100000; i++ {
		db.AddData("biao", content)
	}

	res := db.FindDocId("biao", 3456)

	fmt.Printf("%v\n", res)

}
Exemplo n.º 4
0
func main() {

	fmt.Printf("init FalconEngine.....\n")
	//读取启动参数
	var configFile string
	var search string
	var cores int
	var err error
	flag.StringVar(&configFile, "conf", "search.conf", "configure file full path")
	flag.StringVar(&search, "mode", "search", "start mode[ search | build ]")
	flag.IntVar(&cores, "core", 4, "cpu cores")
	flag.Parse()

	runtime.GOMAXPROCS(cores)
	//读取配置文件
	configure, err := BaseFunctions.NewConfigure(configFile)
	if err != nil {
		fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err)
		return
	}

	//启动日志系统
	logger, err := utils.New("FalconEngine")
	if err != nil {
		fmt.Printf("[ERROR] Create logger Error: %v\n", err)
		//return
	}

	//初始化数据库适配器
	dbAdaptor, err := BaseFunctions.NewDBAdaptor(configure, logger)
	if err != nil {
		fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err)
		return
	}
	defer dbAdaptor.Release()

	//初始化本地redis
	/*
		redisClient, err := BaseFunctions.NewRedisClient(configure, logger)
		if err != nil {
			fmt.Printf("[ERROR] Create redisClient Error: %v\n", err)
			return
		}
		defer redisClient.Release()
	*/
	//初始化远程redis
	remoteRedisClient, err := BaseFunctions.NewRemoteRedisClient(configure, logger)
	if err != nil {
		fmt.Printf("[ERROR] Create redisClient Error: %v\n", err)
		return
	}
	defer remoteRedisClient.Release()

	if search == "search" {

		processor := &BaseFunctions.BaseProcessor{configure, logger, dbAdaptor /*redisClient*/, nil, remoteRedisClient}
		bitmap := utils.NewBitmap()
		fields, err := configure.GetTableFields()
		if err != nil {
			logger.Error("%v", err)
			return
		}
		index_set := indexer.NewIndexSet(bitmap, logger)
		index_set.InitIndexSet(fields)

		data_chan := make(chan builder.UpdateInfo, 1000)
		searcher := NewSearcher(processor, index_set, data_chan) // &Searcher{processor}
		updater := NewUpdater(processor, index_set, data_chan)
		updater.IncUpdating()
		router := &BaseFunctions.Router{configure, logger, map[string]BaseFunctions.FEProcessor{
			"search": searcher,
			"update": updater,
		}}

		builder := NewBuilderEngine(configure, dbAdaptor, logger /*redisClient*/, nil, index_set)
		builder.StartIncUpdate(data_chan)

		logger.Info("Server Start...")
		port, _ := configure.GetPort()
		addr := fmt.Sprintf(":%d", port)
		err = http.ListenAndServe(addr, router)
		if err != nil {
			logger.Error("Server start fail: %v", err)
			os.Exit(1)
		}

	} else if search == "build" {

		builder := NewBuilderEngine(configure, dbAdaptor, logger /*redisClient*/, nil, nil)
		builder.BuidingAllIndex()
	} else {
		logger.Error("Wrong start mode...only support [ search | build ]")
	}

}