示例#1
0
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
	this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
		SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
		memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
		Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
		dict: nil, pkmap: make(map[string]string)}

	metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	//delete by wuyinghao 不使用字典了
	//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
	//if utils.Exist(dictfilename) {
	//	this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
	//	this.dict = tree.NewBTDB(dictfilename, logger)
	//}

	for _, segmentname := range this.SegmentNames {
		segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
		this.segments = append(this.segments, segment)

	}

	//新建空的段
	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}

	this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++

	//读取bitmap
	bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
	this.bitmap = utils.NewBitmap(bitmapname)

	if this.PrimaryKey != "" {
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, logger)
	}

	this.Logger.Info("[INFO] Load Index %v success", this.Name)
	return this

}
示例#2
0
// NewEmptyIndex function description : 新建空索引
// params :
// return :
func NewEmptyIndex(name, pathname string, logger *utils.Log4FE) *Index {

	this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
		SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
		memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
		Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
		dict: nil, pkmap: make(map[string]string)}

	bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
	utils.MakeBitmapFile(bitmapname)
	this.bitmap = utils.NewBitmap(bitmapname)

	//delete by wuyinghao 不使用字典了
	//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
	//this.dict = tree.NewBTDB(dictfilename, logger)

	return this
}
示例#3
0
func main() {

	fmt.Printf("init FalconEngine.....\n")
	//读取启动参数
	var configFile string
	var search string
	var cores int
	var err error
	flag.StringVar(&configFile, "conf", "search.conf", "configure file full path")
	flag.StringVar(&search, "mode", "search", "start mode[ search | build ]")
	flag.IntVar(&cores, "core", 4, "cpu cores")
	flag.Parse()

	runtime.GOMAXPROCS(cores)
	//读取配置文件
	configure, err := BaseFunctions.NewConfigure(configFile)
	if err != nil {
		fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err)
		return
	}

	//启动日志系统
	logger, err := log4jzl.New("FalconEngine")
	if err != nil {
		fmt.Printf("[ERROR] Create logger Error: %v\n", err)
		//return
	}

	//初始化数据库适配器
	dbAdaptor, err := BaseFunctions.NewDBAdaptor(configure, logger)
	if err != nil {
		fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err)
		return
	}
	defer dbAdaptor.Release()

	//初始化本地redis
	redisClient, err := BaseFunctions.NewRedisClient(configure, logger)
	if err != nil {
		fmt.Printf("[ERROR] Create redisClient Error: %v\n", err)
		return
	}
	defer redisClient.Release()

	if search == "search" {

		processor := &BaseFunctions.BaseProcessor{configure, logger, dbAdaptor, redisClient}
		bitmap := utils.NewBitmap()
		fields, err := configure.GetTableFields()
		if err != nil {
			logger.Error("%v", err)
			return
		}
		index_set := indexer.NewIndexSet(bitmap, logger)
		index_set.InitIndexSet(fields)

		searcher := NewSearcher(processor, index_set) // &Searcher{processor}
		data_chan := make(chan builder.UpdateInfo, 1000)
		updater := NewUpdater(processor, index_set, data_chan)
		updater.IncUpdating()
		router := &BaseFunctions.Router{configure, logger, map[string]BaseFunctions.FEProcessor{
			"search": searcher,
			"update": updater,
		}}

		builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, index_set)
		builder.StartIncUpdate(data_chan)

		logger.Info("Server Start...")
		port, _ := configure.GetPort()
		addr := fmt.Sprintf(":%d", port)
		err = http.ListenAndServe(addr, router)
		if err != nil {
			logger.Error("Server start fail: %v", err)
			os.Exit(1)
		}

	} else if search == "build" {

		builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, nil)
		builder.BuidingAllIndex()
	} else {
		logger.Error("Wrong start mode...only support [ search | build ]")
	}

}