// NewIndexWithLocalFile function description : 从文件载入索引 // params : // return : func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index { this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000, SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0), memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname, Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex), dict: nil, pkmap: make(map[string]string)} metaFileName := fmt.Sprintf("%v%v.meta", pathname, name) buffer, err := utils.ReadFromJson(metaFileName) if err != nil { return this } err = json.Unmarshal(buffer, &this) if err != nil { return this } //delete by wuyinghao 不使用字典了 //dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name) //if utils.Exist(dictfilename) { // this.Logger.Info("[INFO] Load dictfilename %v", dictfilename) // this.dict = tree.NewBTDB(dictfilename, logger) //} for _, segmentname := range this.SegmentNames { segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger) this.segments = append(this.segments, segment) } //新建空的段 segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ //读取bitmap bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name) this.bitmap = utils.NewBitmap(bitmapname) if this.PrimaryKey != "" { primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name) this.primary = tree.NewBTDB(primaryname, logger) } this.Logger.Info("[INFO] Load Index %v success", this.Name) return this }
// NewEmptyIndex function description : 新建空索引 // params : // return : func NewEmptyIndex(name, pathname string, logger *utils.Log4FE) *Index { this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000, SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0), memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname, Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex), dict: nil, pkmap: make(map[string]string)} bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name) utils.MakeBitmapFile(bitmapname) this.bitmap = utils.NewBitmap(bitmapname) //delete by wuyinghao 不使用字典了 //dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name) //this.dict = tree.NewBTDB(dictfilename, logger) return this }
func main() { fmt.Printf("init FalconEngine.....\n") //读取启动参数 var configFile string var search string var cores int var err error flag.StringVar(&configFile, "conf", "search.conf", "configure file full path") flag.StringVar(&search, "mode", "search", "start mode[ search | build ]") flag.IntVar(&cores, "core", 4, "cpu cores") flag.Parse() runtime.GOMAXPROCS(cores) //读取配置文件 configure, err := BaseFunctions.NewConfigure(configFile) if err != nil { fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err) return } //启动日志系统 logger, err := log4jzl.New("FalconEngine") if err != nil { fmt.Printf("[ERROR] Create logger Error: %v\n", err) //return } //初始化数据库适配器 dbAdaptor, err := BaseFunctions.NewDBAdaptor(configure, logger) if err != nil { fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err) return } defer dbAdaptor.Release() //初始化本地redis redisClient, err := BaseFunctions.NewRedisClient(configure, logger) if err != nil { fmt.Printf("[ERROR] Create redisClient Error: %v\n", err) return } defer redisClient.Release() if search == "search" { processor := &BaseFunctions.BaseProcessor{configure, logger, dbAdaptor, redisClient} bitmap := utils.NewBitmap() fields, err := configure.GetTableFields() if err != nil { logger.Error("%v", err) return } index_set := indexer.NewIndexSet(bitmap, logger) index_set.InitIndexSet(fields) searcher := NewSearcher(processor, index_set) // &Searcher{processor} data_chan := make(chan builder.UpdateInfo, 1000) updater := NewUpdater(processor, index_set, data_chan) updater.IncUpdating() router := &BaseFunctions.Router{configure, logger, map[string]BaseFunctions.FEProcessor{ "search": searcher, "update": updater, }} builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, index_set) builder.StartIncUpdate(data_chan) logger.Info("Server Start...") port, _ := configure.GetPort() addr := fmt.Sprintf(":%d", port) err = http.ListenAndServe(addr, router) if err != nil { logger.Error("Server start fail: %v", err) os.Exit(1) } } else if search == "build" { builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, nil) builder.BuidingAllIndex() } else { logger.Error("Wrong start mode...only support [ search | build ]") } }