// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//if utils.Exist(dictfilename) {
// this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
// this.dict = tree.NewBTDB(dictfilename, logger)
//}
for _, segmentname := range this.SegmentNames {
segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
this.segments = append(this.segments, segment)
}
//新建空的段
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
//读取bitmap
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
this.bitmap = utils.NewBitmap(bitmapname)
if this.PrimaryKey != "" {
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, logger)
}
this.Logger.Info("[INFO] Load Index %v success", this.Name)
return this
}
// NewEmptyIndex function description : 新建空索引
// params :
// return :
func NewEmptyIndex(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
utils.MakeBitmapFile(bitmapname)
this.bitmap = utils.NewBitmap(bitmapname)
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//this.dict = tree.NewBTDB(dictfilename, logger)
return this
}
func main() {
fmt.Printf("init FalconEngine.....\n")
//读取启动参数
var configFile string
var search string
var cores int
var err error
flag.StringVar(&configFile, "conf", "search.conf", "configure file full path")
flag.StringVar(&search, "mode", "search", "start mode[ search | build ]")
flag.IntVar(&cores, "core", 4, "cpu cores")
flag.Parse()
runtime.GOMAXPROCS(cores)
//读取配置文件
configure, err := BaseFunctions.NewConfigure(configFile)
if err != nil {
fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err)
return
}
//启动日志系统
logger, err := log4jzl.New("FalconEngine")
if err != nil {
fmt.Printf("[ERROR] Create logger Error: %v\n", err)
//return
}
//初始化数据库适配器
dbAdaptor, err := BaseFunctions.NewDBAdaptor(configure, logger)
if err != nil {
fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err)
return
}
defer dbAdaptor.Release()
//初始化本地redis
redisClient, err := BaseFunctions.NewRedisClient(configure, logger)
if err != nil {
fmt.Printf("[ERROR] Create redisClient Error: %v\n", err)
return
}
defer redisClient.Release()
if search == "search" {
processor := &BaseFunctions.BaseProcessor{configure, logger, dbAdaptor, redisClient}
bitmap := utils.NewBitmap()
fields, err := configure.GetTableFields()
if err != nil {
logger.Error("%v", err)
return
}
index_set := indexer.NewIndexSet(bitmap, logger)
index_set.InitIndexSet(fields)
searcher := NewSearcher(processor, index_set) // &Searcher{processor}
data_chan := make(chan builder.UpdateInfo, 1000)
updater := NewUpdater(processor, index_set, data_chan)
updater.IncUpdating()
router := &BaseFunctions.Router{configure, logger, map[string]BaseFunctions.FEProcessor{
"search": searcher,
"update": updater,
}}
builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, index_set)
builder.StartIncUpdate(data_chan)
logger.Info("Server Start...")
port, _ := configure.GetPort()
addr := fmt.Sprintf(":%d", port)
err = http.ListenAndServe(addr, router)
if err != nil {
logger.Error("Server start fail: %v", err)
os.Exit(1)
}
} else if search == "build" {
builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, nil)
builder.BuidingAllIndex()
} else {
logger.Error("Wrong start mode...only support [ search | build ]")
}
}