/*****************************************************************************
* function name : NewIndexSet
* params :
* return :
*
* description : 默认初始化函数
*
******************************************************************************/
func NewIndexSet(bitmap *utils.Bitmap, logger *log4jzl.Log4jzl) *IndexSet {
segment := utils.NewSegmenter("./data/dictionary.txt")
builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}
this := &IndexSet{BitMap: bitmap, IncBuilder: builder, FieldInfo: make(map[string]*IndexFieldInfo), MaxDocId: 0, PrimaryKey: "PK", Segmenter: segment, IvtIndex: make(map[string]IndexInterface), Logger: logger, PflIndex: make(map[string]ProfileInterface)}
return this
}
func main() {
var cores int
var mport int
var lport int
var master int
var localip string
var masterip string
flag.IntVar(&cores, "core", runtime.NumCPU(), "CPU 核心数量")
flag.IntVar(&lport, "p", 9991, "启动端口，默认9991")
flag.IntVar(&master, "m", 0, "启动master，默认启动的为searcher")
flag.StringVar(&localip, "lip", "127.0.0.1", "本机ip地址，默认127.0.0.1")
flag.StringVar(&masterip, "mip", "127.0.0.1", "主节点ip地址，默认127.0.0.1")
flag.IntVar(&mport, "mp", 9990, "主节点端口，默认9990")
flag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
//debug.SetGCPercent(0)
fmt.Printf("CORES:%v\n", runtime.NumCPU())
//启动日志系统
logger, err := utils.New("FalconSearcher")
if err != nil {
fmt.Printf("[ERROR] Create logger Error: %v\n", err)
return
}
//初始化Manager
var engine utils.Engine
if master == 0 {
//初始化分词器
logger.Info("[INFO] Loading Segmenter ...")
utils.GSegmenter = utils.NewSegmenter("./data/dictionary.txt")
//utils.GOptions = utils.NewPy(utils.STYLE_NORMAL, utils.NO_SEGMENT)
logger.Info("[INFO] Init Search Engine ...")
engine = fe.NewDefaultEngine(localip, masterip, lport, mport, logger)
} else {
logger.Info("[INFO] Init Dispatcher ...")
engine = fd.NewDispatcher(localip, logger)
}
//启动性能监控
//go func() {
// log.Println(http.ListenAndServe(":6060", nil))
//}()
utils.GetDocIDsChan, utils.GiveDocIDsChan = utils.DocIdsMaker()
//启动全局缓存
// utils.Cache,err = cache.NewCache("memory", `{"interval":60}`)
// if err != nil {
// fmt.Printf("[ERROR] Create Cache Error: %v\n", err)
// return
// }
logger.Info("[INFO] Starting FalconEngine Service.....")
http := fs.NewHttpService(engine, lport, logger)
http.Start()
}
/*****************************************************************************
* function name : Buiding
* params :
* return :
*
* description : 构建索引文件，并将detail存入redis中，这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {
var fields string
for index, v := range this.Fields {
//构造sql语句
fields = fields + "," + v.Name
//构造索引数据指针
if v.IsIvt {
if v.FType == "T" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
this.Fields[index].IvtStrDic = utils.NewStringIdxDic(10000)
}
if v.FType == "N" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(10000)
}
}
if v.IsPlf {
if v.FType == "T" {
this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
}
if v.FType == "N" {
this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
}
}
}
sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
fmt.Printf("SQL :: %v \n", sql)
rows, err := this.Dbadaptor.QueryFormat(sql)
if err != nil {
this.Logger.Error(" %v", err)
return err
}
defer rows.Close()
var doc_id int64
doc_id = 1
segment := utils.NewSegmenter("./data/dictionary.txt")
builder := &utils.IndexBuilder{segment}
for rows.Next() {
//values := make([]interface{},len(this.Fields))
values := make([]interface{}, len(this.Fields))
writeCols := make([]string, len(this.Fields))
for i, _ := range writeCols {
values[i] = &writeCols[i]
}
err := rows.Scan(values...)
if err != nil {
this.Logger.Error("SQL ERROR : %v", err)
return err
}
redis_map := make(map[string]string)
for index, v := range writeCols {
redis_map[this.Fields[index].Name] = v
//v,_ := value.(string)
if this.Fields[index].IsIvt {
if this.Fields[index].FType == "T" {
err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
err := this.Fields[index].PlfText.PutProfile(doc_id, v)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
}
this.RedisCli.SetFields(doc_id, redis_map)
fieldlist := make([]string, 0)
for k, _ := range redis_map {
fieldlist = append(fieldlist, k)
}
doc_id++
this.Logger.Info("DOC_ID : %v VALUE : %v", doc_id, writeCols)
}
for index, fields := range this.Fields {
if this.Fields[index].IsIvt {
utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
if this.Fields[index].FType == "T" {
utils.WriteToJson(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
}
if this.Fields[index].FType == "N" {
utils.WriteToJson(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
utils.WriteToJson(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
}
if this.Fields[index].FType == "N" {
utils.WriteToJson(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
}
}
}
return nil
}
/*****************************************************************************
* function name : Buiding
* params :
* return :
*
* description : 构建索引文件，并将detail存入redis中，这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {
var fields string
for index, v := range this.Fields {
//构造sql语句
fields = fields + "," + v.Name
//构造索引数据指针
if v.IsIvt {
if v.FType == "T" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
this.Fields[index].IvtStrDic = utils.NewStringIdxDic(5001)
}
if v.FType == "N" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
if v.IsPK {
this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(1)
} else {
this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(4)
}
}
}
if v.IsPlf {
if v.FType == "T" {
this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
}
if v.FType == "N" {
this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
}
}
}
fmt.Printf("%v\n", fields)
this.DetailIdx = indexer.NewDetail()
sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
fmt.Printf("SQL :: %v \n", sql)
rows, err := this.Dbadaptor.QueryFormat(sql)
if err != nil {
this.Logger.Error(" %v", err)
return err
}
defer rows.Close()
var doc_id int64
doc_id = 1
segment := utils.NewSegmenter("./data/dictionary.txt")
builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}
for rows.Next() {
//values := make([]interface{},len(this.Fields))
values := make([]interface{}, len(this.Fields))
writeCols := make([]string, len(this.Fields))
for i, _ := range writeCols {
values[i] = &writeCols[i]
}
err := rows.Scan(values...)
if err != nil {
this.Logger.Error("SQL ERROR : %v", err)
return err
}
redis_map := make(map[string]string)
for index, v := range writeCols {
redis_map[this.Fields[index].Name] = v
//v,_ := value.(string)
if this.Fields[index].IsIvt {
if this.Fields[index].FType == "T" {
err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic, this.Fields[index].SType, false)
//err := builder.BuildTextIndexTemp(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic,this.Fields[index].SType,this.Fields[index].Name)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
v_num = 0
this.Logger.Warn("Warning : name : [%v] , value: [%v] , error : [%v]", this.Fields[index].Name, v, err)
}
err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic, false)
//err = builder.BuildNumberIndexTemp(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic,this.Fields[index].Name)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
//添加日期类型的更新，仅精确到天 add by wuyinghao 2015-08-21
if this.Fields[index].SType == 5 {
vl := strings.Split(v, " ")
v = vl[0]
}
err := this.Fields[index].PlfText.PutProfile(doc_id, v)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
v_num = 0
this.Logger.Warn("Warning : name : %v , value: %v , error : %v", this.Fields[index].Name, v, err)
}
err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
}
/////
//this.RedisCli.SetFields(doc_id, redis_map)
////
if this.DetailIdx.PutDocInfo(doc_id, redis_map) != nil {
this.Logger.Error("PutDocInfo doc_id Error : %v \n", err)
}
fieldlist := make([]string, 0)
for k, _ := range redis_map {
fieldlist = append(fieldlist, k)
}
doc_id++
if doc_id%5000 == 0 {
this.Logger.Info("processing doc_id : %v \n", doc_id)
}
//this.Logger.Info("DOC_ID : %v VALUE : %v", doc_id, writeCols)
}
this.DetailIdx.WriteDetailToFile()
//写入全部数据
//builder.WriteAllTempIndexToFile()
//builder.WriteIndexToFile()
//writeCount:=0
//var writeChan chan int
for index, fields := range this.Fields {
if this.Fields[index].IsIvt {
//utils.WriteToJsonWithChan(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name),writeChan)
//utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
utils.WriteToIndexFile(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.idx", fields.Name))
utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
//writeCount++
if this.Fields[index].FType == "T" {
//utils.WriteToJsonWithChan(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name),writeChan)
utils.WriteToJson(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
//writeCount++
}
if this.Fields[index].FType == "N" {
//utils.WriteToJsonWithChan(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name),writeChan)
utils.WriteToJson(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
//writeCount++
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
//utils.WriteToJsonWithChan(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name),writeChan)
utils.WriteToJson(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
//writeCount++
}
if this.Fields[index].FType == "N" {
//utils.WriteToJsonWithChan(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name),writeChan)
utils.WriteToJson(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
//writeCount++
}
}
}
/*
fmt.Printf("writeCount : %v \n ",writeCount)
for {
select{
case <-writeChan:
writeCount--
fmt.Printf("recive writeCount : %v \n ",writeCount)
if writeCount == 0 {
fmt.Printf("Finish building index...\n")
return nil
}
}
}
*/
return nil
}
/*****************************************************************************
* function name : Buiding
* params :
* return :
*
* description : 构建索引文件，并将detail存入redis中，这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {
var fields string
for index, v := range this.Fields {
//构造sql语句
fields = fields + "," + v.Name
//新建插件
cumstom := plugins.NewPlus(v.Name)
cumstom.Init()
//构造索引数据指针
if v.IsIvt {
if v.FType == "T" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
this.Fields[index].IvtStrDic = utils.NewStringIdxDic(v.Name)
this.Fields[index].CustomeInter = cumstom
}
if v.FType == "N" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(v.Name)
this.Fields[index].CustomeInter = cumstom
}
if v.FType == "I" {
this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
this.Fields[index].IvtStrDic = utils.NewStringIdxDic(v.Name)
this.Fields[index].CustomeInter = cumstom
}
}
if v.IsPlf {
if v.FType == "T" {
this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
this.Fields[index].PlfText.SetCustomInterface(cumstom)
}
if v.FType == "N" {
this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
this.Fields[index].PlfNumber.SetCustomInterface(cumstom)
}
if v.FType == "I" {
this.Fields[index].PlfByte = indexer.NewByteProfile(v.Name)
this.Fields[index].PlfByte.SetCustomInterface(cumstom)
}
}
}
//fmt.Printf("%v\n", fields)
this.DetailIdx = indexer.NewDetail()
sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
fmt.Printf("SQL :: %v \n", sql)
rows, err := this.Dbadaptor.QueryFormat(sql)
if err != nil {
this.Logger.Error(" %v", err)
return err
}
defer rows.Close()
var doc_id int64
doc_id = 1
segment := utils.NewSegmenter("./data/dictionary.txt")
builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}
for rows.Next() {
//values := make([]interface{},len(this.Fields))
values := make([]interface{}, len(this.Fields))
writeCols := make([]string, len(this.Fields))
for i, _ := range writeCols {
values[i] = &writeCols[i]
}
err := rows.Scan(values...)
if err != nil {
this.Logger.Error("SQL ERROR : %v", err)
return err
}
redis_map := make(map[string]string)
for index, v := range writeCols {
redis_map[this.Fields[index].Name] = v
//v,_ := value.(string)
if this.Fields[index].IsIvt {
if this.Fields[index].FType == "T" || this.Fields[index].FType == "I" {
err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic, this.Fields[index].SType, false, this.Fields[index].CustomeInter)
//err := builder.BuildTextIndexTemp(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic,this.Fields[index].SType,this.Fields[index].Name)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
v_num = 0
//this.Logger.Warn("Warning : name : [%v] , value: [%v] , error : [%v]", this.Fields[index].Name, v, err)
}
err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic, this.Fields[index].SType, false, this.Fields[index].CustomeInter)
//err = builder.BuildNumberIndexTemp(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic,this.Fields[index].Name)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
//添加日期类型的更新，仅精确到天 add by wuyinghao 2015-08-21
if this.Fields[index].SType == 5 {
vl := strings.Split(v, " ")
v = vl[0]
} else if this.Fields[index].SType == 9 {
v = this.Fields[index].PlfText.GetCustomInterface().BuildStringProfile(v)
}
err := this.Fields[index].PlfText.PutProfile(doc_id, v)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "N" {
v_num, err := strconv.ParseInt(v, 0, 0)
if err != nil {
v_num = 0
//this.Logger.Warn("Warning : name : %v , value: %v , error : %v", this.Fields[index].Name, v, err)
}
if this.Fields[index].SType == 9 {
v_num = this.Fields[index].PlfNumber.GetCustomInterface().BuildIntProfile(v)
}
err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
if this.Fields[index].FType == "I" {
v_byte := []byte(v)
if this.Fields[index].SType == 9 {
v_byte = this.Fields[index].PlfByte.GetCustomInterface().BuildByteProfile(v_byte)
}
err = this.Fields[index].PlfByte.PutProfile(doc_id, v_byte)
if err != nil {
this.Logger.Error("ERROR : %v", err)
}
}
}
}
/////
//this.RedisCli.SetFields(doc_id, redis_map)
////
//for k,v := range redis_map{
// fmt.Printf("\"%v\":\"%v\",",k,v)
//}
//fmt.Printf("\n")
if this.DetailIdx.PutDocInfo(doc_id, redis_map) != nil {
this.Logger.Error("PutDocInfo doc_id Error : %v \n", err)
}
fieldlist := make([]string, 0)
for k, _ := range redis_map {
fieldlist = append(fieldlist, k)
}
doc_id++
if doc_id%5000 == 0 {
this.Logger.Info("processing doc_id : %v \n", doc_id)
}
//this.Logger.Info("DOC_ID : %v VALUE : %v", doc_id, writeCols)
}
writeCount := 0
writeChan := make(chan string, 1000)
this.DetailIdx.WriteDetailToFile()
this.Logger.Info("Make BitMap File")
utils.MakeBitmapFile()
for index, _ := range this.Fields {
if this.Fields[index].IsIvt {
//utils.WriteToIndexFile(this.Fields[index].IvtIdx, fmt.Sprintf("./index/%v_idx.idx", this.Fields[index].Name))
this.Fields[index].IvtIdx.WriteToFile()
if this.Fields[index].FType == "T" || this.Fields[index].FType == "I" {
this.Fields[index].IvtStrDic.WriteToFile()
}
if this.Fields[index].FType == "N" {
this.Fields[index].IvtNumDic.WriteToFile()
}
}
if this.Fields[index].IsPlf {
if this.Fields[index].FType == "T" {
this.Fields[index].PlfText.WriteToFile()
}
if this.Fields[index].FType == "N" {
this.Fields[index].PlfNumber.WriteToFile()
}
if this.Fields[index].FType == "I" {
this.Fields[index].PlfByte.WriteToFile()
//go this.Fields[index].PlfByte.WriteToFileWithChan(writeChan)
//writeCount++
}
}
}
//fmt.Printf("Waiting %v threads\n ", writeCount)
if writeCount == 0 {
close(writeChan)
return nil
}
for {
select {
case file_name := <-writeChan:
writeCount--
fmt.Printf("Write [%v] finished \n ", file_name)
if writeCount == 0 {
fmt.Printf("Finish building all index...\n")
close(writeChan)
return nil
}
}
}
return nil
}