Example #1
0
/*****************************************************************************
*  function name : NewIndexSet
*  params :
*  return :
*
*  description : 默认初始化函数
*
******************************************************************************/
func NewIndexSet(bitmap *utils.Bitmap, logger *log4jzl.Log4jzl) *IndexSet {
	segment := utils.NewSegmenter("./data/dictionary.txt")
	builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}
	this := &IndexSet{BitMap: bitmap, IncBuilder: builder, FieldInfo: make(map[string]*IndexFieldInfo), MaxDocId: 0, PrimaryKey: "PK", Segmenter: segment, IvtIndex: make(map[string]IndexInterface), Logger: logger, PflIndex: make(map[string]ProfileInterface)}
	return this

}
Example #2
0
func main() {

	var cores int
	var mport int
	var lport int
	var master int
	var localip string
	var masterip string
	flag.IntVar(&cores, "core", runtime.NumCPU(), "CPU 核心数量")
	flag.IntVar(&lport, "p", 9991, "启动端口,默认9991")
	flag.IntVar(&master, "m", 0, "启动master,默认启动的为searcher")
	flag.StringVar(&localip, "lip", "127.0.0.1", "本机ip地址,默认127.0.0.1")
	flag.StringVar(&masterip, "mip", "127.0.0.1", "主节点ip地址,默认127.0.0.1")
	flag.IntVar(&mport, "mp", 9990, "主节点端口,默认9990")
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	//debug.SetGCPercent(0)
	fmt.Printf("CORES:%v\n", runtime.NumCPU())
	//启动日志系统
	logger, err := utils.New("FalconSearcher")
	if err != nil {
		fmt.Printf("[ERROR] Create logger Error: %v\n", err)
		return
	}

	//初始化Manager
	var engine utils.Engine
	if master == 0 {
		//初始化分词器
		logger.Info("[INFO] Loading Segmenter ...")
		utils.GSegmenter = utils.NewSegmenter("./data/dictionary.txt")
		//utils.GOptions = utils.NewPy(utils.STYLE_NORMAL, utils.NO_SEGMENT)
		logger.Info("[INFO] Init Search Engine ...")
		engine = fe.NewDefaultEngine(localip, masterip, lport, mport, logger)
	} else {
		logger.Info("[INFO] Init Dispatcher ...")
		engine = fd.NewDispatcher(localip, logger)
	}

	//启动性能监控
	//go func() {
	//	log.Println(http.ListenAndServe(":6060", nil))
	//}()

	utils.GetDocIDsChan, utils.GiveDocIDsChan = utils.DocIdsMaker()

	//启动全局缓存
	// utils.Cache,err = cache.NewCache("memory", `{"interval":60}`)
	// if err != nil {
	//     fmt.Printf("[ERROR] Create Cache Error: %v\n", err)
	//		return
	// }

	logger.Info("[INFO] Starting FalconEngine Service.....")
	http := fs.NewHttpService(engine, lport, logger)

	http.Start()

}
Example #3
0
/*****************************************************************************
*  function name : Buiding
*  params :
*  return :
*
*  description : 构建索引文件,并将detail存入redis中,这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {

	var fields string
	for index, v := range this.Fields {
		//构造sql语句
		fields = fields + "," + v.Name
		//构造索引数据指针
		if v.IsIvt {

			if v.FType == "T" {
				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
				this.Fields[index].IvtStrDic = utils.NewStringIdxDic(10000)

			}

			if v.FType == "N" {
				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
				this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(10000)

			}
		}

		if v.IsPlf {
			if v.FType == "T" {
				this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
			}

			if v.FType == "N" {
				this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
			}

		}

	}

	sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
	fmt.Printf("SQL :: %v \n", sql)

	rows, err := this.Dbadaptor.QueryFormat(sql)
	if err != nil {
		this.Logger.Error(" %v", err)
		return err
	}
	defer rows.Close()
	var doc_id int64
	doc_id = 1
	segment := utils.NewSegmenter("./data/dictionary.txt")
	builder := &utils.IndexBuilder{segment}

	for rows.Next() {
		//values := make([]interface{},len(this.Fields))
		values := make([]interface{}, len(this.Fields))
		writeCols := make([]string, len(this.Fields))
		for i, _ := range writeCols {
			values[i] = &writeCols[i]
		}

		err := rows.Scan(values...)
		if err != nil {
			this.Logger.Error("SQL ERROR : %v", err)
			return err
		}
		redis_map := make(map[string]string)
		for index, v := range writeCols {
			redis_map[this.Fields[index].Name] = v
			//v,_ := value.(string)
			if this.Fields[index].IsIvt {

				if this.Fields[index].FType == "T" {
					err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}

					err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

			if this.Fields[index].IsPlf {

				if this.Fields[index].FType == "T" {

					err := this.Fields[index].PlfText.PutProfile(doc_id, v)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
					err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

		}

		this.RedisCli.SetFields(doc_id, redis_map)

		fieldlist := make([]string, 0)
		for k, _ := range redis_map {
			fieldlist = append(fieldlist, k)
		}

		doc_id++

		this.Logger.Info("DOC_ID : %v  VALUE : %v", doc_id, writeCols)

	}

	for index, fields := range this.Fields {

		if this.Fields[index].IsIvt {

			utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
			if this.Fields[index].FType == "T" {

				utils.WriteToJson(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))

			}

			if this.Fields[index].FType == "N" {

				utils.WriteToJson(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
			}

		}

		if this.Fields[index].IsPlf {

			if this.Fields[index].FType == "T" {

				utils.WriteToJson(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name))

			}

			if this.Fields[index].FType == "N" {

				utils.WriteToJson(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name))

			}

		}

	}

	return nil

}
Example #4
0
/*****************************************************************************
*  function name : Buiding
*  params :
*  return :
*
*  description : 构建索引文件,并将detail存入redis中,这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {

	var fields string
	for index, v := range this.Fields {
		//构造sql语句
		fields = fields + "," + v.Name
		//构造索引数据指针
		if v.IsIvt {

			if v.FType == "T" {
				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
				this.Fields[index].IvtStrDic = utils.NewStringIdxDic(5001)

			}

			if v.FType == "N" {
				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
				if v.IsPK {
					this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(1)
				} else {
					this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(4)
				}

			}
		}

		if v.IsPlf {
			if v.FType == "T" {
				this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
			}

			if v.FType == "N" {
				this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
			}

		}

	}
	fmt.Printf("%v\n", fields)

	this.DetailIdx = indexer.NewDetail()

	sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
	fmt.Printf("SQL :: %v \n", sql)

	rows, err := this.Dbadaptor.QueryFormat(sql)
	if err != nil {
		this.Logger.Error(" %v", err)
		return err
	}
	defer rows.Close()
	var doc_id int64
	doc_id = 1
	segment := utils.NewSegmenter("./data/dictionary.txt")
	builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}

	for rows.Next() {
		//values := make([]interface{},len(this.Fields))
		values := make([]interface{}, len(this.Fields))
		writeCols := make([]string, len(this.Fields))
		for i, _ := range writeCols {
			values[i] = &writeCols[i]
		}

		err := rows.Scan(values...)
		if err != nil {
			this.Logger.Error("SQL ERROR : %v", err)
			return err
		}
		redis_map := make(map[string]string)
		for index, v := range writeCols {
			redis_map[this.Fields[index].Name] = v
			//v,_ := value.(string)
			if this.Fields[index].IsIvt {

				if this.Fields[index].FType == "T" {
					err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic, this.Fields[index].SType, false)
					//err := builder.BuildTextIndexTemp(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic,this.Fields[index].SType,this.Fields[index].Name)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						v_num = 0
						this.Logger.Warn("Warning : name : [%v] , value: [%v] , error : [%v]", this.Fields[index].Name, v, err)
					}

					err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic, false)
					//err = builder.BuildNumberIndexTemp(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic,this.Fields[index].Name)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

			if this.Fields[index].IsPlf {

				if this.Fields[index].FType == "T" {
					//添加日期类型的更新,仅精确到天 add by wuyinghao 2015-08-21
					if this.Fields[index].SType == 5 {
						vl := strings.Split(v, " ")
						v = vl[0]
					}
					err := this.Fields[index].PlfText.PutProfile(doc_id, v)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						v_num = 0
						this.Logger.Warn("Warning : name : %v , value: %v , error : %v", this.Fields[index].Name, v, err)
					}
					err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

		}
		/////
		//this.RedisCli.SetFields(doc_id, redis_map)
		////
		if this.DetailIdx.PutDocInfo(doc_id, redis_map) != nil {
			this.Logger.Error("PutDocInfo doc_id Error :  %v \n", err)
		}

		fieldlist := make([]string, 0)
		for k, _ := range redis_map {
			fieldlist = append(fieldlist, k)
		}

		doc_id++
		if doc_id%5000 == 0 {
			this.Logger.Info("processing doc_id :  %v \n", doc_id)
		}
		//this.Logger.Info("DOC_ID : %v  VALUE : %v", doc_id, writeCols)

	}

	this.DetailIdx.WriteDetailToFile()

	//写入全部数据
	//builder.WriteAllTempIndexToFile()
	//builder.WriteIndexToFile()

	//writeCount:=0
	//var writeChan chan int
	for index, fields := range this.Fields {

		if this.Fields[index].IsIvt {

			//utils.WriteToJsonWithChan(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name),writeChan)
			//utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
			utils.WriteToIndexFile(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.idx", fields.Name))
			utils.WriteToJson(fields.IvtIdx, fmt.Sprintf("./index/%v_idx.json", fields.Name))
			//writeCount++
			if this.Fields[index].FType == "T" {

				//utils.WriteToJsonWithChan(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name),writeChan)
				utils.WriteToJson(fields.IvtStrDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
				//writeCount++

			}

			if this.Fields[index].FType == "N" {

				//utils.WriteToJsonWithChan(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name),writeChan)
				utils.WriteToJson(fields.IvtNumDic, fmt.Sprintf("./index/%v_dic.json", fields.Name))
				//writeCount++
			}

		}

		if this.Fields[index].IsPlf {

			if this.Fields[index].FType == "T" {

				//utils.WriteToJsonWithChan(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name),writeChan)
				utils.WriteToJson(fields.PlfText, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
				//writeCount++

			}

			if this.Fields[index].FType == "N" {

				//utils.WriteToJsonWithChan(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name),writeChan)
				utils.WriteToJson(fields.PlfNumber, fmt.Sprintf("./index/%v_pfl.json", fields.Name))
				//writeCount++

			}

		}

	}
	/*
		fmt.Printf("writeCount : %v \n ",writeCount)
		for {
			select{
				case <-writeChan:
					writeCount--
					fmt.Printf("recive writeCount : %v \n ",writeCount)
					if writeCount == 0 {
						fmt.Printf("Finish building index...\n")
						return nil
					}
			}
		}
	*/
	return nil

}
Example #5
0
/*****************************************************************************
*  function name : Buiding
*  params :
*  return :
*
*  description : 构建索引文件,并将detail存入redis中,这个函数太啰嗦了。
*
******************************************************************************/
func (this *DBBuilder) Buiding() error {

	var fields string
	for index, v := range this.Fields {
		//构造sql语句
		fields = fields + "," + v.Name

		//新建插件
		cumstom := plugins.NewPlus(v.Name)
		cumstom.Init()

		//构造索引数据指针
		if v.IsIvt {
			if v.FType == "T" {
				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
				this.Fields[index].IvtStrDic = utils.NewStringIdxDic(v.Name)
				this.Fields[index].CustomeInter = cumstom

			}

			if v.FType == "N" {

				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_NUM, v.Name)
				this.Fields[index].IvtNumDic = utils.NewNumberIdxDic(v.Name)
				this.Fields[index].CustomeInter = cumstom
			}

			if v.FType == "I" {

				this.Fields[index].IvtIdx = utils.NewInvertIdx(utils.TYPE_TEXT, v.Name)
				this.Fields[index].IvtStrDic = utils.NewStringIdxDic(v.Name)
				this.Fields[index].CustomeInter = cumstom

			}

		}

		if v.IsPlf {
			if v.FType == "T" {
				this.Fields[index].PlfText = indexer.NewTextProfile(v.Name)
				this.Fields[index].PlfText.SetCustomInterface(cumstom)
			}

			if v.FType == "N" {
				this.Fields[index].PlfNumber = indexer.NewNumberProfile(v.Name)
				this.Fields[index].PlfNumber.SetCustomInterface(cumstom)
			}

			if v.FType == "I" {
				this.Fields[index].PlfByte = indexer.NewByteProfile(v.Name)
				this.Fields[index].PlfByte.SetCustomInterface(cumstom)
			}

		}

	}
	//fmt.Printf("%v\n", fields)

	this.DetailIdx = indexer.NewDetail()

	sql := fmt.Sprintf(this.sql, fields[1:len(fields)])
	fmt.Printf("SQL :: %v \n", sql)

	rows, err := this.Dbadaptor.QueryFormat(sql)
	if err != nil {
		this.Logger.Error(" %v", err)
		return err
	}
	defer rows.Close()
	var doc_id int64
	doc_id = 1
	segment := utils.NewSegmenter("./data/dictionary.txt")
	builder := &utils.IndexBuilder{Segmenter: segment, TempIndex: make(map[string][]utils.TmpIdx), TempIndexNum: make(map[string]int64)}

	for rows.Next() {
		//values := make([]interface{},len(this.Fields))
		values := make([]interface{}, len(this.Fields))
		writeCols := make([]string, len(this.Fields))
		for i, _ := range writeCols {
			values[i] = &writeCols[i]
		}

		err := rows.Scan(values...)
		if err != nil {
			this.Logger.Error("SQL ERROR : %v", err)
			return err
		}
		redis_map := make(map[string]string)
		for index, v := range writeCols {
			redis_map[this.Fields[index].Name] = v
			//v,_ := value.(string)
			if this.Fields[index].IsIvt {

				if this.Fields[index].FType == "T" || this.Fields[index].FType == "I" {
					err := builder.BuildTextIndex(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic, this.Fields[index].SType, false, this.Fields[index].CustomeInter)
					//err := builder.BuildTextIndexTemp(doc_id, v, this.Fields[index].IvtIdx, this.Fields[index].IvtStrDic,this.Fields[index].SType,this.Fields[index].Name)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						v_num = 0
						//this.Logger.Warn("Warning : name : [%v] , value: [%v] , error : [%v]", this.Fields[index].Name, v, err)
					}

					err = builder.BuildNumberIndex(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic, this.Fields[index].SType, false, this.Fields[index].CustomeInter)
					//err = builder.BuildNumberIndexTemp(doc_id, v_num, this.Fields[index].IvtIdx, this.Fields[index].IvtNumDic,this.Fields[index].Name)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

			if this.Fields[index].IsPlf {

				if this.Fields[index].FType == "T" {
					//添加日期类型的更新,仅精确到天 add by wuyinghao 2015-08-21
					if this.Fields[index].SType == 5 {
						vl := strings.Split(v, " ")
						v = vl[0]
					} else if this.Fields[index].SType == 9 {
						v = this.Fields[index].PlfText.GetCustomInterface().BuildStringProfile(v)
					}
					err := this.Fields[index].PlfText.PutProfile(doc_id, v)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "N" {
					v_num, err := strconv.ParseInt(v, 0, 0)
					if err != nil {
						v_num = 0
						//this.Logger.Warn("Warning : name : %v , value: %v , error : %v", this.Fields[index].Name, v, err)
					}
					if this.Fields[index].SType == 9 {
						v_num = this.Fields[index].PlfNumber.GetCustomInterface().BuildIntProfile(v)
					}
					err = this.Fields[index].PlfNumber.PutProfile(doc_id, v_num)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

				if this.Fields[index].FType == "I" {
					v_byte := []byte(v)
					if this.Fields[index].SType == 9 {
						v_byte = this.Fields[index].PlfByte.GetCustomInterface().BuildByteProfile(v_byte)
					}
					err = this.Fields[index].PlfByte.PutProfile(doc_id, v_byte)
					if err != nil {
						this.Logger.Error("ERROR : %v", err)
					}
				}

			}

		}
		/////
		//this.RedisCli.SetFields(doc_id, redis_map)
		////
		//for k,v := range redis_map{
		//	fmt.Printf("\"%v\":\"%v\",",k,v)
		//}
		//fmt.Printf("\n")
		if this.DetailIdx.PutDocInfo(doc_id, redis_map) != nil {
			this.Logger.Error("PutDocInfo doc_id Error :  %v \n", err)
		}

		fieldlist := make([]string, 0)
		for k, _ := range redis_map {
			fieldlist = append(fieldlist, k)
		}

		doc_id++
		if doc_id%5000 == 0 {
			this.Logger.Info("processing doc_id :  %v \n", doc_id)
		}
		//this.Logger.Info("DOC_ID : %v  VALUE : %v", doc_id, writeCols)

	}

	writeCount := 0
	writeChan := make(chan string, 1000)

	this.DetailIdx.WriteDetailToFile()
	this.Logger.Info("Make BitMap File")
	utils.MakeBitmapFile()

	for index, _ := range this.Fields {
		if this.Fields[index].IsIvt {
			//utils.WriteToIndexFile(this.Fields[index].IvtIdx, fmt.Sprintf("./index/%v_idx.idx", this.Fields[index].Name))
			this.Fields[index].IvtIdx.WriteToFile()
			if this.Fields[index].FType == "T" || this.Fields[index].FType == "I" {

				this.Fields[index].IvtStrDic.WriteToFile()

			}

			if this.Fields[index].FType == "N" {
				this.Fields[index].IvtNumDic.WriteToFile()

			}

		}

		if this.Fields[index].IsPlf {

			if this.Fields[index].FType == "T" {

				this.Fields[index].PlfText.WriteToFile()

			}

			if this.Fields[index].FType == "N" {

				this.Fields[index].PlfNumber.WriteToFile()

			}

			if this.Fields[index].FType == "I" {

				this.Fields[index].PlfByte.WriteToFile()
				//go this.Fields[index].PlfByte.WriteToFileWithChan(writeChan)
				//writeCount++

			}

		}

	}

	//fmt.Printf("Waiting %v threads\n ", writeCount)
	if writeCount == 0 {
		close(writeChan)
		return nil
	}
	for {
		select {
		case file_name := <-writeChan:
			writeCount--
			fmt.Printf("Write [%v] finished \n ", file_name)
			if writeCount == 0 {
				fmt.Printf("Finish building all index...\n")
				close(writeChan)
				return nil
			}
		}
	}

	return nil

}