Example #1
0
// NewSegmentWithLocalFile function description : 从文件重建一个段
// params :
// return :
func NewSegmentWithLocalFile(segmentname string, dict *tree.BTreedb, logger *utils.Log4FE) *Segment {

	this := &Segment{btdb: nil, StartDocId: 0, MaxDocId: 0, SegmentName: segmentname,
		idxMmap: nil, dtlMmap: nil, pflMmap: nil, Logger: logger, fields: make(map[string]*FSField),
		FieldInfos: make(map[string]utils.SimpleFieldInfo), isMemory: false, dict: dict}

	metaFileName := fmt.Sprintf("%v.meta", segmentname)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	btdbname := fmt.Sprintf("%v.bt", segmentname)
	if utils.Exist(btdbname) {
		this.Logger.Debug("[INFO] Load B+Tree File : %v", btdbname)
		this.btdb = tree.NewBTDB(btdbname, logger)
	}

	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Invert File : %v.idx ", segmentname)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Profile File : %v.pfl", segmentname)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Detail File : %v.dtl", segmentname)

	for _, field := range this.FieldInfos {
		if field.PflLen == 0 {
			indexer := newEmptyField(field.FieldName, this.StartDocId, field.FieldType, dict, logger)
			this.fields[field.FieldName] = indexer
			continue
		}
		indexer := newFieldWithLocalFile(field.FieldName, segmentname, this.StartDocId,
			this.MaxDocId, field.FieldType, field.PflOffset, field.PflLen,
			this.idxMmap, this.pflMmap, this.dtlMmap, false, this.btdb, dict, logger)
		this.fields[field.FieldName] = indexer
		//this.Logger.Info("[TRACE] %v", this.FieldInfos[field.FieldName])
	}

	return this

}
Example #2
0
func (this *Segment) MergeSegments(sgs []*Segment) error {

	this.Logger.Info("[INFO] MergeSegments [%v] Start", this.SegmentName)
	btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
	if this.btdb == nil {
		this.btdb = tree.NewBTDB(btdbname, this.Logger)
	}

	for name, field := range this.FieldInfos {
		//this.Logger.Info("[INFO] Merge Field[%v]", name)
		fs := make([]*FSField, 0)
		for _, sg := range sgs {
			if _, ok := sg.fields[name]; !ok {
				fakefield := newEmptyFakeField(this.fields[name].fieldName, sg.StartDocId,
					this.fields[name].fieldType,
					uint64(sg.MaxDocId-sg.StartDocId), nil, this.Logger)
				fs = append(fs, fakefield)
				continue
			}
			fs = append(fs, sg.fields[name])
		}
		this.fields[name].mergeField(fs, this.SegmentName, this.btdb)
		field.PflOffset = this.fields[name].pflOffset
		field.PflLen = this.fields[name].pflLen
		this.FieldInfos[name] = field

	}
	this.isMemory = false
	var err error
	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)

	for name := range this.fields {
		this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
	}
	this.Logger.Info("[INFO] MergeSegments [%v] Finish", this.SegmentName)
	this.MaxDocId = sgs[len(sgs)-1].MaxDocId

	return this.storeStruct()

}
Example #3
0
func (this *ByteProfile) ReadDetailFromFile() error {

	var err error
	file_name := fmt.Sprintf("./index/%v.dic", this.Name)
	fmt.Printf("Read File : %v \n", file_name)
	this.dicMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}

	this.DetailList = make([]DetailInfo, 0)

	this.MaxDocId = this.dicMmap.ReadInt64(0)
	this.Offset = this.dicMmap.ReadInt64(8)
	var start int64 = 16
	var i int64 = 0
	for i = 0; i <= this.MaxDocId; i++ {
		start_pos := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		byte_len := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		in_inc := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		this.DetailList = append(this.DetailList, DetailInfo{nil, start_pos, byte_len, in_inc, false})
	}
	this.dicMmap.SetFileEnd(start)

	//mmap详细文件
	file_name = fmt.Sprintf("./index/%v.dat", this.Name)
	fmt.Printf("Read File : %v \n", file_name)
	this.detailMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}

	//mmap增量文件
	file_name = fmt.Sprintf("./index/%v.up", this.Name)
	fmt.Printf("Read File : %v \n", file_name)
	this.upMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}
	up_file_end := this.upMmap.ReadInt64(0)
	if up_file_end == 0 {
		this.upMmap.SetFileEnd(8)
	} else {
		this.upMmap.SetFileEnd(up_file_end)
	}

	return nil

}
Example #4
0
// Serialization function description : 序列化
// params :
// return :
func (this *Segment) Serialization() error {

	btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
	if this.btdb == nil {
		this.btdb = tree.NewBTDB(btdbname, this.Logger)
	}
	this.Logger.Debug("[INFO] Serialization Segment File : [%v] Start", this.SegmentName)
	for name, field := range this.FieldInfos {
		if err := this.fields[name].serialization(this.SegmentName, this.btdb); err != nil {
			this.Logger.Error("[ERROR] Segment --> Serialization %v", err)
			return err
		}
		field.PflOffset = this.fields[name].pflOffset
		field.PflLen = this.fields[name].pflLen
		this.FieldInfos[field.FieldName] = field
		this.Logger.Trace("[TRACE] %v %v %v", name, field.PflOffset, field.PflLen)
	}

	if err := this.storeStruct(); err != nil {
		return err
	}

	this.isMemory = false

	var err error
	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)
	//this.Logger.Info("[INFO] Read Invert File : %v.idx ", this.SegmentName)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)
	//this.Logger.Info("[INFO] Read Invert File : %v.pfl", this.SegmentName)

	for name := range this.fields {
		this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
	}
	this.Logger.Info("[INFO] Serialization Segment File : [%v] Finish", this.SegmentName)
	return nil

}
Example #5
0
// createDetail function description : 创建表的详情文件
// params :
// return :
func (this *STable) createDetail() error {

	var err error

	this.detail, err = utils.NewMmap(this.Pathname+"/"+this.Tablename+TB_DTL_TAIL, utils.MODE_CREATE)

	if err != nil {
		this.Logger.Error("[ERROR] make detail error %v", err)
		return err
	}

	return nil

}
Example #6
0
func (this *NumberProfile) ReadFromFile() error {

	var err error
	file_name := fmt.Sprintf("./index/%v.plf", this.Name)
	this.numMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}

	/*
		f, err := os.Open(file_name)
		defer f.Close()
		if err != nil {
			return err
		}

		fi, err := f.Stat()
		if err != nil {
			fmt.Printf("ERR:%v", err)
		}

		MmapBytes, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_PRIVATE)

		if err != nil {
			fmt.Printf("MAPPING ERROR  %v \n", err)
			return nil
		}

		defer syscall.Munmap(MmapBytes)
	*/

	this.ProfileList = make([]int64, 0)
	this.Len = this.numMmap.ReadInt64(0)  //int64(binary.LittleEndian.Uint64(MmapBytes[:8]))
	this.Type = this.numMmap.ReadInt64(8) // int64(binary.LittleEndian.Uint64(MmapBytes[8:16]))
	//name_lens := int64(binary.LittleEndian.Uint64(MmapBytes[16:24]))
	//this.Name = string(MmapBytes[24 : 24+name_lens])
	var start int64 = 16 //24 + name_lens
	var i int64 = 0
	for i = 1; i < this.Len; i++ {
		value := this.numMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		this.ProfileList = append(this.ProfileList, value)
	}
	this.numMmap.SetFileEnd(start)
	this.IsSearch = true
	return nil
}
Example #7
0
func (this *NumberProfile) ReadFromFile() error {

	var err error
	file_name := fmt.Sprintf("./index/%v.pfl", this.Name)
	this.numMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}

	this.ProfileList = make([]int64, 0)
	this.Len = this.numMmap.ReadInt64(0)
	this.Type = this.numMmap.ReadInt64(8)
	var start int64 = 16
	var i int64 = 0
	for i = 0; i < this.Len; i++ {
		value := this.numMmap.ReadInt64(start)
		start += 8
		this.ProfileList = append(this.ProfileList, value)
	}
	this.numMmap.SetFileEnd(start)
	this.IsSearch = true
	return nil
}
Example #8
0
func (this *Dispatcher) CreateIndex(method string, parms map[string]string, body []byte) error {

	var idxstruct utils.IndexStrct

	if err := json.Unmarshal(body, &idxstruct); err != nil {
		this.Logger.Error("[ERROR] json error  %v", err)
		return err
	}

	if _, ok := this.IndexInfo[idxstruct.IndexName]; ok {
		this.Logger.Error("[ERROR] index [%v] already has ", idxstruct.IndexName)
		return fmt.Errorf("[ERROR] index [%v] already has ", idxstruct.IndexName)
	}

	sis := make([]ShardInfo, 0)
	seeds := make([]bool, len(this.NodeInfos))
	if len(this.NodeInfos) < int(idxstruct.ShardNum) {
		this.Logger.Error("[ERROR] ShardNum[%v] is to large", idxstruct.ShardNum)
		return fmt.Errorf("[ERROR] ShardNum[%v] is to large", idxstruct.ShardNum)
	}
	var nodeinfo utils.NodeIndex
	nodeinfo.IndexName = idxstruct.IndexName
	nodeinfo.ShardNum = idxstruct.ShardNum
	nodeinfo.IndexMapping = idxstruct.IndexMapping
	nodeinfo.ShardNodes = make(map[uint64][]string)
	//nodeinfo.Shard = make([]uint64, idxstruct.ShardNum)
	for i := uint64(0); i < idxstruct.ShardNum; i++ {
		si := ShardInfo{ShardPathName: fmt.Sprintf("%v/%v_%v", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i),
			MaxOffset: 0, NodeInfos: make([]string, 0)}
		si.OffsetMmap, _ = utils.NewMmap(fmt.Sprintf("%v/%v_%v.offset", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i), utils.MODE_APPEND)
		si.DetailMmap, _ = utils.NewMmap(fmt.Sprintf("%v/%v_%v.dtl", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i), utils.MODE_APPEND)

		for {
			seed := rand.Intn(len(seeds))
			if seeds[seed] == false {
				seeds[seed] = true
				net := make([]string, 0)
				si.NodeInfos = append(si.NodeInfos, fmt.Sprintf("%v:%v", this.NodeInfos[seed].Addr, this.NodeInfos[seed].MPort))
				net = append(net, fmt.Sprintf("%v:%v", this.NodeInfos[seed].Addr, this.NodeInfos[seed].MPort))

				nodeinfo.ShardNodes[uint64(seed)] = net
				this.Logger.Info("[INFO] NodeInfos::: %v", si.NodeInfos)
				break
			}
		}

		sis = append(sis, si)

	}
	info := idxInfo{Name: idxstruct.IndexName, ShardNum: idxstruct.ShardNum, MaxCnt: 0,
		IndexMapping: idxstruct.IndexMapping, ShardInfos: sis, ShardField: idxstruct.ShardField}

	this.IndexInfo[idxstruct.IndexName] = info

	for _, iinfo := range this.NodeInfos {
		iinfo.IdxChan <- nodeinfo
	}

	return this.storeStruct()

}
Example #9
0
func (this *Detail) ReadDetailFromFile() error {

	var err error
	file_name := "./index/detail.dic"
	this.dicMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}
	/*
		f, err := os.Open(file_name)
		defer f.Close()
		if err != nil {
			return err
		}

		fi, err := f.Stat()
		if err != nil {
			fmt.Printf("ERR:%v", err)
		}

		MmapBytes, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_PRIVATE)

		if err != nil {
			fmt.Printf("MAPPING ERROR  %v \n", err)
			return nil
		}

		defer syscall.Munmap(MmapBytes)


		this.MaxDocId = int64(binary.LittleEndian.Uint64(MmapBytes[:8]))
		this.Offset = int64(binary.LittleEndian.Uint64(MmapBytes[8:16]))
		var start int64 = 16
		var i int64 = 0
		for i = 0; i <= this.MaxDocId; i++ {
			start_pos := int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
			start += 8
			byte_len := int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
			start += 8
			this.DetailList = append(this.DetailList, DetailInfo{nil,start_pos,byte_len,false})
		}
	*/
	this.MaxDocId = this.dicMmap.ReadInt64(0)
	this.Offset = this.dicMmap.ReadInt64(8)
	var start int64 = 16
	var i int64 = 0
	for i = 0; i <= this.MaxDocId; i++ {
		start_pos := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		byte_len := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		in_inc := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
		start += 8
		this.DetailList = append(this.DetailList, DetailInfo{nil, start_pos, byte_len, in_inc, false})
	}
	this.dicMmap.SetFileEnd(start)

	//mmap详细文件
	file_name = "./index/detail.dat"
	this.detailMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}

	//mmap增量文件
	file_name = "./index/detail.up"
	this.upMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
		return err
	}
	up_file_end := this.upMmap.ReadInt64(0)
	if up_file_end == 0 {
		this.upMmap.SetFileEnd(8)
	} else {
		this.upMmap.SetFileEnd(up_file_end)
	}

	return nil

}