예제 #1
0
// NewSegmentWithLocalFile function description : 从文件重建一个段
// params :
// return :
func NewSegmentWithLocalFile(segmentname string, dict *tree.BTreedb, logger *utils.Log4FE) *Segment {

	this := &Segment{btdb: nil, StartDocId: 0, MaxDocId: 0, SegmentName: segmentname,
		idxMmap: nil, dtlMmap: nil, pflMmap: nil, Logger: logger, fields: make(map[string]*FSField),
		FieldInfos: make(map[string]utils.SimpleFieldInfo), isMemory: false, dict: dict}

	metaFileName := fmt.Sprintf("%v.meta", segmentname)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	btdbname := fmt.Sprintf("%v.bt", segmentname)
	if utils.Exist(btdbname) {
		this.Logger.Debug("[INFO] Load B+Tree File : %v", btdbname)
		this.btdb = tree.NewBTDB(btdbname, logger)
	}

	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Invert File : %v.idx ", segmentname)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Profile File : %v.pfl", segmentname)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", segmentname), utils.MODE_APPEND)
	if err != nil {
		fmt.Printf("mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)
	this.Logger.Debug("[INFO] Load Detail File : %v.dtl", segmentname)

	for _, field := range this.FieldInfos {
		if field.PflLen == 0 {
			indexer := newEmptyField(field.FieldName, this.StartDocId, field.FieldType, dict, logger)
			this.fields[field.FieldName] = indexer
			continue
		}
		indexer := newFieldWithLocalFile(field.FieldName, segmentname, this.StartDocId,
			this.MaxDocId, field.FieldType, field.PflOffset, field.PflLen,
			this.idxMmap, this.pflMmap, this.dtlMmap, false, this.btdb, dict, logger)
		this.fields[field.FieldName] = indexer
		//this.Logger.Info("[TRACE] %v", this.FieldInfos[field.FieldName])
	}

	return this

}
예제 #2
0
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
	this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
		SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
		memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
		Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
		dict: nil, pkmap: make(map[string]string)}

	metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	//delete by wuyinghao 不使用字典了
	//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
	//if utils.Exist(dictfilename) {
	//	this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
	//	this.dict = tree.NewBTDB(dictfilename, logger)
	//}

	for _, segmentname := range this.SegmentNames {
		segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
		this.segments = append(this.segments, segment)

	}

	//新建空的段
	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}

	this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++

	//读取bitmap
	bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
	this.bitmap = utils.NewBitmap(bitmapname)

	if this.PrimaryKey != "" {
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, logger)
	}

	this.Logger.Info("[INFO] Load Index %v success", this.Name)
	return this

}
예제 #3
0
func (this *Segment) MergeSegments(sgs []*Segment) error {

	this.Logger.Info("[INFO] MergeSegments [%v] Start", this.SegmentName)
	btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
	if this.btdb == nil {
		this.btdb = tree.NewBTDB(btdbname, this.Logger)
	}

	for name, field := range this.FieldInfos {
		//this.Logger.Info("[INFO] Merge Field[%v]", name)
		fs := make([]*FSField, 0)
		for _, sg := range sgs {
			if _, ok := sg.fields[name]; !ok {
				fakefield := newEmptyFakeField(this.fields[name].fieldName, sg.StartDocId,
					this.fields[name].fieldType,
					uint64(sg.MaxDocId-sg.StartDocId), nil, this.Logger)
				fs = append(fs, fakefield)
				continue
			}
			fs = append(fs, sg.fields[name])
		}
		this.fields[name].mergeField(fs, this.SegmentName, this.btdb)
		field.PflOffset = this.fields[name].pflOffset
		field.PflLen = this.fields[name].pflLen
		this.FieldInfos[name] = field

	}
	this.isMemory = false
	var err error
	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)

	for name := range this.fields {
		this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
	}
	this.Logger.Info("[INFO] MergeSegments [%v] Finish", this.SegmentName)
	this.MaxDocId = sgs[len(sgs)-1].MaxDocId

	return this.storeStruct()

}
예제 #4
0
// Serialization function description : 序列化
// params :
// return :
func (this *Segment) Serialization() error {

	btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
	if this.btdb == nil {
		this.btdb = tree.NewBTDB(btdbname, this.Logger)
	}
	this.Logger.Debug("[INFO] Serialization Segment File : [%v] Start", this.SegmentName)
	for name, field := range this.FieldInfos {
		if err := this.fields[name].serialization(this.SegmentName, this.btdb); err != nil {
			this.Logger.Error("[ERROR] Segment --> Serialization %v", err)
			return err
		}
		field.PflOffset = this.fields[name].pflOffset
		field.PflLen = this.fields[name].pflLen
		this.FieldInfos[field.FieldName] = field
		this.Logger.Trace("[TRACE] %v %v %v", name, field.PflOffset, field.PflLen)
	}

	if err := this.storeStruct(); err != nil {
		return err
	}

	this.isMemory = false

	var err error
	this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.idxMmap.SetFileEnd(0)
	//this.Logger.Info("[INFO] Read Invert File : %v.idx ", this.SegmentName)

	this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.pflMmap.SetFileEnd(0)

	this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
	if err != nil {
		this.Logger.Error("[ERROR] mmap error : %v \n", err)
	}
	this.dtlMmap.SetFileEnd(0)
	//this.Logger.Info("[INFO] Read Invert File : %v.pfl", this.SegmentName)

	for name := range this.fields {
		this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
	}
	this.Logger.Info("[INFO] Serialization Segment File : [%v] Finish", this.SegmentName)
	return nil

}
예제 #5
0
func (this *Index) AddField(field utils.SimpleFieldInfo) error {

	if _, ok := this.Fields[field.FieldName]; ok {
		this.Logger.Warn("[WARN] field %v Exist ", field.FieldName)
		return nil
	}

	this.Fields[field.FieldName] = field
	//delete by wuyinghao 不使用字典了
	//if field.FieldType == utils.IDX_TYPE_STRING_SEG ||
	//	field.FieldType == utils.IDX_TYPE_STRING_SINGLE {
	//	this.dict.AddBTree(field.FieldName)
	//}
	if field.FieldType == utils.IDX_TYPE_PK {
		this.PrimaryKey = field.FieldName
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, this.Logger)
		this.primary.AddBTree(field.FieldName)
	} else {
		this.idxSegmentMutex.Lock()
		defer this.idxSegmentMutex.Unlock()

		if this.memorySegment == nil {
			segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
			var fields []utils.SimpleFieldInfo
			for _, f := range this.Fields {
				if f.FieldType != utils.IDX_TYPE_PK {
					fields = append(fields, f)
				}

			}
			this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
			this.PrefixSegment++

		} else if this.memorySegment.IsEmpty() {
			err := this.memorySegment.AddField(field)
			if err != nil {
				this.Logger.Error("[ERROR] Add Field Error  %v", err)
				return err
			}
		} else {
			tmpsegment := this.memorySegment
			if err := tmpsegment.Serialization(); err != nil {
				return err
			}
			this.segments = append(this.segments, tmpsegment)
			this.SegmentNames = make([]string, 0)
			for _, seg := range this.segments {
				this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
			}

			segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
			var fields []utils.SimpleFieldInfo
			for _, f := range this.Fields {
				if f.FieldType != utils.IDX_TYPE_PK {
					fields = append(fields, f)
				}

			}
			this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
			this.PrefixSegment++

		}

	}
	return this.storeStruct()
}