Example #1
0
func (this *Index) SyncMemorySegment() error {

	if this.memorySegment == nil {
		return nil
	}
	this.idxSegmentMutex.Lock()
	defer this.idxSegmentMutex.Unlock()

	if this.memorySegment.MaxDocId == this.memorySegment.StartDocId {
		return nil
	}
	//this.Logger.Info("[INFO] Serialization Segment [%v]", this.memorySegment.SegmentName)
	if err := this.memorySegment.Serialization(); err != nil {
		this.Logger.Error("[ERROR] SyncMemorySegment Error %v", err)
		return err
	}
	segmentname := this.memorySegment.SegmentName
	this.memorySegment.Close()
	this.memorySegment = nil
	newSegment := fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
	this.segments = append(this.segments, newSegment)
	this.SegmentNames = append(this.SegmentNames, segmentname)

	return this.storeStruct()

}
Example #2
0
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
	this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
		SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
		memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
		Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
		dict: nil, pkmap: make(map[string]string)}

	metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	//delete by wuyinghao 不使用字典了
	//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
	//if utils.Exist(dictfilename) {
	//	this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
	//	this.dict = tree.NewBTDB(dictfilename, logger)
	//}

	for _, segmentname := range this.SegmentNames {
		segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
		this.segments = append(this.segments, segment)

	}

	//新建空的段
	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}

	this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++

	//读取bitmap
	bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
	this.bitmap = utils.NewBitmap(bitmapname)

	if this.PrimaryKey != "" {
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, logger)
	}

	this.Logger.Info("[INFO] Load Index %v success", this.Name)
	return this

}
Example #3
0
func (this *Index) MergeSegments(start int) error {

	var startIdx int = -1
	this.idxSegmentMutex.Lock()
	defer this.idxSegmentMutex.Unlock()
	//this.Logger.Info("[INFO] segment lenssssssssssss %v", len(this.segments))
	if len(this.segments) == 1 {
		return nil
	}
	if start < 0 {

		for idx := range this.segments {
			if this.segments[idx].MaxDocId-this.segments[idx].StartDocId < 1000000 {
				startIdx = idx
				break
			}
		}
	} else {

		if start >= len(this.segments)-1 {
			return nil
		}
		startIdx = start

	}

	if startIdx == -1 {
		return nil
	}

	mergeSegments := this.segments[startIdx:]

	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}
	tmpSegment := fis.NewEmptySegmentWithFieldsInfo(segmentname, mergeSegments[0].StartDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++
	if err := this.storeStruct(); err != nil {
		return err
	}
	tmpSegment.MergeSegments(mergeSegments)
	//tmpname:=tmpSegment.SegmentName
	tmpSegment.Close()
	tmpSegment = nil

	for _, sg := range mergeSegments {
		sg.Destroy()
	}

	tmpSegment = fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
	if startIdx > 0 {
		this.segments = this.segments[:startIdx]         //make([]*fis.Segment,0)
		this.SegmentNames = this.SegmentNames[:startIdx] //make([]string,0)
	} else {
		this.segments = make([]*fis.Segment, 0)
		this.SegmentNames = make([]string, 0)
	}

	this.segments = append(this.segments, tmpSegment)
	this.SegmentNames = append(this.SegmentNames, segmentname)
	return this.storeStruct()

}