func (this *Index) SyncMemorySegment() error {
if this.memorySegment == nil {
return nil
}
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
if this.memorySegment.MaxDocId == this.memorySegment.StartDocId {
return nil
}
//this.Logger.Info("[INFO] Serialization Segment [%v]", this.memorySegment.SegmentName)
if err := this.memorySegment.Serialization(); err != nil {
this.Logger.Error("[ERROR] SyncMemorySegment Error %v", err)
return err
}
segmentname := this.memorySegment.SegmentName
this.memorySegment.Close()
this.memorySegment = nil
newSegment := fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
this.segments = append(this.segments, newSegment)
this.SegmentNames = append(this.SegmentNames, segmentname)
return this.storeStruct()
}
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//if utils.Exist(dictfilename) {
// this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
// this.dict = tree.NewBTDB(dictfilename, logger)
//}
for _, segmentname := range this.SegmentNames {
segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
this.segments = append(this.segments, segment)
}
//新建空的段
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
//读取bitmap
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
this.bitmap = utils.NewBitmap(bitmapname)
if this.PrimaryKey != "" {
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, logger)
}
this.Logger.Info("[INFO] Load Index %v success", this.Name)
return this
}
func (this *Index) MergeSegments(start int) error {
var startIdx int = -1
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
//this.Logger.Info("[INFO] segment lenssssssssssss %v", len(this.segments))
if len(this.segments) == 1 {
return nil
}
if start < 0 {
for idx := range this.segments {
if this.segments[idx].MaxDocId-this.segments[idx].StartDocId < 1000000 {
startIdx = idx
break
}
}
} else {
if start >= len(this.segments)-1 {
return nil
}
startIdx = start
}
if startIdx == -1 {
return nil
}
mergeSegments := this.segments[startIdx:]
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
tmpSegment := fis.NewEmptySegmentWithFieldsInfo(segmentname, mergeSegments[0].StartDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
if err := this.storeStruct(); err != nil {
return err
}
tmpSegment.MergeSegments(mergeSegments)
//tmpname:=tmpSegment.SegmentName
tmpSegment.Close()
tmpSegment = nil
for _, sg := range mergeSegments {
sg.Destroy()
}
tmpSegment = fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
if startIdx > 0 {
this.segments = this.segments[:startIdx] //make([]*fis.Segment,0)
this.SegmentNames = this.SegmentNames[:startIdx] //make([]string,0)
} else {
this.segments = make([]*fis.Segment, 0)
this.SegmentNames = make([]string, 0)
}
this.segments = append(this.segments, tmpSegment)
this.SegmentNames = append(this.SegmentNames, segmentname)
return this.storeStruct()
}