// NewSegmentWithLocalFile function description : 从文件重建一个段
// params :
// return :
func NewSegmentWithLocalFile(segmentname string, dict *tree.BTreedb, logger *utils.Log4FE) *Segment {
this := &Segment{btdb: nil, StartDocId: 0, MaxDocId: 0, SegmentName: segmentname,
idxMmap: nil, dtlMmap: nil, pflMmap: nil, Logger: logger, fields: make(map[string]*FSField),
FieldInfos: make(map[string]utils.SimpleFieldInfo), isMemory: false, dict: dict}
metaFileName := fmt.Sprintf("%v.meta", segmentname)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
btdbname := fmt.Sprintf("%v.bt", segmentname)
if utils.Exist(btdbname) {
this.Logger.Debug("[INFO] Load B+Tree File : %v", btdbname)
this.btdb = tree.NewBTDB(btdbname, logger)
}
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Invert File : %v.idx ", segmentname)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Profile File : %v.pfl", segmentname)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Detail File : %v.dtl", segmentname)
for _, field := range this.FieldInfos {
if field.PflLen == 0 {
indexer := newEmptyField(field.FieldName, this.StartDocId, field.FieldType, dict, logger)
this.fields[field.FieldName] = indexer
continue
}
indexer := newFieldWithLocalFile(field.FieldName, segmentname, this.StartDocId,
this.MaxDocId, field.FieldType, field.PflOffset, field.PflLen,
this.idxMmap, this.pflMmap, this.dtlMmap, false, this.btdb, dict, logger)
this.fields[field.FieldName] = indexer
//this.Logger.Info("[TRACE] %v", this.FieldInfos[field.FieldName])
}
return this
}
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//if utils.Exist(dictfilename) {
// this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
// this.dict = tree.NewBTDB(dictfilename, logger)
//}
for _, segmentname := range this.SegmentNames {
segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
this.segments = append(this.segments, segment)
}
//新建空的段
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
//读取bitmap
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
this.bitmap = utils.NewBitmap(bitmapname)
if this.PrimaryKey != "" {
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, logger)
}
this.Logger.Info("[INFO] Load Index %v success", this.Name)
return this
}
func (this *Segment) MergeSegments(sgs []*Segment) error {
this.Logger.Info("[INFO] MergeSegments [%v] Start", this.SegmentName)
btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
if this.btdb == nil {
this.btdb = tree.NewBTDB(btdbname, this.Logger)
}
for name, field := range this.FieldInfos {
//this.Logger.Info("[INFO] Merge Field[%v]", name)
fs := make([]*FSField, 0)
for _, sg := range sgs {
if _, ok := sg.fields[name]; !ok {
fakefield := newEmptyFakeField(this.fields[name].fieldName, sg.StartDocId,
this.fields[name].fieldType,
uint64(sg.MaxDocId-sg.StartDocId), nil, this.Logger)
fs = append(fs, fakefield)
continue
}
fs = append(fs, sg.fields[name])
}
this.fields[name].mergeField(fs, this.SegmentName, this.btdb)
field.PflOffset = this.fields[name].pflOffset
field.PflLen = this.fields[name].pflLen
this.FieldInfos[name] = field
}
this.isMemory = false
var err error
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
for name := range this.fields {
this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
}
this.Logger.Info("[INFO] MergeSegments [%v] Finish", this.SegmentName)
this.MaxDocId = sgs[len(sgs)-1].MaxDocId
return this.storeStruct()
}
// Serialization function description : 序列化
// params :
// return :
func (this *Segment) Serialization() error {
btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
if this.btdb == nil {
this.btdb = tree.NewBTDB(btdbname, this.Logger)
}
this.Logger.Debug("[INFO] Serialization Segment File : [%v] Start", this.SegmentName)
for name, field := range this.FieldInfos {
if err := this.fields[name].serialization(this.SegmentName, this.btdb); err != nil {
this.Logger.Error("[ERROR] Segment --> Serialization %v", err)
return err
}
field.PflOffset = this.fields[name].pflOffset
field.PflLen = this.fields[name].pflLen
this.FieldInfos[field.FieldName] = field
this.Logger.Trace("[TRACE] %v %v %v", name, field.PflOffset, field.PflLen)
}
if err := this.storeStruct(); err != nil {
return err
}
this.isMemory = false
var err error
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
//this.Logger.Info("[INFO] Read Invert File : %v.idx ", this.SegmentName)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
//this.Logger.Info("[INFO] Read Invert File : %v.pfl", this.SegmentName)
for name := range this.fields {
this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
}
this.Logger.Info("[INFO] Serialization Segment File : [%v] Finish", this.SegmentName)
return nil
}
func (this *Index) AddField(field utils.SimpleFieldInfo) error {
if _, ok := this.Fields[field.FieldName]; ok {
this.Logger.Warn("[WARN] field %v Exist ", field.FieldName)
return nil
}
this.Fields[field.FieldName] = field
//delete by wuyinghao 不使用字典了
//if field.FieldType == utils.IDX_TYPE_STRING_SEG ||
// field.FieldType == utils.IDX_TYPE_STRING_SINGLE {
// this.dict.AddBTree(field.FieldName)
//}
if field.FieldType == utils.IDX_TYPE_PK {
this.PrimaryKey = field.FieldName
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, this.Logger)
this.primary.AddBTree(field.FieldName)
} else {
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
if this.memorySegment == nil {
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
} else if this.memorySegment.IsEmpty() {
err := this.memorySegment.AddField(field)
if err != nil {
this.Logger.Error("[ERROR] Add Field Error %v", err)
return err
}
} else {
tmpsegment := this.memorySegment
if err := tmpsegment.Serialization(); err != nil {
return err
}
this.segments = append(this.segments, tmpsegment)
this.SegmentNames = make([]string, 0)
for _, seg := range this.segments {
this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
}
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
}
}
return this.storeStruct()
}