// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//if utils.Exist(dictfilename) {
// this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
// this.dict = tree.NewBTDB(dictfilename, logger)
//}
for _, segmentname := range this.SegmentNames {
segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
this.segments = append(this.segments, segment)
}
//新建空的段
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
//读取bitmap
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
this.bitmap = utils.NewBitmap(bitmapname)
if this.PrimaryKey != "" {
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, logger)
}
this.Logger.Info("[INFO] Load Index %v success", this.Name)
return this
}
func (this *Index) DeleteField(fieldname string) error {
if _, ok := this.Fields[fieldname]; !ok {
this.Logger.Warn("[WARN] field %v not found ", fieldname)
return nil
}
if fieldname == this.PrimaryKey {
this.Logger.Warn("[WARN] field %v is primary key can not delete ", fieldname)
return nil
}
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
if this.memorySegment == nil {
this.memorySegment.DeleteField(fieldname)
delete(this.Fields, fieldname)
return this.storeStruct()
}
if this.memorySegment.IsEmpty() {
this.memorySegment.DeleteField(fieldname)
delete(this.Fields, fieldname)
return this.storeStruct()
}
delete(this.Fields, fieldname)
tmpsegment := this.memorySegment
if err := tmpsegment.Serialization(); err != nil {
return err
}
this.segments = append(this.segments, tmpsegment)
this.SegmentNames = make([]string, 0)
for _, seg := range this.segments {
this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
}
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
return this.storeStruct()
}
func (this *Index) MergeSegments(start int) error {
var startIdx int = -1
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
//this.Logger.Info("[INFO] segment lenssssssssssss %v", len(this.segments))
if len(this.segments) == 1 {
return nil
}
if start < 0 {
for idx := range this.segments {
if this.segments[idx].MaxDocId-this.segments[idx].StartDocId < 1000000 {
startIdx = idx
break
}
}
} else {
if start >= len(this.segments)-1 {
return nil
}
startIdx = start
}
if startIdx == -1 {
return nil
}
mergeSegments := this.segments[startIdx:]
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
tmpSegment := fis.NewEmptySegmentWithFieldsInfo(segmentname, mergeSegments[0].StartDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
if err := this.storeStruct(); err != nil {
return err
}
tmpSegment.MergeSegments(mergeSegments)
//tmpname:=tmpSegment.SegmentName
tmpSegment.Close()
tmpSegment = nil
for _, sg := range mergeSegments {
sg.Destroy()
}
tmpSegment = fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
if startIdx > 0 {
this.segments = this.segments[:startIdx] //make([]*fis.Segment,0)
this.SegmentNames = this.SegmentNames[:startIdx] //make([]string,0)
} else {
this.segments = make([]*fis.Segment, 0)
this.SegmentNames = make([]string, 0)
}
this.segments = append(this.segments, tmpSegment)
this.SegmentNames = append(this.SegmentNames, segmentname)
return this.storeStruct()
}
func (this *Index) UpdateDocument(content map[string]string, updateType uint64) (uint32, error) {
if len(this.Fields) == 0 {
this.Logger.Error("[ERROR] No Field or Segment is nil")
return 0, errors.New("no field or segment is nil")
}
if this.memorySegment == nil {
this.idxSegmentMutex.Lock()
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
if err := this.storeStruct(); err != nil {
this.idxSegmentMutex.Unlock()
return 0, err
}
this.idxSegmentMutex.Unlock()
}
docid := this.MaxDocId
this.MaxDocId++
if updateType == utils.UPDATE_TYPE_ADD {
//直接添加主键，不检查
if this.PrimaryKey != "" {
this.pkmap[content[this.PrimaryKey]] = fmt.Sprintf("%v", docid)
//if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil {
// return 0, err
//}
if this.MaxDocId%50000 == 0 {
startTime := time.Now()
this.Logger.Debug("[INFO] start muti set %v", startTime)
this.primary.MutiSet(this.PrimaryKey, this.pkmap)
endTime := time.Now()
this.Logger.Debug("[INFO] cost muti set %v", endTime.Sub(startTime))
this.pkmap = nil
this.pkmap = make(map[string]string)
}
}
//无主键的表直接添加
return docid, this.memorySegment.AddDocument(docid, content)
}
//this.Logger.Info("[INFO] update content %v", content)
if _, hasPrimary := content[this.PrimaryKey]; !hasPrimary {
this.Logger.Error("[ERROR] Primary Key Not Found %v", this.PrimaryKey)
return 0, errors.New("No Primary Key")
}
oldDocid, founddoc := this.findPrimaryKey(content[this.PrimaryKey])
if founddoc {
//this.Logger.Info("[INFO] old Doc ID %v new DocId %v ", oldDocid.Docid, docid)
this.bitmap.SetBit(uint64(oldDocid.Docid), 1)
}
if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil {
return 0, err
}
//this.Logger.Info("[INFO] AddDocument update content %v", content)
return docid, this.memorySegment.AddDocument(docid, content)
}
func (this *Index) AddField(field utils.SimpleFieldInfo) error {
if _, ok := this.Fields[field.FieldName]; ok {
this.Logger.Warn("[WARN] field %v Exist ", field.FieldName)
return nil
}
this.Fields[field.FieldName] = field
//delete by wuyinghao 不使用字典了
//if field.FieldType == utils.IDX_TYPE_STRING_SEG ||
// field.FieldType == utils.IDX_TYPE_STRING_SINGLE {
// this.dict.AddBTree(field.FieldName)
//}
if field.FieldType == utils.IDX_TYPE_PK {
this.PrimaryKey = field.FieldName
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, this.Logger)
this.primary.AddBTree(field.FieldName)
} else {
this.idxSegmentMutex.Lock()
defer this.idxSegmentMutex.Unlock()
if this.memorySegment == nil {
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
} else if this.memorySegment.IsEmpty() {
err := this.memorySegment.AddField(field)
if err != nil {
this.Logger.Error("[ERROR] Add Field Error %v", err)
return err
}
} else {
tmpsegment := this.memorySegment
if err := tmpsegment.Serialization(); err != nil {
return err
}
this.segments = append(this.segments, tmpsegment)
this.SegmentNames = make([]string, 0)
for _, seg := range this.segments {
this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
}
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
}
}
return this.storeStruct()
}