// NewIndexWithLocalFile function description : 从文件载入索引 // params : // return : func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index { this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000, SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0), memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname, Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex), dict: nil, pkmap: make(map[string]string)} metaFileName := fmt.Sprintf("%v%v.meta", pathname, name) buffer, err := utils.ReadFromJson(metaFileName) if err != nil { return this } err = json.Unmarshal(buffer, &this) if err != nil { return this } //delete by wuyinghao 不使用字典了 //dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name) //if utils.Exist(dictfilename) { // this.Logger.Info("[INFO] Load dictfilename %v", dictfilename) // this.dict = tree.NewBTDB(dictfilename, logger) //} for _, segmentname := range this.SegmentNames { segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger) this.segments = append(this.segments, segment) } //新建空的段 segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ //读取bitmap bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name) this.bitmap = utils.NewBitmap(bitmapname) if this.PrimaryKey != "" { primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name) this.primary = tree.NewBTDB(primaryname, logger) } this.Logger.Info("[INFO] Load Index %v success", this.Name) return this }
func (this *Index) DeleteField(fieldname string) error { if _, ok := this.Fields[fieldname]; !ok { this.Logger.Warn("[WARN] field %v not found ", fieldname) return nil } if fieldname == this.PrimaryKey { this.Logger.Warn("[WARN] field %v is primary key can not delete ", fieldname) return nil } this.idxSegmentMutex.Lock() defer this.idxSegmentMutex.Unlock() if this.memorySegment == nil { this.memorySegment.DeleteField(fieldname) delete(this.Fields, fieldname) return this.storeStruct() } if this.memorySegment.IsEmpty() { this.memorySegment.DeleteField(fieldname) delete(this.Fields, fieldname) return this.storeStruct() } delete(this.Fields, fieldname) tmpsegment := this.memorySegment if err := tmpsegment.Serialization(); err != nil { return err } this.segments = append(this.segments, tmpsegment) this.SegmentNames = make([]string, 0) for _, seg := range this.segments { this.SegmentNames = append(this.SegmentNames, seg.SegmentName) } segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ return this.storeStruct() }
func (this *Index) MergeSegments(start int) error { var startIdx int = -1 this.idxSegmentMutex.Lock() defer this.idxSegmentMutex.Unlock() //this.Logger.Info("[INFO] segment lenssssssssssss %v", len(this.segments)) if len(this.segments) == 1 { return nil } if start < 0 { for idx := range this.segments { if this.segments[idx].MaxDocId-this.segments[idx].StartDocId < 1000000 { startIdx = idx break } } } else { if start >= len(this.segments)-1 { return nil } startIdx = start } if startIdx == -1 { return nil } mergeSegments := this.segments[startIdx:] segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } tmpSegment := fis.NewEmptySegmentWithFieldsInfo(segmentname, mergeSegments[0].StartDocId, fields, this.dict, this.Logger) this.PrefixSegment++ if err := this.storeStruct(); err != nil { return err } tmpSegment.MergeSegments(mergeSegments) //tmpname:=tmpSegment.SegmentName tmpSegment.Close() tmpSegment = nil for _, sg := range mergeSegments { sg.Destroy() } tmpSegment = fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger) if startIdx > 0 { this.segments = this.segments[:startIdx] //make([]*fis.Segment,0) this.SegmentNames = this.SegmentNames[:startIdx] //make([]string,0) } else { this.segments = make([]*fis.Segment, 0) this.SegmentNames = make([]string, 0) } this.segments = append(this.segments, tmpSegment) this.SegmentNames = append(this.SegmentNames, segmentname) return this.storeStruct() }
func (this *Index) UpdateDocument(content map[string]string, updateType uint64) (uint32, error) { if len(this.Fields) == 0 { this.Logger.Error("[ERROR] No Field or Segment is nil") return 0, errors.New("no field or segment is nil") } if this.memorySegment == nil { this.idxSegmentMutex.Lock() segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ if err := this.storeStruct(); err != nil { this.idxSegmentMutex.Unlock() return 0, err } this.idxSegmentMutex.Unlock() } docid := this.MaxDocId this.MaxDocId++ if updateType == utils.UPDATE_TYPE_ADD { //直接添加主键,不检查 if this.PrimaryKey != "" { this.pkmap[content[this.PrimaryKey]] = fmt.Sprintf("%v", docid) //if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil { // return 0, err //} if this.MaxDocId%50000 == 0 { startTime := time.Now() this.Logger.Debug("[INFO] start muti set %v", startTime) this.primary.MutiSet(this.PrimaryKey, this.pkmap) endTime := time.Now() this.Logger.Debug("[INFO] cost muti set %v", endTime.Sub(startTime)) this.pkmap = nil this.pkmap = make(map[string]string) } } //无主键的表直接添加 return docid, this.memorySegment.AddDocument(docid, content) } //this.Logger.Info("[INFO] update content %v", content) if _, hasPrimary := content[this.PrimaryKey]; !hasPrimary { this.Logger.Error("[ERROR] Primary Key Not Found %v", this.PrimaryKey) return 0, errors.New("No Primary Key") } oldDocid, founddoc := this.findPrimaryKey(content[this.PrimaryKey]) if founddoc { //this.Logger.Info("[INFO] old Doc ID %v new DocId %v ", oldDocid.Docid, docid) this.bitmap.SetBit(uint64(oldDocid.Docid), 1) } if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil { return 0, err } //this.Logger.Info("[INFO] AddDocument update content %v", content) return docid, this.memorySegment.AddDocument(docid, content) }
func (this *Index) AddField(field utils.SimpleFieldInfo) error { if _, ok := this.Fields[field.FieldName]; ok { this.Logger.Warn("[WARN] field %v Exist ", field.FieldName) return nil } this.Fields[field.FieldName] = field //delete by wuyinghao 不使用字典了 //if field.FieldType == utils.IDX_TYPE_STRING_SEG || // field.FieldType == utils.IDX_TYPE_STRING_SINGLE { // this.dict.AddBTree(field.FieldName) //} if field.FieldType == utils.IDX_TYPE_PK { this.PrimaryKey = field.FieldName primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name) this.primary = tree.NewBTDB(primaryname, this.Logger) this.primary.AddBTree(field.FieldName) } else { this.idxSegmentMutex.Lock() defer this.idxSegmentMutex.Unlock() if this.memorySegment == nil { segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ } else if this.memorySegment.IsEmpty() { err := this.memorySegment.AddField(field) if err != nil { this.Logger.Error("[ERROR] Add Field Error %v", err) return err } } else { tmpsegment := this.memorySegment if err := tmpsegment.Serialization(); err != nil { return err } this.segments = append(this.segments, tmpsegment) this.SegmentNames = make([]string, 0) for _, seg := range this.segments { this.SegmentNames = append(this.SegmentNames, seg.SegmentName) } segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment) var fields []utils.SimpleFieldInfo for _, f := range this.Fields { if f.FieldType != utils.IDX_TYPE_PK { fields = append(fields, f) } } this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger) this.PrefixSegment++ } } return this.storeStruct() }