Example #1
0
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
	this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
		SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
		memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
		Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
		dict: nil, pkmap: make(map[string]string)}

	metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
	buffer, err := utils.ReadFromJson(metaFileName)
	if err != nil {
		return this
	}

	err = json.Unmarshal(buffer, &this)
	if err != nil {
		return this
	}

	//delete by wuyinghao 不使用字典了
	//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
	//if utils.Exist(dictfilename) {
	//	this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
	//	this.dict = tree.NewBTDB(dictfilename, logger)
	//}

	for _, segmentname := range this.SegmentNames {
		segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
		this.segments = append(this.segments, segment)

	}

	//新建空的段
	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}

	this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++

	//读取bitmap
	bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
	this.bitmap = utils.NewBitmap(bitmapname)

	if this.PrimaryKey != "" {
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, logger)
	}

	this.Logger.Info("[INFO] Load Index %v success", this.Name)
	return this

}
Example #2
0
func (this *Index) DeleteField(fieldname string) error {

	if _, ok := this.Fields[fieldname]; !ok {
		this.Logger.Warn("[WARN] field %v not found ", fieldname)
		return nil
	}

	if fieldname == this.PrimaryKey {
		this.Logger.Warn("[WARN] field %v is primary key can not delete ", fieldname)
		return nil
	}

	this.idxSegmentMutex.Lock()
	defer this.idxSegmentMutex.Unlock()

	if this.memorySegment == nil {
		this.memorySegment.DeleteField(fieldname)
		delete(this.Fields, fieldname)
		return this.storeStruct()
	}

	if this.memorySegment.IsEmpty() {
		this.memorySegment.DeleteField(fieldname)
		delete(this.Fields, fieldname)
		return this.storeStruct()
	}

	delete(this.Fields, fieldname)

	tmpsegment := this.memorySegment
	if err := tmpsegment.Serialization(); err != nil {
		return err
	}
	this.segments = append(this.segments, tmpsegment)
	this.SegmentNames = make([]string, 0)
	for _, seg := range this.segments {
		this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
	}

	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}
	this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++

	return this.storeStruct()

}
Example #3
0
func (this *Index) MergeSegments(start int) error {

	var startIdx int = -1
	this.idxSegmentMutex.Lock()
	defer this.idxSegmentMutex.Unlock()
	//this.Logger.Info("[INFO] segment lenssssssssssss %v", len(this.segments))
	if len(this.segments) == 1 {
		return nil
	}
	if start < 0 {

		for idx := range this.segments {
			if this.segments[idx].MaxDocId-this.segments[idx].StartDocId < 1000000 {
				startIdx = idx
				break
			}
		}
	} else {

		if start >= len(this.segments)-1 {
			return nil
		}
		startIdx = start

	}

	if startIdx == -1 {
		return nil
	}

	mergeSegments := this.segments[startIdx:]

	segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
	var fields []utils.SimpleFieldInfo
	for _, f := range this.Fields {
		if f.FieldType != utils.IDX_TYPE_PK {
			fields = append(fields, f)
		}

	}
	tmpSegment := fis.NewEmptySegmentWithFieldsInfo(segmentname, mergeSegments[0].StartDocId, fields, this.dict, this.Logger)
	this.PrefixSegment++
	if err := this.storeStruct(); err != nil {
		return err
	}
	tmpSegment.MergeSegments(mergeSegments)
	//tmpname:=tmpSegment.SegmentName
	tmpSegment.Close()
	tmpSegment = nil

	for _, sg := range mergeSegments {
		sg.Destroy()
	}

	tmpSegment = fis.NewSegmentWithLocalFile(segmentname, this.dict, this.Logger)
	if startIdx > 0 {
		this.segments = this.segments[:startIdx]         //make([]*fis.Segment,0)
		this.SegmentNames = this.SegmentNames[:startIdx] //make([]string,0)
	} else {
		this.segments = make([]*fis.Segment, 0)
		this.SegmentNames = make([]string, 0)
	}

	this.segments = append(this.segments, tmpSegment)
	this.SegmentNames = append(this.SegmentNames, segmentname)
	return this.storeStruct()

}
Example #4
0
func (this *Index) UpdateDocument(content map[string]string, updateType uint64) (uint32, error) {

	if len(this.Fields) == 0 {
		this.Logger.Error("[ERROR] No Field or Segment is nil")
		return 0, errors.New("no field or segment is nil")
	}

	if this.memorySegment == nil {
		this.idxSegmentMutex.Lock()
		segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
		var fields []utils.SimpleFieldInfo
		for _, f := range this.Fields {
			if f.FieldType != utils.IDX_TYPE_PK {
				fields = append(fields, f)
			}

		}
		this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
		this.PrefixSegment++
		if err := this.storeStruct(); err != nil {
			this.idxSegmentMutex.Unlock()
			return 0, err
		}
		this.idxSegmentMutex.Unlock()
	}

	docid := this.MaxDocId
	this.MaxDocId++

	if updateType == utils.UPDATE_TYPE_ADD {
		//直接添加主键,不检查
		if this.PrimaryKey != "" {
			this.pkmap[content[this.PrimaryKey]] = fmt.Sprintf("%v", docid)
			//if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil {
			//	return 0, err
			//}
			if this.MaxDocId%50000 == 0 {
				startTime := time.Now()
				this.Logger.Debug("[INFO] start muti set %v", startTime)
				this.primary.MutiSet(this.PrimaryKey, this.pkmap)
				endTime := time.Now()
				this.Logger.Debug("[INFO] cost  muti set  %v", endTime.Sub(startTime))
				this.pkmap = nil
				this.pkmap = make(map[string]string)
			}

		}
		//无主键的表直接添加
		return docid, this.memorySegment.AddDocument(docid, content)
	}
	//this.Logger.Info("[INFO] update content %v", content)
	if _, hasPrimary := content[this.PrimaryKey]; !hasPrimary {
		this.Logger.Error("[ERROR] Primary Key Not Found %v", this.PrimaryKey)
		return 0, errors.New("No Primary Key")
	}

	oldDocid, founddoc := this.findPrimaryKey(content[this.PrimaryKey])
	if founddoc {
		//this.Logger.Info("[INFO] old Doc ID %v  new DocId %v ", oldDocid.Docid, docid)
		this.bitmap.SetBit(uint64(oldDocid.Docid), 1)
	}

	if err := this.updatePrimaryKey(content[this.PrimaryKey], docid); err != nil {
		return 0, err
	}
	//this.Logger.Info("[INFO] AddDocument update content %v", content)
	return docid, this.memorySegment.AddDocument(docid, content)

}
Example #5
0
func (this *Index) AddField(field utils.SimpleFieldInfo) error {

	if _, ok := this.Fields[field.FieldName]; ok {
		this.Logger.Warn("[WARN] field %v Exist ", field.FieldName)
		return nil
	}

	this.Fields[field.FieldName] = field
	//delete by wuyinghao 不使用字典了
	//if field.FieldType == utils.IDX_TYPE_STRING_SEG ||
	//	field.FieldType == utils.IDX_TYPE_STRING_SINGLE {
	//	this.dict.AddBTree(field.FieldName)
	//}
	if field.FieldType == utils.IDX_TYPE_PK {
		this.PrimaryKey = field.FieldName
		primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
		this.primary = tree.NewBTDB(primaryname, this.Logger)
		this.primary.AddBTree(field.FieldName)
	} else {
		this.idxSegmentMutex.Lock()
		defer this.idxSegmentMutex.Unlock()

		if this.memorySegment == nil {
			segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
			var fields []utils.SimpleFieldInfo
			for _, f := range this.Fields {
				if f.FieldType != utils.IDX_TYPE_PK {
					fields = append(fields, f)
				}

			}
			this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
			this.PrefixSegment++

		} else if this.memorySegment.IsEmpty() {
			err := this.memorySegment.AddField(field)
			if err != nil {
				this.Logger.Error("[ERROR] Add Field Error  %v", err)
				return err
			}
		} else {
			tmpsegment := this.memorySegment
			if err := tmpsegment.Serialization(); err != nil {
				return err
			}
			this.segments = append(this.segments, tmpsegment)
			this.SegmentNames = make([]string, 0)
			for _, seg := range this.segments {
				this.SegmentNames = append(this.SegmentNames, seg.SegmentName)
			}

			segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
			var fields []utils.SimpleFieldInfo
			for _, f := range this.Fields {
				if f.FieldType != utils.IDX_TYPE_PK {
					fields = append(fields, f)
				}

			}
			this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
			this.PrefixSegment++

		}

	}
	return this.storeStruct()
}