func OpenSDatabase(dbname, dbpath string, logger *utils.Log4FE) *SDatabase {
this := &SDatabase{Logger: logger, Pathname: dbpath, Name: dbname}
this.Fullname = dbpath + "/" + dbname
buffer, err := utils.ReadFromJson(this.Fullname + "/_dbinfo.meta")
if err != nil {
return nil
}
err = json.Unmarshal(buffer, this)
if err != nil {
return nil
}
for _, tablename := range this.TableNames {
//TODO
}
return this
}
// NewSegmentWithLocalFile function description : 从文件重建一个段
// params :
// return :
func NewSegmentWithLocalFile(segmentname string, dict *tree.BTreedb, logger *utils.Log4FE) *Segment {
this := &Segment{btdb: nil, StartDocId: 0, MaxDocId: 0, SegmentName: segmentname,
idxMmap: nil, dtlMmap: nil, pflMmap: nil, Logger: logger, fields: make(map[string]*FSField),
FieldInfos: make(map[string]utils.SimpleFieldInfo), isMemory: false, dict: dict}
metaFileName := fmt.Sprintf("%v.meta", segmentname)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
btdbname := fmt.Sprintf("%v.bt", segmentname)
if utils.Exist(btdbname) {
this.Logger.Debug("[INFO] Load B+Tree File : %v", btdbname)
this.btdb = tree.NewBTDB(btdbname, logger)
}
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Invert File : %v.idx ", segmentname)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Profile File : %v.pfl", segmentname)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Detail File : %v.dtl", segmentname)
for _, field := range this.FieldInfos {
if field.PflLen == 0 {
indexer := newEmptyField(field.FieldName, this.StartDocId, field.FieldType, dict, logger)
this.fields[field.FieldName] = indexer
continue
}
indexer := newFieldWithLocalFile(field.FieldName, segmentname, this.StartDocId,
this.MaxDocId, field.FieldType, field.PflOffset, field.PflLen,
this.idxMmap, this.pflMmap, this.dtlMmap, false, this.btdb, dict, logger)
this.fields[field.FieldName] = indexer
//this.Logger.Info("[TRACE] %v", this.FieldInfos[field.FieldName])
}
return this
}
// NewIndexWithLocalFile function description : 从文件载入索引
// params :
// return :
func NewIndexWithLocalFile(name, pathname string, logger *utils.Log4FE) *Index {
this := &Index{Name: name, Logger: logger, StartDocId: 0, MaxDocId: 0, PrefixSegment: 1000,
SegmentNames: make([]string, 0), PrimaryKey: "", segments: make([]*fis.Segment, 0),
memorySegment: nil, primary: nil, bitmap: nil, Pathname: pathname,
Fields: make(map[string]utils.SimpleFieldInfo), idxSegmentMutex: new(sync.Mutex),
dict: nil, pkmap: make(map[string]string)}
metaFileName := fmt.Sprintf("%v%v.meta", pathname, name)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
//delete by wuyinghao 不使用字典了
//dictfilename := fmt.Sprintf("%v%v_dict.dic", this.Pathname, this.Name)
//if utils.Exist(dictfilename) {
// this.Logger.Info("[INFO] Load dictfilename %v", dictfilename)
// this.dict = tree.NewBTDB(dictfilename, logger)
//}
for _, segmentname := range this.SegmentNames {
segment := fis.NewSegmentWithLocalFile(segmentname, this.dict, logger)
this.segments = append(this.segments, segment)
}
//新建空的段
segmentname := fmt.Sprintf("%v%v_%v", this.Pathname, this.Name, this.PrefixSegment)
var fields []utils.SimpleFieldInfo
for _, f := range this.Fields {
if f.FieldType != utils.IDX_TYPE_PK {
fields = append(fields, f)
}
}
this.memorySegment = fis.NewEmptySegmentWithFieldsInfo(segmentname, this.MaxDocId, fields, this.dict, this.Logger)
this.PrefixSegment++
//读取bitmap
bitmapname := fmt.Sprintf("%v%v.bitmap", pathname, name)
this.bitmap = utils.NewBitmap(bitmapname)
if this.PrimaryKey != "" {
primaryname := fmt.Sprintf("%v%v_primary.pk", this.Pathname, this.Name)
this.primary = tree.NewBTDB(primaryname, logger)
}
this.Logger.Info("[INFO] Load Index %v success", this.Name)
return this
}
/*****************************************************************************
* function name : InitIndexSet
* params : 需要初始化的字段
* return :
*
* description : 根据配置文件初始化建立索引和正排的字段并读入内存中
*
******************************************************************************/
func (this *IndexSet) InitIndexSet(fields map[string]string) error {
for k, v := range fields {
l := strings.Split(v, ",")
if len(l) != 5 {
this.Logger.Error("%v", errors.New("Wrong config file"))
return errors.New("Wrong configure for index")
}
this.FieldInfo[k] = &IndexFieldInfo{false, false, false, "N", k, 0}
if l[0] == "1" {
this.PrimaryKey = k
this.FieldInfo[k].IsPK = true
}
this.Logger.Info("========= Loading Index/Dictionary and Profile [ %v ] =========", k)
if l[1] == "1" {
this.FieldInfo[k].IsIvt = true
idx_name := fmt.Sprintf("./index/%v_idx.json", k)
dic_name := fmt.Sprintf("./index/%v_dic.json", k)
bidx, _ := utils.ReadFromJson(idx_name)
bdic, _ := utils.ReadFromJson(dic_name)
var idx utils.InvertIdx
err := json.Unmarshal(bidx, &idx)
if err != nil {
return err
}
this.Logger.Info("Loading Index [ %v ] ...", idx_name)
if l[3] == "T" { //text ivt
this.FieldInfo[k].FType = "T"
var dic utils.StringIdxDic
this.Logger.Info("Loading Index Dictionary [ %v ] type : Text ...", dic_name)
err = json.Unmarshal(bdic, &dic)
if err != nil {
return err
}
index := NewTextIndex(k, &idx, &dic)
this.PutIndex(k, index)
stype, err := strconv.ParseInt(l[4], 0, 0)
if err != nil {
return err
}
this.FieldInfo[k].SType = stype
} else { //number ivt
var dic utils.NumberIdxDic
this.Logger.Info("Loading Index Dictionary [ %v ] type : Number...", dic_name)
err = json.Unmarshal(bdic, &dic)
if err != nil {
return err
}
index := NewNumberIndex(k, &idx, &dic)
this.PutIndex(k, index)
}
}
if l[2] == "1" {
this.FieldInfo[k].IsPlf = true
pfl_name := fmt.Sprintf("./index/%v_pfl.json", k)
bpfl, _ := utils.ReadFromJson(pfl_name)
if l[3] == "T" {
this.FieldInfo[k].FType = "T"
var pfl TextProfile
this.Logger.Info("Loading Index Profile [ %v ] type : Text ...", pfl_name)
err := json.Unmarshal(bpfl, &pfl)
if err != nil {
return err
}
this.PutProfile(k, &pfl)
} else {
var pfl NumberProfile
this.Logger.Info("Loading Index Profile [ %v ] type : Number ...", pfl_name)
err := json.Unmarshal(bpfl, &pfl)
if err != nil {
return err
}
this.PutProfile(k, &pfl)
}
}
}
//读取detail文件
this.Logger.Info("Loading Detail idx .....")
bidx, err := utils.ReadFromJson("./index/detail.idx.json")
if err != nil {
this.Logger.Info("Read Detail Error .....%v ", err)
return err
}
var detail Detail
err = json.Unmarshal(bidx, &detail)
if err != nil {
this.Logger.Info("Loading Detail Error .....%v ", err)
return err
}
this.Detail = &detail
//保存最大DocId
this.MaxDocId = this.PflIndex[this.PrimaryKey].GetMaxDocId()
return nil
}
/*****************************************************************************
* function name : InitIndexSet
* params : 需要初始化的字段
* return :
*
* description : 根据配置文件初始化建立索引和正排的字段并读入内存中
*
******************************************************************************/
func (this *IndexSet) InitIndexSet(fields map[string]string) error {
for k, v := range fields {
//this.Logger.Info(" KEY:%v VALUE:%v\n",k,v)
l := strings.Split(v, ",")
if len(l) != 5 {
this.Logger.Error("%v", errors.New("Wrong config file"))
return errors.New("Wrong configure for index")
}
this.FieldInfo[k] = &IndexFieldInfo{false, false, false, "N", k, 0}
stype, err := strconv.ParseInt(l[4], 0, 0)
if err != nil {
this.Logger.Error("Error to ParseInt[%v], %v", l[4], err)
return err
}
this.FieldInfo[k].SType = stype
if l[0] == "1" {
this.PrimaryKey = k
this.FieldInfo[k].IsPK = true
}
this.Logger.Info("========= Loading Index/Dictionary and Profile [ %v ] =========", k)
if l[1] == "1" {
this.FieldInfo[k].IsIvt = true
idx := utils.NewInvertIdxWithName(k)
this.Logger.Info("\t Loading Invert Index [ %v.idx.dic ] ", k)
idx.ReadFromFile()
if l[3] == "T" { //text ivt
this.FieldInfo[k].FType = "T"
dic := utils.NewStringIdxDic(k)
this.Logger.Info("\t Loading Invert Index Dictionary [ %v.dic ] ", k)
dic.ReadFromFile()
index := NewTextIndex(k, idx, dic)
this.PutIndex(k, index)
} else { //number ivt
this.FieldInfo[k].FType = "N"
dic := utils.NewNumberIdxDic(k)
this.Logger.Info("\t Loading Invert Index Dictionary [ %v.dic ] ", k)
dic.ReadFromFile()
dic.Display()
index := NewNumberIndex(k, idx, dic)
this.PutIndex(k, index)
}
}
if l[2] == "1" {
this.FieldInfo[k].IsPlf = true
pfl_name := fmt.Sprintf("./index/%v_pfl.json", k)
bpfl, _ := utils.ReadFromJson(pfl_name)
if l[3] == "T" {
this.FieldInfo[k].FType = "T"
pfl := NewTextProfile(k)
this.Logger.Info("\t Loading Text Profile [ %v.pfl ] ", k)
pfl.ReadFromFile()
this.PutProfile(k, pfl)
} else if l[3] == "N" {
this.FieldInfo[k].FType = "N"
pfl := NewNumberProfile(k)
this.Logger.Info("\t Loading Number Profile [ %v.pfl ] ", k)
pfl.ReadFromFile()
this.PutProfile(k, pfl)
} else if l[3] == "I" {
this.FieldInfo[k].FType = "I"
var pfl ByteProfile
this.Logger.Info("\t Loading Byte Profile [ %v.pfl ] ", pfl_name)
err := json.Unmarshal(bpfl, &pfl)
if err != nil {
this.Logger.Error("Error to unmarshal[%v], %v", k, err)
return err
}
this.PutProfile(k, &pfl)
}
}
}
//读取detail文件
this.Logger.Info("Loading Detail idx .....")
this.Detail = NewDetailWithFile()
this.Detail.ReadDetailFromFile()
/*
bidx, err := utils.ReadFromJson("./index/detail.idx.json")
if err != nil {
this.Logger.Info("Read Detail Error .....%v ", err)
return err
}
var detail Detail
err = json.Unmarshal(bidx, &detail)
if err != nil {
this.Logger.Info("Loading Detail Error .....%v ", err)
return err
}
this.Detail = &detail
*/
//保存最大DocId
this.MaxDocId = this.PflIndex[this.PrimaryKey].GetMaxDocId()
return nil
}