// NewSegmentWithLocalFile function description : 从文件重建一个段
// params :
// return :
func NewSegmentWithLocalFile(segmentname string, dict *tree.BTreedb, logger *utils.Log4FE) *Segment {
this := &Segment{btdb: nil, StartDocId: 0, MaxDocId: 0, SegmentName: segmentname,
idxMmap: nil, dtlMmap: nil, pflMmap: nil, Logger: logger, fields: make(map[string]*FSField),
FieldInfos: make(map[string]utils.SimpleFieldInfo), isMemory: false, dict: dict}
metaFileName := fmt.Sprintf("%v.meta", segmentname)
buffer, err := utils.ReadFromJson(metaFileName)
if err != nil {
return this
}
err = json.Unmarshal(buffer, &this)
if err != nil {
return this
}
btdbname := fmt.Sprintf("%v.bt", segmentname)
if utils.Exist(btdbname) {
this.Logger.Debug("[INFO] Load B+Tree File : %v", btdbname)
this.btdb = tree.NewBTDB(btdbname, logger)
}
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Invert File : %v.idx ", segmentname)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Profile File : %v.pfl", segmentname)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", segmentname), utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
this.Logger.Debug("[INFO] Load Detail File : %v.dtl", segmentname)
for _, field := range this.FieldInfos {
if field.PflLen == 0 {
indexer := newEmptyField(field.FieldName, this.StartDocId, field.FieldType, dict, logger)
this.fields[field.FieldName] = indexer
continue
}
indexer := newFieldWithLocalFile(field.FieldName, segmentname, this.StartDocId,
this.MaxDocId, field.FieldType, field.PflOffset, field.PflLen,
this.idxMmap, this.pflMmap, this.dtlMmap, false, this.btdb, dict, logger)
this.fields[field.FieldName] = indexer
//this.Logger.Info("[TRACE] %v", this.FieldInfos[field.FieldName])
}
return this
}
func (this *Segment) MergeSegments(sgs []*Segment) error {
this.Logger.Info("[INFO] MergeSegments [%v] Start", this.SegmentName)
btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
if this.btdb == nil {
this.btdb = tree.NewBTDB(btdbname, this.Logger)
}
for name, field := range this.FieldInfos {
//this.Logger.Info("[INFO] Merge Field[%v]", name)
fs := make([]*FSField, 0)
for _, sg := range sgs {
if _, ok := sg.fields[name]; !ok {
fakefield := newEmptyFakeField(this.fields[name].fieldName, sg.StartDocId,
this.fields[name].fieldType,
uint64(sg.MaxDocId-sg.StartDocId), nil, this.Logger)
fs = append(fs, fakefield)
continue
}
fs = append(fs, sg.fields[name])
}
this.fields[name].mergeField(fs, this.SegmentName, this.btdb)
field.PflOffset = this.fields[name].pflOffset
field.PflLen = this.fields[name].pflLen
this.FieldInfos[name] = field
}
this.isMemory = false
var err error
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
for name := range this.fields {
this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
}
this.Logger.Info("[INFO] MergeSegments [%v] Finish", this.SegmentName)
this.MaxDocId = sgs[len(sgs)-1].MaxDocId
return this.storeStruct()
}
func (this *ByteProfile) ReadDetailFromFile() error {
var err error
file_name := fmt.Sprintf("./index/%v.dic", this.Name)
fmt.Printf("Read File : %v \n", file_name)
this.dicMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
this.DetailList = make([]DetailInfo, 0)
this.MaxDocId = this.dicMmap.ReadInt64(0)
this.Offset = this.dicMmap.ReadInt64(8)
var start int64 = 16
var i int64 = 0
for i = 0; i <= this.MaxDocId; i++ {
start_pos := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
byte_len := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
in_inc := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
this.DetailList = append(this.DetailList, DetailInfo{nil, start_pos, byte_len, in_inc, false})
}
this.dicMmap.SetFileEnd(start)
//mmap详细文件
file_name = fmt.Sprintf("./index/%v.dat", this.Name)
fmt.Printf("Read File : %v \n", file_name)
this.detailMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
//mmap增量文件
file_name = fmt.Sprintf("./index/%v.up", this.Name)
fmt.Printf("Read File : %v \n", file_name)
this.upMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
up_file_end := this.upMmap.ReadInt64(0)
if up_file_end == 0 {
this.upMmap.SetFileEnd(8)
} else {
this.upMmap.SetFileEnd(up_file_end)
}
return nil
}
// Serialization function description : 序列化
// params :
// return :
func (this *Segment) Serialization() error {
btdbname := fmt.Sprintf("%v.bt", this.SegmentName)
if this.btdb == nil {
this.btdb = tree.NewBTDB(btdbname, this.Logger)
}
this.Logger.Debug("[INFO] Serialization Segment File : [%v] Start", this.SegmentName)
for name, field := range this.FieldInfos {
if err := this.fields[name].serialization(this.SegmentName, this.btdb); err != nil {
this.Logger.Error("[ERROR] Segment --> Serialization %v", err)
return err
}
field.PflOffset = this.fields[name].pflOffset
field.PflLen = this.fields[name].pflLen
this.FieldInfos[field.FieldName] = field
this.Logger.Trace("[TRACE] %v %v %v", name, field.PflOffset, field.PflLen)
}
if err := this.storeStruct(); err != nil {
return err
}
this.isMemory = false
var err error
this.idxMmap, err = utils.NewMmap(fmt.Sprintf("%v.idx", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.idxMmap.SetFileEnd(0)
//this.Logger.Info("[INFO] Read Invert File : %v.idx ", this.SegmentName)
this.pflMmap, err = utils.NewMmap(fmt.Sprintf("%v.pfl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.pflMmap.SetFileEnd(0)
this.dtlMmap, err = utils.NewMmap(fmt.Sprintf("%v.dtl", this.SegmentName), utils.MODE_APPEND)
if err != nil {
this.Logger.Error("[ERROR] mmap error : %v \n", err)
}
this.dtlMmap.SetFileEnd(0)
//this.Logger.Info("[INFO] Read Invert File : %v.pfl", this.SegmentName)
for name := range this.fields {
this.fields[name].setMmap(this.idxMmap, this.pflMmap, this.dtlMmap)
}
this.Logger.Info("[INFO] Serialization Segment File : [%v] Finish", this.SegmentName)
return nil
}
// createDetail function description : 创建表的详情文件
// params :
// return :
func (this *STable) createDetail() error {
var err error
this.detail, err = utils.NewMmap(this.Pathname+"/"+this.Tablename+TB_DTL_TAIL, utils.MODE_CREATE)
if err != nil {
this.Logger.Error("[ERROR] make detail error %v", err)
return err
}
return nil
}
func (this *NumberProfile) ReadFromFile() error {
var err error
file_name := fmt.Sprintf("./index/%v.plf", this.Name)
this.numMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
/*
f, err := os.Open(file_name)
defer f.Close()
if err != nil {
return err
}
fi, err := f.Stat()
if err != nil {
fmt.Printf("ERR:%v", err)
}
MmapBytes, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
fmt.Printf("MAPPING ERROR %v \n", err)
return nil
}
defer syscall.Munmap(MmapBytes)
*/
this.ProfileList = make([]int64, 0)
this.Len = this.numMmap.ReadInt64(0) //int64(binary.LittleEndian.Uint64(MmapBytes[:8]))
this.Type = this.numMmap.ReadInt64(8) // int64(binary.LittleEndian.Uint64(MmapBytes[8:16]))
//name_lens := int64(binary.LittleEndian.Uint64(MmapBytes[16:24]))
//this.Name = string(MmapBytes[24 : 24+name_lens])
var start int64 = 16 //24 + name_lens
var i int64 = 0
for i = 1; i < this.Len; i++ {
value := this.numMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
this.ProfileList = append(this.ProfileList, value)
}
this.numMmap.SetFileEnd(start)
this.IsSearch = true
return nil
}
func (this *NumberProfile) ReadFromFile() error {
var err error
file_name := fmt.Sprintf("./index/%v.pfl", this.Name)
this.numMmap, err = u.NewMmap(file_name, u.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
this.ProfileList = make([]int64, 0)
this.Len = this.numMmap.ReadInt64(0)
this.Type = this.numMmap.ReadInt64(8)
var start int64 = 16
var i int64 = 0
for i = 0; i < this.Len; i++ {
value := this.numMmap.ReadInt64(start)
start += 8
this.ProfileList = append(this.ProfileList, value)
}
this.numMmap.SetFileEnd(start)
this.IsSearch = true
return nil
}
func (this *Dispatcher) CreateIndex(method string, parms map[string]string, body []byte) error {
var idxstruct utils.IndexStrct
if err := json.Unmarshal(body, &idxstruct); err != nil {
this.Logger.Error("[ERROR] json error %v", err)
return err
}
if _, ok := this.IndexInfo[idxstruct.IndexName]; ok {
this.Logger.Error("[ERROR] index [%v] already has ", idxstruct.IndexName)
return fmt.Errorf("[ERROR] index [%v] already has ", idxstruct.IndexName)
}
sis := make([]ShardInfo, 0)
seeds := make([]bool, len(this.NodeInfos))
if len(this.NodeInfos) < int(idxstruct.ShardNum) {
this.Logger.Error("[ERROR] ShardNum[%v] is to large", idxstruct.ShardNum)
return fmt.Errorf("[ERROR] ShardNum[%v] is to large", idxstruct.ShardNum)
}
var nodeinfo utils.NodeIndex
nodeinfo.IndexName = idxstruct.IndexName
nodeinfo.ShardNum = idxstruct.ShardNum
nodeinfo.IndexMapping = idxstruct.IndexMapping
nodeinfo.ShardNodes = make(map[uint64][]string)
//nodeinfo.Shard = make([]uint64, idxstruct.ShardNum)
for i := uint64(0); i < idxstruct.ShardNum; i++ {
si := ShardInfo{ShardPathName: fmt.Sprintf("%v/%v_%v", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i),
MaxOffset: 0, NodeInfos: make([]string, 0)}
si.OffsetMmap, _ = utils.NewMmap(fmt.Sprintf("%v/%v_%v.offset", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i), utils.MODE_APPEND)
si.DetailMmap, _ = utils.NewMmap(fmt.Sprintf("%v/%v_%v.dtl", utils.IDX_DETAIL_PATH, idxstruct.IndexName, i), utils.MODE_APPEND)
for {
seed := rand.Intn(len(seeds))
if seeds[seed] == false {
seeds[seed] = true
net := make([]string, 0)
si.NodeInfos = append(si.NodeInfos, fmt.Sprintf("%v:%v", this.NodeInfos[seed].Addr, this.NodeInfos[seed].MPort))
net = append(net, fmt.Sprintf("%v:%v", this.NodeInfos[seed].Addr, this.NodeInfos[seed].MPort))
nodeinfo.ShardNodes[uint64(seed)] = net
this.Logger.Info("[INFO] NodeInfos::: %v", si.NodeInfos)
break
}
}
sis = append(sis, si)
}
info := idxInfo{Name: idxstruct.IndexName, ShardNum: idxstruct.ShardNum, MaxCnt: 0,
IndexMapping: idxstruct.IndexMapping, ShardInfos: sis, ShardField: idxstruct.ShardField}
this.IndexInfo[idxstruct.IndexName] = info
for _, iinfo := range this.NodeInfos {
iinfo.IdxChan <- nodeinfo
}
return this.storeStruct()
}
func (this *Detail) ReadDetailFromFile() error {
var err error
file_name := "./index/detail.dic"
this.dicMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
/*
f, err := os.Open(file_name)
defer f.Close()
if err != nil {
return err
}
fi, err := f.Stat()
if err != nil {
fmt.Printf("ERR:%v", err)
}
MmapBytes, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
fmt.Printf("MAPPING ERROR %v \n", err)
return nil
}
defer syscall.Munmap(MmapBytes)
this.MaxDocId = int64(binary.LittleEndian.Uint64(MmapBytes[:8]))
this.Offset = int64(binary.LittleEndian.Uint64(MmapBytes[8:16]))
var start int64 = 16
var i int64 = 0
for i = 0; i <= this.MaxDocId; i++ {
start_pos := int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
byte_len := int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
this.DetailList = append(this.DetailList, DetailInfo{nil,start_pos,byte_len,false})
}
*/
this.MaxDocId = this.dicMmap.ReadInt64(0)
this.Offset = this.dicMmap.ReadInt64(8)
var start int64 = 16
var i int64 = 0
for i = 0; i <= this.MaxDocId; i++ {
start_pos := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
byte_len := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
in_inc := this.dicMmap.ReadInt64(start) //int64(binary.LittleEndian.Uint64(MmapBytes[start : start+8]))
start += 8
this.DetailList = append(this.DetailList, DetailInfo{nil, start_pos, byte_len, in_inc, false})
}
this.dicMmap.SetFileEnd(start)
//mmap详细文件
file_name = "./index/detail.dat"
this.detailMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
//mmap增量文件
file_name = "./index/detail.up"
this.upMmap, err = utils.NewMmap(file_name, utils.MODE_APPEND)
if err != nil {
fmt.Printf("mmap error : %v \n", err)
return err
}
up_file_end := this.upMmap.ReadInt64(0)
if up_file_end == 0 {
this.upMmap.SetFileEnd(8)
} else {
this.upMmap.SetFileEnd(up_file_end)
}
return nil
}
