예제 #1
0
파일: bigfile.go 프로젝트: caohao2008/goose
// 读取数据,外部需要准备好够存放的desBuf
func (this *BigFile) Read(i BigFileIndex, desBuf []byte) error {
	if i.FileNo >= this.bigfileStat.FileCnt {
		return log.Error("BigFile.Read FileNo[%d] Error", i.FileNo)
	}
	if i.Length > uint32(len(desBuf)) {
		return log.Error("BigFile.Read BigFileIndex.Length[%d] > len(desBuf)[%d]",
			i.Length, uint32(len(desBuf)))
	}

	var f *os.File
	if i.FileNo == this.bigfileStat.FileCnt-1 {
		f = this.readwriteFile
	} else {
		f = this.readOnlyFile[i.FileNo]
	}
	n, err := f.ReadAt(desBuf[:i.Length], int64(i.Offset))
	if err == io.EOF {
		if uint32(n) == i.Length {
			// 刚刚好读完
			return nil
		}
	}
	if uint32(n) != i.Length {
		return log.Error("Read Length Error offset[%d] destBuf len[%d],ReadAt len[%d]",
			i.Offset, i.Length, n)
	}
	if err != nil {
		return log.Error("ReadAt file", err.Error())
	}
	return nil
}
예제 #2
0
파일: bigfile.go 프로젝트: caohao2008/goose
// 追加数据,返回追加数据的存储信息.不可并发进行写操作.
func (this *BigFile) Append(buf []byte) (*BigFileIndex, error) {

	f, err := this.getRwFile()
	if err != nil {
		return nil, err
	}

	i := BigFileIndex{}
	i.FileNo = this.bigfileStat.FileCnt - 1
	i.Length = uint32(len(buf))
	off, err := this.readwriteFile.Seek(0, 1)
	i.Offset = uint32(off)
	if i.Offset != this.bigfileStat.LastFileOffset {
		return nil, log.Error("BigFile.Append getOffset[%d] LastFileOffset[%d]",
			i.Offset, this.bigfileStat.LastFileOffset)
	}

	n, err := f.Write(buf)
	if err != nil {
		return nil, log.Error("BigFile.Append write fail : %s", err.Error())
	}
	if uint32(n) != i.Length {
		// 写成功,但是写入长度跟期望对不上
		// 回滚文件指针
		this.readwriteFile.Seek(int64(i.Offset), 0)
		return nil, log.Error("BigFile.Append write succ bug length error : %s",
			err.Error())
	}
	// 更新状态文件
	this.bigfileStat.LastFileOffset = i.Offset + i.Length
	this.saveStatFile()

	return &i, nil
}
예제 #3
0
파일: Goose.go 프로젝트: caohao2008/goose
// 检索模式运行
func (this *Goose) searchModeRun() {

	log.Debug("run in search mode")

	if this.searchSty == nil {
		log.Error("Please set search strategy,see Goose.SetSearchStrategy()")
		return
	}

	if this.indexSty == nil {
		log.Warn("can't build index real time witout Index Strategy")
	}

	gooseSearch := NewGooseSearch()
	err := gooseSearch.Init(this.confPath, this.indexSty, this.searchSty)
	if err != nil {
		log.Error(err)
		return
	}

	log.Debug("goose search init succ")

	err = gooseSearch.Run()
	if err != nil {
		log.Error(err)
		return
	}
}
예제 #4
0
// 读取Data数据,可以并发.
func (this *DataManager) ReadData(inId InIdType, buf *Data) error {
	if inId < 1 || inId > this.dataStatus.MaxInId {
		return log.Error("inId [%d] illegal MaxInId[%d]", inId, this.dataStatus.MaxInId)
	}

	// 读一级索引
	bigFileI, err := this.readData0(inId)
	if err != nil {
		return err
	}
	if bigFileI.Length == 0 {
		return log.Error("Read data0 inId[%d],fileNo[%d],length[%d],offset[%d]",
			inId, bigFileI.FileNo, bigFileI.Length, bigFileI.Offset)
	}

	// 读二级索引
	if bigFileI.Length > uint32(buf.Len()) {
		*buf = NewData(int(bigFileI.Length))
	}
	err = this.data1.Read(bigFileI, *buf)
	if err != nil {
		return err
	}

	return nil
}
예제 #5
0
파일: bigfile.go 프로젝트: caohao2008/goose
// 打开已存在的大文件,如果不存在,直接返回错误
func (this *BigFile) Open(path string, name string) error {

	// 是打开已有数据文件状态
	this.fileModel = bigFileModelOpen

	this.filePath = path
	this.fileName = name

	this.statFileFullPath = filepath.Join(this.filePath,
		fmt.Sprintf("%s%s", this.fileName, statFileSuffix))

	// 解析获取文件信息
	err := this.parseStatFile()
	if err != nil {
		return log.Warn(err)
	}
	// 检验状态文件
	if this.bigfileStat.SuggestFileSize == 0 {
		return log.Error("BigFile.Open stat file error")
	}

	// 除了最后一个文件,其它以只读方式打开
	readOnlyFileCnt := uint8(0)
	if this.bigfileStat.FileCnt > 0 {
		readOnlyFileCnt = this.bigfileStat.FileCnt - 1
	}
	this.readOnlyFile = make([]*os.File, readOnlyFileCnt)
	for i := 0; uint8(i) < readOnlyFileCnt; i++ {
		f, err := this.openRoFile(uint8(i))
		if err != nil {
			return err
		}
		this.readOnlyFile[i] = f
		// 校验这些只读文件的大小,他们肯定是大于等于配置才对
		// TODO
	}

	// 最后一个文件已读写方式打开
	if this.bigfileStat.FileCnt > 0 {
		err = this.openRwFile(this.bigfileStat.FileCnt - 1)
		if err != nil {
			return err
		}
		// 设置文件指针
		this.readwriteFile.Seek(int64(this.bigfileStat.LastFileOffset), 0)

		// 最后一个文件的文件指针应该就是文件大小
		sz, _ := FileSize(this.readwriteFile)
		if sz != int64(this.bigfileStat.LastFileOffset) {
			return log.Error("BigFile.Open", "FileStatInfo Error LastFileOffset:[%d] != FileSize:[%d]",
				this.bigfileStat.LastFileOffset, sz)
		}
	} else {
		this.readwriteFile = nil
	}

	return nil
}
예제 #6
0
// 根据唯一外部ID,分配内部ID,可并发内部有锁控制按顺序分配
func (this *DBSearcher) AllocID(outID OutIdType) (InIdType, error) {
	if this.varIndex == nil {
		return 0, log.Error("No Var Index")
	}
	if this.idMgr == nil {
		return 0, log.Error("no id manager")
	}
	return this.idMgr.AllocID(outID)
}
예제 #7
0
// 写入Value数据,可并发写入.
func (this *DBSearcher) WriteValue(InID InIdType, v Value) error {
	if this.varIndex == nil {
		return log.Error("No Var Index")
	}
	if this.valueMgr == nil {
		return log.Error("no value manager")
	}

	return this.valueMgr.WriteValue(InID, v)
}
예제 #8
0
// 写入Data数据,可并发调用.
func (this *DBSearcher) WriteData(InID InIdType, d Data) error {
	if this.varIndex == nil {
		return log.Error("No Var Index")
	}
	if this.dataMgr == nil {
		return log.Error("no data manager")
	}

	// dataMgr内部锁控制,并发写顺序写入
	return this.dataMgr.Append(InID, d)
}
예제 #9
0
// 创建全新的磁盘索引,初始化后只允许进行索引写入.
// maxFileSz 索引大文件单个文件的最大大小.
// MaxTermCnt 是预期要写入的term的总数量.
func (this *DiskIndex) Init(path string, name string, maxFileSz uint32, MaxTermCnt int64) error {
	this.lock.Lock()
	defer this.lock.Unlock()

	if this.indexStatus != DiskIndexInit {
		return log.Error("index status error")
	}

	if len(path) == 0 || len(name) == 0 {
		return log.Error("path[%s] name[%s] error")
	}

	this.filePath = path
	this.fileName = name

	this.StatusFilePath = filepath.Join(this.filePath,
		fmt.Sprintf("%s.index.stat", this.fileName))
	// 磁盘状态文件需要设置的两个步骤:(1)指示要写入的结构;(2)设置写入路径
	this.SelfStatus = &this.diskStatus

	this.diskStatus.MaxTermCount = MaxTermCnt

	// 初始化三级索引
	this.index3 = &BigFile{}
	ind3name := fmt.Sprintf("%s.index3", this.fileName)
	err := this.index3.Init(this.filePath, ind3name, maxFileSz)
	if err != nil {
		return log.Error(err)
	}

	// 打开二级索引
	ind2name := filepath.Join(this.filePath, fmt.Sprintf("%s.index2", this.fileName))
	// 打开新文件,创建|截断|只写
	this.index2, err = os.OpenFile(ind2name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return log.Error(err)
	}

	// 计算预期一级索引大小
	index1Sz := this.diskStatus.MaxTermCount * int64(binary.Size(TermSign(0)))

	// 打开一级索引
	this.index1 = new(MmapFile)
	ind1name := fmt.Sprintf("%s.index1", this.fileName)
	err = this.index1.OpenFile(this.filePath, ind1name, uint32(index1Sz))
	if err != nil {
		return log.Error("mmap open[%s] size[%d] fail : %s", ind1name, index1Sz, err)
	}

	this.indexStatus = DiskIndexWriteOnly

	return this.SaveJsonFile()
}
예제 #10
0
func (this *GooseSearch) runIndexServer(listenPort int, requestBufSize int) error {

	if 0 == listenPort || 0 == requestBufSize {
		return log.Error("arg error istenPort[%d] requestBufSize[%d]",
			listenPort, requestBufSize)
	}

	if this.varIndexer == nil {
		return nil
	}

	listener, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", listenPort))
	if err != nil {
		log.Error("runIndexServer listen fail : %s", err.Error())
		return err
	}

	// 简单一个协程完成接受请求和完成处理.索引更新不要求高并发性.
	go func() {
		reqbuf := make([]byte, requestBufSize)
		for {
			var reqlen int

			conn, err := listener.Accept()
			if err != nil {
				log.Warn("IndexServer accept fail : %s", err.Error())
				goto LabelError
			}

			// receive data
			reqlen, err = conn.Read(reqbuf)
			if err != nil {
				log.Warn("IndexSearcher read fail : %s", err.Error())
				goto LabelError
			}

			// index
			err = this.varIndexer.BuildIndex(NewBufferIterOnce(reqbuf[:reqlen]))
			if err != nil {
				log.Warn("IndexSearcher BuildIndex fail : %s", err.Error())
				goto LabelError
			}

		LabelError:
			conn.Close()
		}
	}()

	return nil
}
예제 #11
0
// 读取offset开始的destSz个字节作为数字返回
func (this *MmapFile) ReadNum(offset uint32, destSz uint32) (uint64, error) {
	if destSz == 0 {
		return 0, log.Error("Mmapfile.ReadNum not a basic num")
	}
	if int64(offset)+int64(destSz) > int64(len(this.fileMmap)) {
		return 0, log.Error("Mmapfile.ReadNum over length limit")
	}

	buf := this.fileMmap[offset : offset+uint32(destSz)]

	order := binary.BigEndian
	switch destSz {
	case 1:
		return uint64(buf[0]), nil
	case 2:
		return uint64(order.Uint16(buf)), nil
	case 4:
		return uint64(order.Uint32(buf)), nil
	case 8:
		return uint64(order.Uint64(buf)), nil
	default:
		return 0, log.Error("MmapFile.ReadNum Wrong Type")
	}
	return 0, nil
	/*
	   switch v := n.(type) {
	   case *int8:
	       *v = int8(buf[0])
	   case *uint8:
	       *v = buf[0]
	   case *int16:
	       *v = int16(order.Uint16(buf))
	   case *uint16:
	       *v = uint16(order.Uint16(buf))
	   case *int32:
	       *v = int32(order.Uint32(buf))
	   case *uint32:
	       *v = uint32(order.Uint32(buf))
	   case *int64:
	       *v = int64(order.Uint64(buf))
	   case *uint64:
	       *v = uint64(order.Uint64(buf))
	   default:
	       return NewGooseError("MmapFile.ReadNum","Wrong Type","")
	   }

	   return nil
	*/
}
예제 #12
0
// 写入索引,不可并发写入
func (this *DBBuilder) WriteIndex(InID InIdType, termlist []TermInDoc) error {
	if this.transformMgr == nil {
		return log.Error("no transform manager")
	}

	return this.transformMgr.WriteIndex(InID, termlist)
}
예제 #13
0
// 写入Value数据,可并发写入
func (this *DBBuilder) WriteValue(InID InIdType, v Value) error {
	if this.valueMgr == nil {
		return log.Error("no value manager")
	}

	return this.valueMgr.WriteValue(InID, v)
}
예제 #14
0
파일: Indexer.go 프로젝트: caohao2008/goose
func (this *StaticIndexer) parseDoc() {

	// context
	context := NewStyContext()

	// 一直从chan中获取doc,直到这个chan被close
	for doc := range this.parseDocChan {
		var err error
		// parse
		parseRes := &docParsed{}
		parseRes.outId, parseRes.termList, parseRes.value, parseRes.data,
			err = this.strategy.ParseDoc(doc, context)
		if err != nil {
			log.Error(err)
			parseRes = nil
		}
		// 打印策略日志
		context.Log.PrintAllInfo()

		// toWriteDbQueue是待写入db的队列.
		// 阻塞等待队列有空余位置然后写入队列.
		this.writeDbQueue <- parseRes
	}
	log.Info("Finish parseDoc , goroutine exit.")
}
예제 #15
0
// read bytes (reference)
func (this *MmapFile) ReadBytes(offset uint32, length uint32) ([]byte, error) {
	if uint64(offset+length) > uint64(len(this.fileMmap)) {
		return nil, log.Error("Mmapfile.ReadBytes over length limit")
	}

	return this.fileMmap[offset : offset+length], nil
}
예제 #16
0
// 分配内部id
func (this *IdManager) AllocID(outId OutIdType) (InIdType, error) {
	this.lock.Lock()
	defer this.lock.Unlock()

	if outId == 0 {
		return 0, log.Warn("illegal outId [%d]", 0)
	}

	if this.idStatus.CurId >= this.idStatus.MaxInId {
		return 0, log.Error("InId [%d] out of limit MaxInId[%d]", this.idStatus.CurId, this.idStatus.MaxInId)
	}

	inID := this.idStatus.CurId

	// 分配信息,写入mmap
	offset := inID * idSize
	err := this.mfile.WriteNum(uint32(offset), uint32(outId))
	if err != nil {
		return 0, err
	}

	// 确认分配成功才真正占用这个id
	this.idStatus.CurId++

	return inID, nil
}
예제 #17
0
// 全新初始化数据文件
func (this *DataManager) Init(path string, maxId InIdType, maxFileSz uint32) error {

	this.dataStatus.MaxInId = maxId
	this.maxDataFileSize = maxFileSz
	this.filePath = path

	// 磁盘状态文件需要设置的两个步骤:(1)指示要写入的结构;(2)设置写入路径
	this.SelfStatus = &this.dataStatus
	this.StatusFilePath = filepath.Join(this.filePath, "data.stat")

	// 一级索引mmap打开
	// id有效范围[1,MaxInId],0不使用导致后面要多分配一个空间
	data0Size := uint32(1+this.dataStatus.MaxInId) * uint32(binary.Size(BigFileIndex{}))
	data0Name := fmt.Sprintf("data.d0")
	err := this.data0.OpenFile(this.filePath, data0Name, data0Size)
	if err != nil {
		return log.Error("mmap open[%s] size[%d] fail : %s", data0Name, data0Size, err)
	}
	// 二级索引BigFile打开
	this.data1 = new(BigFile)
	data1Name := fmt.Sprintf("data.d1")
	err = this.data1.Init(this.filePath, data1Name, this.maxDataFileSize)
	if err != nil {
		return err
	}

	return this.SaveJsonFile()
}
예제 #18
0
func (this *ValueManager) Init(path string, maxId InIdType, valueSz uint32) error {
	this.lock.Lock()
	defer this.lock.Unlock()

	this.filePath = path

	this.valueStatus.MaxInId = maxId
	this.valueStatus.ValueSize = valueSz

	// 磁盘状态文件需要设置的两个步骤:(1)指示要写入的结构;(2)设置写入路径
	this.SelfStatus = &this.valueStatus
	this.StatusFilePath = filepath.Join(this.filePath, "value.stat")

	this.fileValueMaxCnt = uint32(maxValueFileSize / this.valueStatus.ValueSize)
	this.fileCnt = uint32(uint32(maxId)/this.fileValueMaxCnt) + 1
	this.mfile = make([]MmapFile, this.fileCnt)

	// 分配磁盘空间
	for i := 0; uint32(i) < this.fileCnt; i++ {
		tname := fmt.Sprintf("value.n%d", i)
		sz := uint32(this.fileValueMaxCnt * this.valueStatus.ValueSize)
		err := this.mfile[i].OpenFile(path, tname, sz)
		if err != nil {
			return log.Error("open mfile[%d],szie[%d] fail[%s]", i, sz, err.Error())
		}
	}

	return this.SaveJsonFile()
}
예제 #19
0
// 打开已存在的数据文件
func (this *DataManager) Open(path string) error {

	this.filePath = path

	// 磁盘状态文件需要设置的两个步骤:(1)指示要写入的结构;(2)设置写入路径
	this.SelfStatus = &this.dataStatus
	this.StatusFilePath = filepath.Join(this.filePath, "data.stat")
	err := this.ParseJsonFile()
	if err != nil {
		return err
	}

	// 一级索引mmap打开
	// id有效范围[1,MaxInId],0不使用导致后面要多分配一个空间
	// 打开也要多打开一个单位的空间
	data0Size := uint32(1+this.dataStatus.MaxInId) * uint32(binary.Size(BigFileIndex{}))
	data0Name := fmt.Sprintf("data.d0")
	err = this.data0.OpenFile(this.filePath, data0Name, data0Size)
	if err != nil {
		return log.Error("mmap open[%s] size[%d] fail : %s", data0Name, data0Size, err)
	}
	// 二级索引BigFile打开
	this.data1 = new(BigFile)
	data1Name := fmt.Sprintf("data.d1")
	err = this.data1.Open(this.filePath, data1Name)
	if err != nil {
		return err
	}

	return nil
}
예제 #20
0
// 写入Data数据,可并发调用,内部锁控制
func (this *DBBuilder) WriteData(InID InIdType, d Data) error {
	if this.dataMgr == nil {
		return log.Error("no data manager")
	}

	// dataMgr内部锁控制,并发写顺序写入
	return this.dataMgr.Append(InID, d)
}
예제 #21
0
// 根据配置文件进行初始化.
// 需要外部指定索引策略,策略可以重新设计.
// 需要外部知道被索引文件(这个易变信息不适合放配置)
func (this *GooseBuild) Init(confPath string, indexSty IndexStrategy, toIndexFile string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = log.Error(r)
		}
	}()

	// load conf
	this.conf, err = config.NewConf(confPath)
	if err != nil {
		return
	}

	// set max procs
	maxProcs := int(this.conf.Int64("GooseBuild.MaxProcs"))
	if maxProcs <= 0 {
		maxProcs = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(maxProcs)

	// init dbbuilder
	dbPath := this.conf.String("GooseBuild.DataBase.DbPath")
	transformMaxTermCnt := this.conf.Int64("GooseBuild.DataBase.TransformMaxTermCnt")
	maxId := this.conf.Int64("GooseBuild.DataBase.MaxId")
	maxIndexFileSize := this.conf.Int64("GooseBuild.DataBase.MaxIndexFileSize")
	maxDataFileSize := this.conf.Int64("GooseBuild.DataBase.MaxDataFileSize")
	valueSize := this.conf.Int64("GooseBuild.DataBase.ValueSize")

	this.staticDB = NewDBBuilder()
	err = this.staticDB.Init(dbPath, int(transformMaxTermCnt), InIdType(maxId),
		uint32(valueSize), uint32(maxIndexFileSize), uint32(maxDataFileSize))
	if err != nil {
		return
	}

	// index strategy global init
	err = indexSty.Init(this.conf)
	if err != nil {
		return
	}

	// static indexer
	this.staticIndexer, err = NewStaticIndexer(this.staticDB, indexSty)
	if err != nil {
		return
	}

	// open data file
	this.fileHd, err = os.OpenFile(toIndexFile, os.O_RDONLY, 0644)
	if err != nil {
		return
	}

	// file iter
	this.fileIter = NewFileIter(this.fileHd)

	return nil
}
예제 #22
0
파일: Indexer.go 프로젝트: caohao2008/goose
func (this *StaticIndexer) writeDoc() {

	for parseRes := range this.writeDbQueue {

		if parseRes == nil {
			log.Error("get nil pointer from queue")
			this.finishedWg.Done()
			continue
		}

		// id
		inId, err := this.db.AllocID(parseRes.outId)
		if err != nil {
			log.Error(err)
			this.finishedWg.Done()
			continue
		}

		// index
		err = this.db.WriteIndex(inId, parseRes.termList)
		if err != nil {
			log.Error(err)
			this.finishedWg.Done()
			continue
		}

		// value
		err = this.db.WriteValue(inId, parseRes.value)
		if err != nil {
			log.Error(err)
			this.finishedWg.Done()
			continue
		}

		// data
		err = this.db.WriteData(inId, parseRes.data)
		if err != nil {
			log.Error(err)
			this.finishedWg.Done()
			continue
		}
		this.finishedWg.Done()
	}
	log.Info("Finish writeDoc,goroutine exit.")
}
예제 #23
0
// read bytes (copy slice)
func (this *MmapFile) ReadBytesCopy(offset int32, length int32) ([]byte, error) {
	if int64(offset+length) > int64(len(this.fileMmap)) {
		return nil, log.Error("Mmapfile.ReadBytes over length limit")
	}

	newbuf := make([]byte, length)
	copy(newbuf, this.fileMmap[offset:offset+length])
	return newbuf, nil
}
예제 #24
0
파일: bigfile.go 프로젝트: caohao2008/goose
func (this *BigFile) openRoFile(fileno uint8) (*os.File, error) {
	var err error
	tname := fmt.Sprintf("%s%s%d", this.fileName, dataFileSuffix, fileno)
	f, err := os.OpenFile(filepath.Join(this.filePath, tname), os.O_RDONLY, 0644)
	if err != nil {
		return nil, log.Error("open readonly file fail : %s", err.Error())
	}
	return f, nil
}
예제 #25
0
// 读取value的引用,value只能进行读操作,任何写操作都是非法的
func (this *ValueManager) ReadValue(inId InIdType) (Value, error) {
	if inId > this.valueStatus.MaxInId {
		return nil, log.Error("inId [%d] illegal MaxInId[%d]", inId, this.valueStatus.MaxInId)
	}

	fileNo := uint32(int64(inId) / int64(this.fileValueMaxCnt))
	offset := uint32(int64(inId)%int64(this.fileValueMaxCnt)) * this.valueStatus.ValueSize

	if fileNo >= this.fileCnt {
		return nil, log.Error("inId out of limit")
	}

	v, err := this.mfile[fileNo].ReadBytes(offset, this.valueStatus.ValueSize)
	if err != nil {
		return nil, err
	}
	return v[:], nil
}
예제 #26
0
// 读取索引,每次查询在内部分配一块内存返回
func (this *DiskIndex) ReadIndex(t TermSign) (*InvList, error) {
	// 读取不加锁

	// 打开的磁盘只读索引下才允许读取
	if this.indexStatus != DiskIndexReadOnly {
		return nil, log.Error("DiskIndex.Read status error")
	}
	return this.readIndex3(t)
}
예제 #27
0
// 写入Value.可并发写
func (this *ValueManager) WriteValue(inId InIdType, v Value) error {
	if inId > this.valueStatus.MaxInId {
		return log.Error("inId [%d] illegal MaxInId[%d]", inId, this.valueStatus.MaxInId)
	}

	fileNo := uint32(int64(inId) / int64(this.fileValueMaxCnt))
	offset := uint32(int64(inId)%int64(this.fileValueMaxCnt)) * this.valueStatus.ValueSize

	if fileNo >= this.fileCnt {
		return log.Error("inId out of limit")
	}

	// 最多写入this.valueStatus.ValueSize个字节
	err := this.mfile[fileNo].WriteBytes(offset, v[:], this.valueStatus.ValueSize)
	if err != nil {
		return err
	}
	return nil
}
예제 #28
0
// 写入索引,不可并发写入.
func (this *DBSearcher) WriteIndex(InID InIdType, termlist []TermInDoc) error {
	if this.varIndex == nil {
		return log.Error("No Var Index")
	}
	for _, term := range termlist {
		l := NewInvList(1)
		l.Append(Index{InID: InID, Weight: term.Weight})
		this.varIndex.WriteIndex(term.Sign, &l)
	}
	return nil
}
예제 #29
0
파일: bigfile.go 프로젝트: caohao2008/goose
// 根据文件数量打开最后一个可读写文件
func (this *BigFile) openRwFile(fileno uint8) error {
	var err error
	tname := fmt.Sprintf("%s%s%d", this.fileName, dataFileSuffix, fileno)
	this.readwriteFileFullPath = filepath.Join(this.filePath, tname)
	if this.fileModel == bigFileModelInit {
		// 全新初始化的,文件打开后直接做截断处理
		this.readwriteFile, err = os.OpenFile(this.readwriteFileFullPath,
			os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	} else if this.fileModel == bigFileModelOpen {
		// 已有文件,追加写
		this.readwriteFile, err = os.OpenFile(this.readwriteFileFullPath,
			os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	} else {
		return log.Error("File Status Error : [%d]", this.fileModel)
	}
	if err != nil {
		return log.Error("open readwrite file fail : %s", err.Error())
	}
	return nil
}
예제 #30
0
파일: bigfile.go 프로젝트: caohao2008/goose
func (this *BigFileIndex) Encode(buf []byte) error {
	order := binary.BigEndian
	if len(buf) < 9 {
		return log.Error("BigFileIndex.Decode buf length [%d] error", len(buf))
	}

	buf[0] = this.FileNo
	order.PutUint32(buf[1:5], this.Offset)
	order.PutUint32(buf[5:9], this.Length)
	return nil
}