Exemplo n.º 1
0
/* Reads the most recent FieldInfos of the given segment info. */
func ReadFieldInfos(info *SegmentCommitInfo) (fis FieldInfos, err error) {
	var dir store.Directory
	var closeDir bool
	if info.FieldInfosGen() == -1 && info.Info.IsCompoundFile() {
		// no fieldInfos gen and segment uses a compound file
		if dir, err = store.NewCompoundFileDirectory(info.Info.Dir,
			util.SegmentFileName(info.Info.Name, "", store.COMPOUND_FILE_EXTENSION),
			store.IO_CONTEXT_READONCE, false); err != nil {
			return
		}
		closeDir = true
	} else {
		// gen'd FIS are read outside CFS, or the segment doesn't use a compound file
		dir = info.Info.Dir
		closeDir = false
	}

	defer func() {
		if closeDir {
			err = mergeError(err, dir.Close())
		}
	}()

	var segmentSuffix string
	if n := info.FieldInfosGen(); n != -1 {
		segmentSuffix = strconv.FormatInt(n, 36)
	}
	codec := info.Info.Codec().(Codec)
	fisFormat := codec.FieldInfosFormat()
	return fisFormat.FieldInfosReader()(dir, info.Info.Name, segmentSuffix, store.IO_CONTEXT_READONCE)
}
Exemplo n.º 2
0
func TestReadFieldInfos(t *testing.T) {
	path := "../../search/testdata/osx/belfrysample"
	d, err := store.OpenFSDirectory(path)
	if err != nil {
		t.Error(err)
	}
	ctx := store.NewIOContextBool(false)
	cd, err := store.NewCompoundFileDirectory(d, "_0.cfs", ctx, false)
	if err != nil {
		t.Error(err)
	}
	fis, err := Lucene42FieldInfosReader(cd, "_0", "", store.IO_CONTEXT_READONCE)
	if err != nil {
		t.Error(err)
	}
	if !fis.HasNorms || fis.HasDocValues {
		t.Errorf("hasNorms must be true and hasDocValues must be false, but found %v", fis)
	}
}
Exemplo n.º 3
0
func newSegmentCoreReaders(owner *SegmentReader, dir store.Directory, si *SegmentCommitInfo,
	context store.IOContext, termsIndexDivisor int) (self *SegmentCoreReaders, err error) {

	assert2(termsIndexDivisor != 0,
		"indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)")
	// fmt.Println("Initializing SegmentCoreReaders from directory:", dir)

	self = &SegmentCoreReaders{
		refCount: 1,
		normsLocal: func() map[string]interface{} {
			return make(map[string]interface{})
		},
	}
	self.fieldsReaderLocal = func() StoredFieldsReader {
		return self.fieldsReaderOrig.Clone()
	}

	// fmt.Println("Initializing listeners...")
	self.addListener = make(chan CoreClosedListener)
	self.removeListener = make(chan CoreClosedListener)
	self.notifyListener = make(chan bool)
	// TODO re-enable later
	go func() { // ensure listners are synchronized
		coreClosedListeners := make([]CoreClosedListener, 0)
		isRunning := true
		var listener CoreClosedListener
		for isRunning {
			// fmt.Println("Listening for events...")
			select {
			case listener = <-self.addListener:
				coreClosedListeners = append(coreClosedListeners, listener)
			case listener = <-self.removeListener:
				n := len(coreClosedListeners)
				for i, v := range coreClosedListeners {
					if v == listener {
						newListeners := make([]CoreClosedListener, 0, n-1)
						newListeners = append(newListeners, coreClosedListeners[0:i]...)
						newListeners = append(newListeners, coreClosedListeners[i+1:]...)
						coreClosedListeners = newListeners
						break
					}
				}
			case <-self.notifyListener:
				fmt.Println("Shutting down SegmentCoreReaders...")
				isRunning = false
				for _, v := range coreClosedListeners {
					v.onClose(self)
				}
			}
		}
		fmt.Println("Listeners are done.")
	}()

	var success = false
	ans := self
	defer func() {
		if !success {
			fmt.Println("Failed to initialize SegmentCoreReaders.")
			ans.decRef()
		}
	}()

	codec := si.Info.Codec().(Codec)
	// fmt.Println("Obtaining CFS Directory...")
	var cfsDir store.Directory // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
	if si.Info.IsCompoundFile() {
		// fmt.Println("Detected CompoundFile.")
		name := util.SegmentFileName(si.Info.Name, "", store.COMPOUND_FILE_EXTENSION)
		if self.cfsReader, err = store.NewCompoundFileDirectory(dir, name, context, false); err != nil {
			return nil, err
		}
		// fmt.Println("CompoundFileDirectory: ", self.cfsReader)
		cfsDir = self.cfsReader
	} else {
		cfsDir = dir
	}
	// fmt.Println("CFS Directory:", cfsDir)

	// fmt.Println("Reading FieldInfos...")
	fieldInfos := owner.fieldInfos

	self.termsIndexDivisor = termsIndexDivisor
	format := codec.PostingsFormat()

	// fmt.Println("Obtaining SegmentReadState...")
	segmentReadState := NewSegmentReadState(cfsDir, si.Info, fieldInfos, context, termsIndexDivisor)
	// Ask codec for its Fields
	// fmt.Println("Obtaining FieldsProducer...")
	if self.fields, err = format.FieldsProducer(segmentReadState); err != nil {
		return nil, err
	}
	assert(self.fields != nil)
	// ask codec for its Norms:
	// TODO: since we don't write any norms file if there are no norms,
	// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!

	if fieldInfos.HasNorms {
		// fmt.Println("Obtaining NormsDocValuesProducer...")
		if self.normsProducer, err = codec.NormsFormat().NormsProducer(segmentReadState); err != nil {
			return nil, err
		}
		assert(self.normsProducer != nil)
	}

	// fmt.Println("Obtaining StoredFieldsReader...")
	if self.fieldsReaderOrig, err = si.Info.Codec().(Codec).StoredFieldsFormat().FieldsReader(cfsDir, si.Info, fieldInfos, context); err != nil {
		return nil, err
	}

	if fieldInfos.HasVectors { // open term vector files only as needed
		// fmt.Println("Obtaining TermVectorsReader...")
		if self.termVectorsReaderOrig, err = si.Info.Codec().(Codec).TermVectorsFormat().VectorsReader(cfsDir, si.Info, fieldInfos, context); err != nil {
			return nil, err
		}
	}

	// fmt.Println("Success")
	success = true

	return self, nil
}
Exemplo n.º 4
0
func newSegmentCoreReaders(owner *SegmentReader, dir store.Directory, si *SegmentInfoPerCommit,
	context store.IOContext, termsIndexDivisor int) (self SegmentCoreReaders, err error) {
	if termsIndexDivisor == 0 {
		panic("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)")
	}
	log.Printf("Initializing SegmentCoreReaders from directory: %v", dir)

	self = SegmentCoreReaders{
		refCount: 1,
		normsLocal: func() map[string]interface{} {
			return make(map[string]interface{})
		},
	}
	self.fieldsReaderLocal = func() StoredFieldsReader {
		return self.fieldsReaderOrig.Clone()
	}

	log.Print("Initializing listeners...")
	self.addListener = make(chan CoreClosedListener)
	self.removeListener = make(chan CoreClosedListener)
	self.notifyListener = make(chan *SegmentReader)
	// TODO re-enable later
	go func() { // ensure listners are synchronized
		coreClosedListeners := make([]CoreClosedListener, 0)
		isRunning := true
		var listener CoreClosedListener
		for isRunning {
			log.Print("Listening for events...")
			select {
			case listener = <-self.addListener:
				coreClosedListeners = append(coreClosedListeners, listener)
			case listener = <-self.removeListener:
				n := len(coreClosedListeners)
				for i, v := range coreClosedListeners {
					if v == listener {
						newListeners := make([]CoreClosedListener, 0, n-1)
						newListeners = append(newListeners, coreClosedListeners[0:i]...)
						newListeners = append(newListeners, coreClosedListeners[i+1:]...)
						coreClosedListeners = newListeners
						break
					}
				}
			case owner := <-self.notifyListener:
				log.Print("Shutting down SegmentCoreReaders...")
				isRunning = false
				for _, v := range coreClosedListeners {
					v.onClose(owner)
				}
			}
		}
		log.Print("Listeners are done.")
	}()

	success := false
	defer func() {
		if !success {
			log.Print("Failed to initialize SegmentCoreReaders.")
			self.decRef()
		}
	}()

	codec := si.info.Codec().(Codec)
	log.Print("Obtaining CFS Directory...")
	var cfsDir store.Directory // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
	if si.info.IsCompoundFile() {
		log.Print("Detected CompoundFile.")
		name := util.SegmentFileName(si.info.Name, "", store.COMPOUND_FILE_EXTENSION)
		self.cfsReader, err = store.NewCompoundFileDirectory(dir, name, context, false)
		if err != nil {
			return self, err
		}
		log.Printf("CompoundFileDirectory: %v", self.cfsReader)
		cfsDir = self.cfsReader
	} else {
		cfsDir = dir
	}
	log.Printf("CFS Directory: %v", cfsDir)
	log.Print("Reading FieldInfos...")
	self.fieldInfos, err = codec.FieldInfosFormat().FieldInfosReader()(cfsDir, si.info.Name, store.IO_CONTEXT_READONCE)
	if err != nil {
		return self, err
	}

	self.termsIndexDivisor = termsIndexDivisor
	format := codec.PostingsFormat()

	log.Print("Obtaining SegmentReadState...")
	segmentReadState := newSegmentReadState(cfsDir, si.info, self.fieldInfos, context, termsIndexDivisor)
	// Ask codec for its Fields
	log.Print("Obtaining FieldsProducer...")
	self.fields, err = format.FieldsProducer(segmentReadState)
	if err != nil {
		return self, err
	}
	assert(self.fields != nil)
	// ask codec for its Norms:
	// TODO: since we don't write any norms file if there are no norms,
	// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!

	if self.fieldInfos.HasDocValues {
		log.Print("Obtaining DocValuesProducer...")
		self.dvProducer, err = codec.DocValuesFormat().FieldsProducer(segmentReadState)
		if err != nil {
			return self, err
		}
		assert(self.dvProducer != nil)
	} else {
		// self.dvProducer = nil
	}

	if self.fieldInfos.HasNorms {
		log.Print("Obtaining NormsDocValuesProducer...")
		self.normsProducer, err = codec.NormsFormat().NormsProducer(segmentReadState)
		if err != nil {
			return self, err
		}
		assert(self.normsProducer != nil)
	} else {
		// self.normsProducer = nil
	}

	log.Print("Obtaining StoredFieldsReader...")
	self.fieldsReaderOrig, err = si.info.Codec().(Codec).StoredFieldsFormat().FieldsReader(cfsDir, si.info, self.fieldInfos, context)
	if err != nil {
		return self, err
	}

	if self.fieldInfos.HasVectors { // open term vector files only as needed
		log.Print("Obtaining TermVectorsReader...")
		self.termVectorsReaderOrig, err = si.info.Codec().(Codec).TermVectorsFormat().VectorsReader(cfsDir, si.info, self.fieldInfos, context)
		if err != nil {
			return self, err
		}
	} else {
		// self.termVectorsReaderOrig = nil
	}

	log.Print("Success")
	success = true

	// Must assign this at the end -- if we hit an
	// exception above core, we don't want to attempt to
	// purge the FieldCache (will hit NPE because core is
	// not assigned yet).
	self.owner = owner
	return self, nil
}