/* Reads the most recent FieldInfos of the given segment info. */ func ReadFieldInfos(info *SegmentCommitInfo) (fis FieldInfos, err error) { var dir store.Directory var closeDir bool if info.FieldInfosGen() == -1 && info.Info.IsCompoundFile() { // no fieldInfos gen and segment uses a compound file if dir, err = store.NewCompoundFileDirectory(info.Info.Dir, util.SegmentFileName(info.Info.Name, "", store.COMPOUND_FILE_EXTENSION), store.IO_CONTEXT_READONCE, false); err != nil { return } closeDir = true } else { // gen'd FIS are read outside CFS, or the segment doesn't use a compound file dir = info.Info.Dir closeDir = false } defer func() { if closeDir { err = mergeError(err, dir.Close()) } }() var segmentSuffix string if n := info.FieldInfosGen(); n != -1 { segmentSuffix = strconv.FormatInt(n, 36) } codec := info.Info.Codec().(Codec) fisFormat := codec.FieldInfosFormat() return fisFormat.FieldInfosReader()(dir, info.Info.Name, segmentSuffix, store.IO_CONTEXT_READONCE) }
func TestReadFieldInfos(t *testing.T) { path := "../../search/testdata/osx/belfrysample" d, err := store.OpenFSDirectory(path) if err != nil { t.Error(err) } ctx := store.NewIOContextBool(false) cd, err := store.NewCompoundFileDirectory(d, "_0.cfs", ctx, false) if err != nil { t.Error(err) } fis, err := Lucene42FieldInfosReader(cd, "_0", "", store.IO_CONTEXT_READONCE) if err != nil { t.Error(err) } if !fis.HasNorms || fis.HasDocValues { t.Errorf("hasNorms must be true and hasDocValues must be false, but found %v", fis) } }
func newSegmentCoreReaders(owner *SegmentReader, dir store.Directory, si *SegmentCommitInfo, context store.IOContext, termsIndexDivisor int) (self *SegmentCoreReaders, err error) { assert2(termsIndexDivisor != 0, "indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)") // fmt.Println("Initializing SegmentCoreReaders from directory:", dir) self = &SegmentCoreReaders{ refCount: 1, normsLocal: func() map[string]interface{} { return make(map[string]interface{}) }, } self.fieldsReaderLocal = func() StoredFieldsReader { return self.fieldsReaderOrig.Clone() } // fmt.Println("Initializing listeners...") self.addListener = make(chan CoreClosedListener) self.removeListener = make(chan CoreClosedListener) self.notifyListener = make(chan bool) // TODO re-enable later go func() { // ensure listners are synchronized coreClosedListeners := make([]CoreClosedListener, 0) isRunning := true var listener CoreClosedListener for isRunning { // fmt.Println("Listening for events...") select { case listener = <-self.addListener: coreClosedListeners = append(coreClosedListeners, listener) case listener = <-self.removeListener: n := len(coreClosedListeners) for i, v := range coreClosedListeners { if v == listener { newListeners := make([]CoreClosedListener, 0, n-1) newListeners = append(newListeners, coreClosedListeners[0:i]...) newListeners = append(newListeners, coreClosedListeners[i+1:]...) coreClosedListeners = newListeners break } } case <-self.notifyListener: fmt.Println("Shutting down SegmentCoreReaders...") isRunning = false for _, v := range coreClosedListeners { v.onClose(self) } } } fmt.Println("Listeners are done.") }() var success = false ans := self defer func() { if !success { fmt.Println("Failed to initialize SegmentCoreReaders.") ans.decRef() } }() codec := si.Info.Codec().(Codec) // fmt.Println("Obtaining CFS Directory...") var cfsDir store.Directory // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory. if si.Info.IsCompoundFile() { // fmt.Println("Detected CompoundFile.") name := util.SegmentFileName(si.Info.Name, "", store.COMPOUND_FILE_EXTENSION) if self.cfsReader, err = store.NewCompoundFileDirectory(dir, name, context, false); err != nil { return nil, err } // fmt.Println("CompoundFileDirectory: ", self.cfsReader) cfsDir = self.cfsReader } else { cfsDir = dir } // fmt.Println("CFS Directory:", cfsDir) // fmt.Println("Reading FieldInfos...") fieldInfos := owner.fieldInfos self.termsIndexDivisor = termsIndexDivisor format := codec.PostingsFormat() // fmt.Println("Obtaining SegmentReadState...") segmentReadState := NewSegmentReadState(cfsDir, si.Info, fieldInfos, context, termsIndexDivisor) // Ask codec for its Fields // fmt.Println("Obtaining FieldsProducer...") if self.fields, err = format.FieldsProducer(segmentReadState); err != nil { return nil, err } assert(self.fields != nil) // ask codec for its Norms: // TODO: since we don't write any norms file if there are no norms, // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! if fieldInfos.HasNorms { // fmt.Println("Obtaining NormsDocValuesProducer...") if self.normsProducer, err = codec.NormsFormat().NormsProducer(segmentReadState); err != nil { return nil, err } assert(self.normsProducer != nil) } // fmt.Println("Obtaining StoredFieldsReader...") if self.fieldsReaderOrig, err = si.Info.Codec().(Codec).StoredFieldsFormat().FieldsReader(cfsDir, si.Info, fieldInfos, context); err != nil { return nil, err } if fieldInfos.HasVectors { // open term vector files only as needed // fmt.Println("Obtaining TermVectorsReader...") if self.termVectorsReaderOrig, err = si.Info.Codec().(Codec).TermVectorsFormat().VectorsReader(cfsDir, si.Info, fieldInfos, context); err != nil { return nil, err } } // fmt.Println("Success") success = true return self, nil }
func newSegmentCoreReaders(owner *SegmentReader, dir store.Directory, si *SegmentInfoPerCommit, context store.IOContext, termsIndexDivisor int) (self SegmentCoreReaders, err error) { if termsIndexDivisor == 0 { panic("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)") } log.Printf("Initializing SegmentCoreReaders from directory: %v", dir) self = SegmentCoreReaders{ refCount: 1, normsLocal: func() map[string]interface{} { return make(map[string]interface{}) }, } self.fieldsReaderLocal = func() StoredFieldsReader { return self.fieldsReaderOrig.Clone() } log.Print("Initializing listeners...") self.addListener = make(chan CoreClosedListener) self.removeListener = make(chan CoreClosedListener) self.notifyListener = make(chan *SegmentReader) // TODO re-enable later go func() { // ensure listners are synchronized coreClosedListeners := make([]CoreClosedListener, 0) isRunning := true var listener CoreClosedListener for isRunning { log.Print("Listening for events...") select { case listener = <-self.addListener: coreClosedListeners = append(coreClosedListeners, listener) case listener = <-self.removeListener: n := len(coreClosedListeners) for i, v := range coreClosedListeners { if v == listener { newListeners := make([]CoreClosedListener, 0, n-1) newListeners = append(newListeners, coreClosedListeners[0:i]...) newListeners = append(newListeners, coreClosedListeners[i+1:]...) coreClosedListeners = newListeners break } } case owner := <-self.notifyListener: log.Print("Shutting down SegmentCoreReaders...") isRunning = false for _, v := range coreClosedListeners { v.onClose(owner) } } } log.Print("Listeners are done.") }() success := false defer func() { if !success { log.Print("Failed to initialize SegmentCoreReaders.") self.decRef() } }() codec := si.info.Codec().(Codec) log.Print("Obtaining CFS Directory...") var cfsDir store.Directory // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory. if si.info.IsCompoundFile() { log.Print("Detected CompoundFile.") name := util.SegmentFileName(si.info.Name, "", store.COMPOUND_FILE_EXTENSION) self.cfsReader, err = store.NewCompoundFileDirectory(dir, name, context, false) if err != nil { return self, err } log.Printf("CompoundFileDirectory: %v", self.cfsReader) cfsDir = self.cfsReader } else { cfsDir = dir } log.Printf("CFS Directory: %v", cfsDir) log.Print("Reading FieldInfos...") self.fieldInfos, err = codec.FieldInfosFormat().FieldInfosReader()(cfsDir, si.info.Name, store.IO_CONTEXT_READONCE) if err != nil { return self, err } self.termsIndexDivisor = termsIndexDivisor format := codec.PostingsFormat() log.Print("Obtaining SegmentReadState...") segmentReadState := newSegmentReadState(cfsDir, si.info, self.fieldInfos, context, termsIndexDivisor) // Ask codec for its Fields log.Print("Obtaining FieldsProducer...") self.fields, err = format.FieldsProducer(segmentReadState) if err != nil { return self, err } assert(self.fields != nil) // ask codec for its Norms: // TODO: since we don't write any norms file if there are no norms, // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! if self.fieldInfos.HasDocValues { log.Print("Obtaining DocValuesProducer...") self.dvProducer, err = codec.DocValuesFormat().FieldsProducer(segmentReadState) if err != nil { return self, err } assert(self.dvProducer != nil) } else { // self.dvProducer = nil } if self.fieldInfos.HasNorms { log.Print("Obtaining NormsDocValuesProducer...") self.normsProducer, err = codec.NormsFormat().NormsProducer(segmentReadState) if err != nil { return self, err } assert(self.normsProducer != nil) } else { // self.normsProducer = nil } log.Print("Obtaining StoredFieldsReader...") self.fieldsReaderOrig, err = si.info.Codec().(Codec).StoredFieldsFormat().FieldsReader(cfsDir, si.info, self.fieldInfos, context) if err != nil { return self, err } if self.fieldInfos.HasVectors { // open term vector files only as needed log.Print("Obtaining TermVectorsReader...") self.termVectorsReaderOrig, err = si.info.Codec().(Codec).TermVectorsFormat().VectorsReader(cfsDir, si.info, self.fieldInfos, context) if err != nil { return self, err } } else { // self.termVectorsReaderOrig = nil } log.Print("Success") success = true // Must assign this at the end -- if we hit an // exception above core, we don't want to attempt to // purge the FieldCache (will hit NPE because core is // not assigned yet). self.owner = owner return self, nil }