/* Closes all resouces and writes the entry table */ func (w *CompoundFileWriter) Close() (err error) { if w.closed { fmt.Println("CompoundFileWriter is already closed.") return nil } // TODO this code should clean up after itself (remove partial .cfs/.cfe) if err = func() (err error) { var success = false defer func() { if success { util.Close(w.dataOut) } else { util.CloseWhileSuppressingError(w.dataOut) } }() assert2(w.pendingEntries.Len() == 0 && !w.outputTaken.Get(), "CFS has pending open files") w.closed = true // open the compound stream; we can safely use IO_CONTEXT_DEFAULT // here because this will only open the output if no file was // added to the CFS _, err = w.output(IO_CONTEXT_DEFAULT) if err != nil { return } assert(w.dataOut != nil) err = codec.WriteFooter(w.dataOut) if err != nil { return } success = true return nil }(); err != nil { return } var entryTableOut IndexOutput var success = false defer func() { if success { util.Close(entryTableOut) } else { util.CloseWhileSuppressingError(entryTableOut) } }() entryTableOut, err = w.directory.CreateOutput(w.entryTableName, IO_CONTEXT_DEFAULT) if err != nil { return } err = w.writeEntryTable(w.entries, entryTableOut) if err != nil { return } success = true return }
func (w *PerFieldPostingsWriter) Close() error { var subs []io.Closer for _, v := range w.formats { subs = append(subs, v) } return util.Close(subs...) }
func (r *CompressingTermVectorsReader) Close() (err error) { if !r.closed { err = util.Close(r.vectorsStream) r.closed = true } return err }
func (w *CompressingStoredFieldsWriter) Close() error { defer func() { w.fieldsStream = nil w.indexWriter = nil }() return util.Close(w.fieldsStream, w.indexWriter) }
/* Copies the file src to 'to' under the new file name dest. If you want to copy the entire source directory to the destination one, you can do so like this: var to Directory // the directory to copy to for _, file := range dir.ListAll() { dir.Copy(to, file, newFile, IO_CONTEXT_DEFAULT) // newFile can be either file, or a new name } NOTE: this method does not check whether dest exists and will overwrite it if it does. */ func (d *DirectoryImpl) Copy(to Directory, src, dest string, ctx IOContext) (err error) { var os IndexOutput var is IndexInput var success = false defer func() { if success { err = util.Close(os, is) } else { util.CloseWhileSuppressingError(os, is) } defer func() { recover() // ignore panic }() to.DeleteFile(dest) // ignore error }() os, err = to.CreateOutput(dest, ctx) if err != nil { return err } is, err = d.spi.OpenInput(src, ctx) if err != nil { return err } err = os.CopyBytes(is, is.Length()) if err != nil { return err } success = true return nil }
func (w *Lucene41PostingsWriter) Close() (err error) { var success = false defer func() { if success { err = util.Close(w.docOut, w.posOut, w.payOut) } else { util.CloseWhileSuppressingError(w.docOut, w.posOut, w.payOut) } w.docOut = nil w.posOut = nil w.payOut = nil }() if err == nil && w.docOut != nil { err = codec.WriteFooter(w.docOut) } if err == nil && w.posOut != nil { err = codec.WriteFooter(w.posOut) } if err == nil && w.payOut != nil { err = codec.WriteFooter(w.payOut) } if err != nil { return } success = true return nil }
func (tvc *TermVectorsConsumer) flush(fieldsToFlush map[string]TermsHashPerField, state *model.SegmentWriteState) (err error) { if tvc.writer != nil { numDocs := state.SegmentInfo.DocCount() assert(numDocs > 0) // At least one doc in this run had term vectors enabled func() { defer func() { err = mergeError(err, util.Close(tvc.writer)) tvc.writer = nil tvc.lastDocId = 0 tvc.hasVectors = false }() err = tvc.fill(numDocs) if err == nil { err = tvc.writer.Finish(state.FieldInfos, numDocs) } }() if err != nil { return err } } return }
func (nc *NormsConsumer) Close() (err error) { var success = false defer func() { if success { err = util.Close(nc.data, nc.meta) } else { util.CloseWhileSuppressingError(nc.data, nc.meta) } }() if nc.meta != nil { if err = nc.meta.WriteVInt(-1); err != nil { // write EOF marker return } if err = codec.WriteFooter(nc.meta); err != nil { // write checksum return } } if nc.data != nil { if err = codec.WriteFooter(nc.data); err != nil { // write checksum return } } success = true return nil }
// Close the underlying IndexInputs func (r *CompressingStoredFieldsReader) Close() (err error) { if !r.closed { if err = util.Close(r.fieldsStream); err == nil { r.closed = true } } return }
func (r *SegmentCoreReaders) decRef() { if atomic.AddInt32(&r.refCount, -1) == 0 { util.Close( /*self.termVectorsLocal, self.fieldsReaderLocal, docValuesLocal, r.normsLocal,*/ r.fields, r.dvProducer, r.termVectorsReaderOrig, r.fieldsReaderOrig, r.cfsReader, r.normsProducer) r.notifyListener <- r.owner } }
func (r *BlockTreeTermsReader) Close() error { defer func() { // Clear so refs to terms index is GCable even if // app hangs onto us: r.fields = make(map[string]FieldReader) }() return util.Close(r.in, r.postingsReader) }
func (w *BlockTreeTermsWriter) Close() (err error) { var success = false defer func() { if success { util.Close(w.out, w.indexOut, w.postingsWriter) } else { util.CloseWhileSuppressingError(w.out, w.indexOut, w.postingsWriter) } }() dirStart := w.out.FilePointer() indexDirStart := w.indexOut.FilePointer() if err = w.out.WriteVInt(int32(len(w.fields))); err != nil { return } for _, field := range w.fields { // fmt.Printf(" field %v %v terms\n", field.fieldInfo.Name, field.numTerms) if err = w.out.WriteVInt(field.fieldInfo.Number); err == nil { assert(field.numTerms > 0) if err = w.out.WriteVLong(field.numTerms); err == nil { if err = w.out.WriteVInt(int32(len(field.rootCode))); err == nil { err = w.out.WriteBytes(field.rootCode) if err == nil && field.fieldInfo.IndexOptions() != INDEX_OPT_DOCS_ONLY { err = w.out.WriteVLong(field.sumTotalTermFreq) } if err == nil { if err = w.out.WriteVLong(field.sumDocFreq); err == nil { if err = w.out.WriteVInt(int32(field.docCount)); err == nil { if err = w.out.WriteVInt(int32(field.longsSize)); err == nil { if err = w.indexOut.WriteVLong(field.indexStartFP); err == nil { if err = writeBytesRef(w.out, field.minTerm); err == nil { err = writeBytesRef(w.out, field.maxTerm) } } } } } } } } } } if err == nil { if err = w.writeTrailer(w.out, dirStart); err == nil { if err = codec.WriteFooter(w.out); err == nil { if err = w.writeIndexTrailer(w.indexOut, indexDirStart); err == nil { if err = codec.WriteFooter(w.indexOut); err == nil { success = true } } } } } return }
func (r *SegmentCoreReaders) decRef() { if atomic.AddInt32(&r.refCount, -1) == 0 { fmt.Println("--- closing core readers") util.Close( /*self.termVectorsLocal, self.fieldsReaderLocal, r.normsLocal,*/ r.fields, r.termVectorsReaderOrig, r.fieldsReaderOrig, r.cfsReader, r.normsProducer) r.notifyListener <- true } }
func (w *CompressingStoredFieldsWriter) Close() error { assert(w != nil) defer func() { if w != nil { w.fieldsStream = nil w.indexWriter = nil } }() return util.Close(w.fieldsStream, w.indexWriter) }
func (dvp *PerFieldDocValuesReader) Close() error { fps := make([]DocValuesProducer, 0) for _, v := range dvp.formats { fps = append(fps, v) } items := make([]io.Closer, len(fps)) for i, v := range fps { items[i] = v } return util.Close(items...) }
func (r *PerFieldPostingsReader) Close() error { fps := make([]FieldsProducer, 0) for _, v := range r.formats { fps = append(fps, v) } items := make([]io.Closer, len(fps)) for i, v := range fps { items[i] = v } return util.Close(items...) }
func newLucene42DocValuesProducer(state SegmentReadState, dataCodec, dataExtension, metaCodec, metaExtension string) (dvp *Lucene42DocValuesProducer, err error) { dvp = &Lucene42DocValuesProducer{ numericInstances: make(map[int]NumericDocValues), } dvp.maxDoc = state.segmentInfo.DocCount() metaName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, metaExtension) // read in the entries from the metadata file. in, err := state.dir.OpenInput(metaName, state.context) if err != nil { return dvp, err } success := false defer func() { if success { err = util.Close(in) } else { util.CloseWhileSuppressingError(in) } }() version, err := codec.CheckHeader(in, metaCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT) if err != nil { return dvp, err } dvp.numerics = make(map[int]NumericEntry) dvp.binaries = make(map[int]BinaryEntry) dvp.fsts = make(map[int]FSTEntry) err = dvp.readFields(in) if err != nil { return dvp, err } success = true success = false dataName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, dataExtension) dvp.data, err = state.dir.OpenInput(dataName, state.context) if err != nil { return dvp, err } version2, err := codec.CheckHeader(dvp.data, dataCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT) if err != nil { return dvp, err } if version != version2 { return dvp, errors.New("Format versions mismatch") } return dvp, nil }
func (d *CompoundFileDirectory) Close() error { d.Lock() // syncronized defer d.Unlock() // fmt.Printf("Closing %v...\n", d) if !d.IsOpen { fmt.Println("CompoundFileDirectory is already closed.") // allow double close - usually to be consistent with other closeables return nil // already closed } d.IsOpen = false if d.writer != nil { assert(d.openForWrite) return d.writer.Close() } else { return util.Close(d.handle) } }
/* Copy the contents of the file with specified extension into the provided output stream. */ func (w *CompoundFileWriter) copyFileEntry(dataOut IndexOutput, fileEntry *FileEntry) (n int64, err error) { var is IndexInput is, err = fileEntry.dir.OpenInput(fileEntry.file, IO_CONTEXT_READONCE) if err != nil { return 0, err } var success = false defer func() { if success { err = util.Close(is) // copy successful - delete file if err == nil { fileEntry.dir.DeleteFile(fileEntry.file) // ignore error } } else { util.CloseWhileSuppressingError(is) } }() startPtr := dataOut.FilePointer() length := fileEntry.length err = dataOut.CopyBytes(is, length) if err != nil { return 0, err } // verify that the output length diff is equal to original file endPtr := dataOut.FilePointer() diff := endPtr - startPtr if diff != length { return 0, errors.New(fmt.Sprintf( "Difference in the output file offsets %v does not match the original file length %v", diff, length)) } fileEntry.offset = startPtr success = true return length, nil }
func (w *FreqProxTermsWriter) flush(fieldsToFlush map[string]TermsHashPerField, state *model.SegmentWriteState) (err error) { if err = w.TermsHashImpl.flush(fieldsToFlush, state); err != nil { return } // Gather all FieldData's that have postings, across all ThreadStates var allFields []*FreqProxTermsWriterPerField for _, f := range fieldsToFlush { if perField := f.(*FreqProxTermsWriterPerField); perField.bytesHash.Size() > 0 { allFields = append(allFields, perField) } } // Sort by field name util.IntroSort(FreqProxTermsWriterPerFields(allFields)) var consumer FieldsConsumer if consumer, err = state.SegmentInfo.Codec().(Codec).PostingsFormat().FieldsConsumer(state); err != nil { return } var success = false defer func() { if success { err = util.Close(consumer) } else { util.CloseWhileSuppressingError(consumer) } }() var termsHash TermsHash // Current writer chain: // FieldsConsumer // -> IMPL: FormatPostingsTermsDictWriter // -> TermsConsumer // -> IMPL: FormatPostingsTermsDictWriter.TermsWriter // -> DocsConsumer // -> IMPL: FormatPostingsDocWriter // -> PositionsConsumer // -> IMPL: FormatPostingsPositionsWriter for _, fieldWriter := range allFields { fieldInfo := fieldWriter.fieldInfo // If this field has postings then add them to the segment if err = fieldWriter.flush(fieldInfo.Name, consumer, state); err != nil { return } assert(termsHash == nil || termsHash == fieldWriter.termsHash) termsHash = fieldWriter.termsHash fieldWriter.reset() } if termsHash != nil { termsHash.reset() } success = true return nil }
func (d *CompoundFileDirectory) readEntries(handle IndexInput, dir Directory, name string) (mapping map[string]FileSlice, err error) { var stream IndexInput = nil var entriesStream ChecksumIndexInput = nil // read the first VInt. If it is negative, it's the version number // otherwise it's the count (pre-3.1 indexes) var success = false defer func() { if success { err = util.Close(stream, entriesStream) } else { util.CloseWhileSuppressingError(stream, entriesStream) } }() stream = handle.Clone() // fmt.Printf("Reading from stream: %v\n", stream) firstInt, err := stream.ReadVInt() if err != nil { return nil, err } // impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible // and separate norms/etc are outside of cfs. if firstInt == int32(CODEC_MAGIC_BYTE1) { if secondByte, err := stream.ReadByte(); err == nil { if thirdByte, err := stream.ReadByte(); err == nil { if fourthByte, err := stream.ReadByte(); err == nil { if secondByte != CODEC_MAGIC_BYTE2 || thirdByte != CODEC_MAGIC_BYTE3 || fourthByte != CODEC_MAGIC_BYTE4 { return nil, errors.New(fmt.Sprintf( "Illegal/impossible header for CFS file: %v,%v,%v", secondByte, thirdByte, fourthByte)) } } } } if err != nil { return nil, err } d.version, err = int32ToInt(codec.CheckHeaderNoMagic(stream, CFD_DATA_CODEC, CFD_VERSION_START, CFD_VERSION_CURRENT)) if err != nil { return nil, err } entriesFileName := util.SegmentFileName(util.StripExtension(name), "", COMPOUND_FILE_ENTRIES_EXTENSION) entriesStream, err = dir.OpenChecksumInput(entriesFileName, IO_CONTEXT_READONCE) if err != nil { return nil, err } _, err = codec.CheckHeader(entriesStream, CFD_ENTRY_CODEC, CFD_VERSION_START, CFD_VERSION_CURRENT) if err != nil { return nil, err } numEntries, err := entriesStream.ReadVInt() if err != nil { return nil, err } mapping = make(map[string]FileSlice) // fmt.Printf("Entries number: %v\n", numEntries) for i := int32(0); i < numEntries; i++ { id, err := entriesStream.ReadString() if err != nil { return nil, err } if _, ok := mapping[id]; ok { return nil, errors.New(fmt.Sprintf( "Duplicate cfs entry id=%v in CFS: %v", id, entriesStream)) } // log.Printf("Found entry: %v", id) offset, err := entriesStream.ReadLong() if err != nil { return nil, err } length, err := entriesStream.ReadLong() if err != nil { return nil, err } mapping[id] = FileSlice{offset, length} } if d.version >= CFD_VERSION_CHECKSUM { _, err = codec.CheckFooter(entriesStream) } else { err = codec.CheckEOF(entriesStream) } if err != nil { return nil, err } } else { // TODO remove once 3.x is not supported anymore panic("not supported yet; will also be obsolete soon") } success = true return mapping, nil }
func (r *Lucene41PostingsReader) Close() error { return util.Close(r.docIn, r.posIn, r.payIn) }
func newLucene49NormsProducer(state SegmentReadState, dataCodec, dataExtension, metaCodec, metaExtension string) (np *NormsProducer, err error) { np = &NormsProducer{ Locker: new(sync.Mutex), norms: make(map[int]*NormsEntry), instances: make(map[int]NumericDocValues), maxDoc: state.SegmentInfo.DocCount(), ramBytesUsed: util.ShallowSizeOfInstance(reflect.TypeOf(np)), } metaName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension) // read in the entries from the metadta file. var in store.ChecksumIndexInput if in, err = state.Dir.OpenChecksumInput(metaName, state.Context); err != nil { return nil, err } if err = func() error { var success = false defer func() { if success { err = util.Close(in) } else { util.CloseWhileSuppressingError(in) } }() if np.version, err = codec.CheckHeader(in, metaCodec, VERSION_START, VERSION_CURRENT); err != nil { return err } if err = np.readFields(in, state.FieldInfos); err != nil { return err } if _, err = codec.CheckFooter(in); err != nil { return err } success = true return nil }(); err != nil { return nil, err } dataName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension) if np.data, err = state.Dir.OpenInput(dataName, state.Context); err != nil { return nil, err } var success = false defer func() { if !success { util.CloseWhileSuppressingError(np.data) } }() var version2 int32 if version2, err = codec.CheckHeader(np.data, dataCodec, VERSION_START, VERSION_CURRENT); err != nil { return nil, err } if version2 != np.version { return nil, errors.New("Format versions mismatch") } // NOTE: data file is too costly to verify checksum against all the // bytes on open, but fo rnow we at least verify proper structure // of the checksum footer: which looks for FOOTER_MATIC + // algorithmID. This is cheap and can detect some forms of // corruption such as file trucation. if _, err = codec.RetrieveChecksum(np.data); err != nil { return nil, err } success = true return np, nil }
// Sole constructor func newCompressingStoredFieldsReader(d store.Directory, si *model.SegmentInfo, segmentSuffix string, fn model.FieldInfos, ctx store.IOContext, formatName string, compressionMode compressing.CompressionMode) (r *CompressingStoredFieldsReader, err error) { r = &CompressingStoredFieldsReader{} r.compressionMode = compressionMode segment := si.Name r.fieldInfos = fn r.numDocs = si.DocCount() var indexStream store.IndexInput success := false defer func() { if !success { log.Println("Failed to initialize CompressionStoredFieldsReader.") if err != nil { log.Print(err) } util.Close(r, indexStream) } }() // Load the index into memory indexStreamFN := util.SegmentFileName(segment, segmentSuffix, lucene40.FIELDS_INDEX_EXTENSION) indexStream, err = d.OpenInput(indexStreamFN, ctx) if err != nil { return nil, err } codecNameIdx := formatName + CODEC_SFX_IDX codec.CheckHeader(indexStream, codecNameIdx, CODEC_SFX_VERSION_START, CODEC_SFX_VERSION_CURRENT) if int64(codec.HeaderLength(codecNameIdx)) != indexStream.FilePointer() { panic("assert fail") } r.indexReader, err = newCompressingStoredFieldsIndexReader(indexStream, si) if err != nil { return nil, err } err = indexStream.Close() if err != nil { return nil, err } indexStream = nil // Open the data file and read metadata fieldsStreamFN := util.SegmentFileName(segment, segmentSuffix, lucene40.FIELDS_EXTENSION) r.fieldsStream, err = d.OpenInput(fieldsStreamFN, ctx) if err != nil { return nil, err } codecNameDat := formatName + CODEC_SFX_DAT codec.CheckHeader(r.fieldsStream, codecNameDat, CODEC_SFX_VERSION_START, CODEC_SFX_VERSION_CURRENT) if int64(codec.HeaderLength(codecNameDat)) != r.fieldsStream.FilePointer() { panic("assert fail") } n, err := r.fieldsStream.ReadVInt() if err != nil { return nil, err } r.packedIntsVersion = int(n) r.decompressor = compressionMode.NewDecompressor() r.bytes = make([]byte, 0) success = true return r, nil }