/* Closes all resouces and writes the entry table */
func (w *CompoundFileWriter) Close() (err error) {
if w.closed {
fmt.Println("CompoundFileWriter is already closed.")
return nil
}
// TODO this code should clean up after itself (remove partial .cfs/.cfe)
if err = func() (err error) {
var success = false
defer func() {
if success {
util.Close(w.dataOut)
} else {
util.CloseWhileSuppressingError(w.dataOut)
}
}()
assert2(w.pendingEntries.Len() == 0 && !w.outputTaken.Get(),
"CFS has pending open files")
w.closed = true
// open the compound stream; we can safely use IO_CONTEXT_DEFAULT
// here because this will only open the output if no file was
// added to the CFS
_, err = w.output(IO_CONTEXT_DEFAULT)
if err != nil {
return
}
assert(w.dataOut != nil)
err = codec.WriteFooter(w.dataOut)
if err != nil {
return
}
success = true
return nil
}(); err != nil {
return
}
var entryTableOut IndexOutput
var success = false
defer func() {
if success {
util.Close(entryTableOut)
} else {
util.CloseWhileSuppressingError(entryTableOut)
}
}()
entryTableOut, err = w.directory.CreateOutput(w.entryTableName, IO_CONTEXT_DEFAULT)
if err != nil {
return
}
err = w.writeEntryTable(w.entries, entryTableOut)
if err != nil {
return
}
success = true
return
}
func (w *PerFieldPostingsWriter) Close() error {
var subs []io.Closer
for _, v := range w.formats {
subs = append(subs, v)
}
return util.Close(subs...)
}
func (r *CompressingTermVectorsReader) Close() (err error) {
if !r.closed {
err = util.Close(r.vectorsStream)
r.closed = true
}
return err
}
func (w *CompressingStoredFieldsWriter) Close() error {
defer func() {
w.fieldsStream = nil
w.indexWriter = nil
}()
return util.Close(w.fieldsStream, w.indexWriter)
}
/*
Copies the file src to 'to' under the new file name dest.
If you want to copy the entire source directory to the destination
one, you can do so like this:
var to Directory // the directory to copy to
for _, file := range dir.ListAll() {
dir.Copy(to, file, newFile, IO_CONTEXT_DEFAULT)
// newFile can be either file, or a new name
}
NOTE: this method does not check whether dest exists and will
overwrite it if it does.
*/
func (d *DirectoryImpl) Copy(to Directory, src, dest string, ctx IOContext) (err error) {
var os IndexOutput
var is IndexInput
var success = false
defer func() {
if success {
err = util.Close(os, is)
} else {
util.CloseWhileSuppressingError(os, is)
}
defer func() {
recover() // ignore panic
}()
to.DeleteFile(dest) // ignore error
}()
os, err = to.CreateOutput(dest, ctx)
if err != nil {
return err
}
is, err = d.spi.OpenInput(src, ctx)
if err != nil {
return err
}
err = os.CopyBytes(is, is.Length())
if err != nil {
return err
}
success = true
return nil
}
func (w *Lucene41PostingsWriter) Close() (err error) {
var success = false
defer func() {
if success {
err = util.Close(w.docOut, w.posOut, w.payOut)
} else {
util.CloseWhileSuppressingError(w.docOut, w.posOut, w.payOut)
}
w.docOut = nil
w.posOut = nil
w.payOut = nil
}()
if err == nil && w.docOut != nil {
err = codec.WriteFooter(w.docOut)
}
if err == nil && w.posOut != nil {
err = codec.WriteFooter(w.posOut)
}
if err == nil && w.payOut != nil {
err = codec.WriteFooter(w.payOut)
}
if err != nil {
return
}
success = true
return nil
}
func (tvc *TermVectorsConsumer) flush(fieldsToFlush map[string]TermsHashPerField,
state *model.SegmentWriteState) (err error) {
if tvc.writer != nil {
numDocs := state.SegmentInfo.DocCount()
assert(numDocs > 0)
// At least one doc in this run had term vectors enabled
func() {
defer func() {
err = mergeError(err, util.Close(tvc.writer))
tvc.writer = nil
tvc.lastDocId = 0
tvc.hasVectors = false
}()
err = tvc.fill(numDocs)
if err == nil {
err = tvc.writer.Finish(state.FieldInfos, numDocs)
}
}()
if err != nil {
return err
}
}
return
}
func (nc *NormsConsumer) Close() (err error) {
var success = false
defer func() {
if success {
err = util.Close(nc.data, nc.meta)
} else {
util.CloseWhileSuppressingError(nc.data, nc.meta)
}
}()
if nc.meta != nil {
if err = nc.meta.WriteVInt(-1); err != nil { // write EOF marker
return
}
if err = codec.WriteFooter(nc.meta); err != nil { // write checksum
return
}
}
if nc.data != nil {
if err = codec.WriteFooter(nc.data); err != nil { // write checksum
return
}
}
success = true
return nil
}
// Close the underlying IndexInputs
func (r *CompressingStoredFieldsReader) Close() (err error) {
if !r.closed {
if err = util.Close(r.fieldsStream); err == nil {
r.closed = true
}
}
return
}
func (r *SegmentCoreReaders) decRef() {
if atomic.AddInt32(&r.refCount, -1) == 0 {
util.Close( /*self.termVectorsLocal, self.fieldsReaderLocal, docValuesLocal, r.normsLocal,*/
r.fields, r.dvProducer, r.termVectorsReaderOrig, r.fieldsReaderOrig,
r.cfsReader, r.normsProducer)
r.notifyListener <- r.owner
}
}
func (r *BlockTreeTermsReader) Close() error {
defer func() {
// Clear so refs to terms index is GCable even if
// app hangs onto us:
r.fields = make(map[string]FieldReader)
}()
return util.Close(r.in, r.postingsReader)
}
func (w *BlockTreeTermsWriter) Close() (err error) {
var success = false
defer func() {
if success {
util.Close(w.out, w.indexOut, w.postingsWriter)
} else {
util.CloseWhileSuppressingError(w.out, w.indexOut, w.postingsWriter)
}
}()
dirStart := w.out.FilePointer()
indexDirStart := w.indexOut.FilePointer()
if err = w.out.WriteVInt(int32(len(w.fields))); err != nil {
return
}
for _, field := range w.fields {
// fmt.Printf(" field %v %v terms\n", field.fieldInfo.Name, field.numTerms)
if err = w.out.WriteVInt(field.fieldInfo.Number); err == nil {
assert(field.numTerms > 0)
if err = w.out.WriteVLong(field.numTerms); err == nil {
if err = w.out.WriteVInt(int32(len(field.rootCode))); err == nil {
err = w.out.WriteBytes(field.rootCode)
if err == nil && field.fieldInfo.IndexOptions() != INDEX_OPT_DOCS_ONLY {
err = w.out.WriteVLong(field.sumTotalTermFreq)
}
if err == nil {
if err = w.out.WriteVLong(field.sumDocFreq); err == nil {
if err = w.out.WriteVInt(int32(field.docCount)); err == nil {
if err = w.out.WriteVInt(int32(field.longsSize)); err == nil {
if err = w.indexOut.WriteVLong(field.indexStartFP); err == nil {
if err = writeBytesRef(w.out, field.minTerm); err == nil {
err = writeBytesRef(w.out, field.maxTerm)
}
}
}
}
}
}
}
}
}
}
if err == nil {
if err = w.writeTrailer(w.out, dirStart); err == nil {
if err = codec.WriteFooter(w.out); err == nil {
if err = w.writeIndexTrailer(w.indexOut, indexDirStart); err == nil {
if err = codec.WriteFooter(w.indexOut); err == nil {
success = true
}
}
}
}
}
return
}
func (r *SegmentCoreReaders) decRef() {
if atomic.AddInt32(&r.refCount, -1) == 0 {
fmt.Println("--- closing core readers")
util.Close( /*self.termVectorsLocal, self.fieldsReaderLocal, r.normsLocal,*/
r.fields, r.termVectorsReaderOrig, r.fieldsReaderOrig,
r.cfsReader, r.normsProducer)
r.notifyListener <- true
}
}
func (w *CompressingStoredFieldsWriter) Close() error {
assert(w != nil)
defer func() {
if w != nil {
w.fieldsStream = nil
w.indexWriter = nil
}
}()
return util.Close(w.fieldsStream, w.indexWriter)
}
func (dvp *PerFieldDocValuesReader) Close() error {
fps := make([]DocValuesProducer, 0)
for _, v := range dvp.formats {
fps = append(fps, v)
}
items := make([]io.Closer, len(fps))
for i, v := range fps {
items[i] = v
}
return util.Close(items...)
}
func (r *PerFieldPostingsReader) Close() error {
fps := make([]FieldsProducer, 0)
for _, v := range r.formats {
fps = append(fps, v)
}
items := make([]io.Closer, len(fps))
for i, v := range fps {
items[i] = v
}
return util.Close(items...)
}
func newLucene42DocValuesProducer(state SegmentReadState,
dataCodec, dataExtension, metaCodec, metaExtension string) (dvp *Lucene42DocValuesProducer, err error) {
dvp = &Lucene42DocValuesProducer{
numericInstances: make(map[int]NumericDocValues),
}
dvp.maxDoc = state.segmentInfo.DocCount()
metaName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, metaExtension)
// read in the entries from the metadata file.
in, err := state.dir.OpenInput(metaName, state.context)
if err != nil {
return dvp, err
}
success := false
defer func() {
if success {
err = util.Close(in)
} else {
util.CloseWhileSuppressingError(in)
}
}()
version, err := codec.CheckHeader(in, metaCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT)
if err != nil {
return dvp, err
}
dvp.numerics = make(map[int]NumericEntry)
dvp.binaries = make(map[int]BinaryEntry)
dvp.fsts = make(map[int]FSTEntry)
err = dvp.readFields(in)
if err != nil {
return dvp, err
}
success = true
success = false
dataName := util.SegmentFileName(state.segmentInfo.Name, state.segmentSuffix, dataExtension)
dvp.data, err = state.dir.OpenInput(dataName, state.context)
if err != nil {
return dvp, err
}
version2, err := codec.CheckHeader(dvp.data, dataCodec, LUCENE42_DV_VERSION_START, LUCENE42_DV_VERSION_CURRENT)
if err != nil {
return dvp, err
}
if version != version2 {
return dvp, errors.New("Format versions mismatch")
}
return dvp, nil
}
func (d *CompoundFileDirectory) Close() error {
d.Lock() // syncronized
defer d.Unlock()
// fmt.Printf("Closing %v...\n", d)
if !d.IsOpen {
fmt.Println("CompoundFileDirectory is already closed.")
// allow double close - usually to be consistent with other closeables
return nil // already closed
}
d.IsOpen = false
if d.writer != nil {
assert(d.openForWrite)
return d.writer.Close()
} else {
return util.Close(d.handle)
}
}
/* Copy the contents of the file with specified extension into the provided output stream. */
func (w *CompoundFileWriter) copyFileEntry(dataOut IndexOutput, fileEntry *FileEntry) (n int64, err error) {
var is IndexInput
is, err = fileEntry.dir.OpenInput(fileEntry.file, IO_CONTEXT_READONCE)
if err != nil {
return 0, err
}
var success = false
defer func() {
if success {
err = util.Close(is)
// copy successful - delete file
if err == nil {
fileEntry.dir.DeleteFile(fileEntry.file) // ignore error
}
} else {
util.CloseWhileSuppressingError(is)
}
}()
startPtr := dataOut.FilePointer()
length := fileEntry.length
err = dataOut.CopyBytes(is, length)
if err != nil {
return 0, err
}
// verify that the output length diff is equal to original file
endPtr := dataOut.FilePointer()
diff := endPtr - startPtr
if diff != length {
return 0, errors.New(fmt.Sprintf(
"Difference in the output file offsets %v does not match the original file length %v",
diff, length))
}
fileEntry.offset = startPtr
success = true
return length, nil
}
func (w *FreqProxTermsWriter) flush(fieldsToFlush map[string]TermsHashPerField,
state *model.SegmentWriteState) (err error) {
if err = w.TermsHashImpl.flush(fieldsToFlush, state); err != nil {
return
}
// Gather all FieldData's that have postings, across all ThreadStates
var allFields []*FreqProxTermsWriterPerField
for _, f := range fieldsToFlush {
if perField := f.(*FreqProxTermsWriterPerField); perField.bytesHash.Size() > 0 {
allFields = append(allFields, perField)
}
}
// Sort by field name
util.IntroSort(FreqProxTermsWriterPerFields(allFields))
var consumer FieldsConsumer
if consumer, err = state.SegmentInfo.Codec().(Codec).PostingsFormat().FieldsConsumer(state); err != nil {
return
}
var success = false
defer func() {
if success {
err = util.Close(consumer)
} else {
util.CloseWhileSuppressingError(consumer)
}
}()
var termsHash TermsHash
// Current writer chain:
// FieldsConsumer
// -> IMPL: FormatPostingsTermsDictWriter
// -> TermsConsumer
// -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
// -> DocsConsumer
// -> IMPL: FormatPostingsDocWriter
// -> PositionsConsumer
// -> IMPL: FormatPostingsPositionsWriter
for _, fieldWriter := range allFields {
fieldInfo := fieldWriter.fieldInfo
// If this field has postings then add them to the segment
if err = fieldWriter.flush(fieldInfo.Name, consumer, state); err != nil {
return
}
assert(termsHash == nil || termsHash == fieldWriter.termsHash)
termsHash = fieldWriter.termsHash
fieldWriter.reset()
}
if termsHash != nil {
termsHash.reset()
}
success = true
return nil
}
func (d *CompoundFileDirectory) readEntries(handle IndexInput, dir Directory, name string) (mapping map[string]FileSlice, err error) {
var stream IndexInput = nil
var entriesStream ChecksumIndexInput = nil
// read the first VInt. If it is negative, it's the version number
// otherwise it's the count (pre-3.1 indexes)
var success = false
defer func() {
if success {
err = util.Close(stream, entriesStream)
} else {
util.CloseWhileSuppressingError(stream, entriesStream)
}
}()
stream = handle.Clone()
// fmt.Printf("Reading from stream: %v\n", stream)
firstInt, err := stream.ReadVInt()
if err != nil {
return nil, err
}
// impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible
// and separate norms/etc are outside of cfs.
if firstInt == int32(CODEC_MAGIC_BYTE1) {
if secondByte, err := stream.ReadByte(); err == nil {
if thirdByte, err := stream.ReadByte(); err == nil {
if fourthByte, err := stream.ReadByte(); err == nil {
if secondByte != CODEC_MAGIC_BYTE2 ||
thirdByte != CODEC_MAGIC_BYTE3 ||
fourthByte != CODEC_MAGIC_BYTE4 {
return nil, errors.New(fmt.Sprintf(
"Illegal/impossible header for CFS file: %v,%v,%v",
secondByte, thirdByte, fourthByte))
}
}
}
}
if err != nil {
return nil, err
}
d.version, err = int32ToInt(codec.CheckHeaderNoMagic(stream, CFD_DATA_CODEC, CFD_VERSION_START, CFD_VERSION_CURRENT))
if err != nil {
return nil, err
}
entriesFileName := util.SegmentFileName(util.StripExtension(name), "", COMPOUND_FILE_ENTRIES_EXTENSION)
entriesStream, err = dir.OpenChecksumInput(entriesFileName, IO_CONTEXT_READONCE)
if err != nil {
return nil, err
}
_, err = codec.CheckHeader(entriesStream, CFD_ENTRY_CODEC, CFD_VERSION_START, CFD_VERSION_CURRENT)
if err != nil {
return nil, err
}
numEntries, err := entriesStream.ReadVInt()
if err != nil {
return nil, err
}
mapping = make(map[string]FileSlice)
// fmt.Printf("Entries number: %v\n", numEntries)
for i := int32(0); i < numEntries; i++ {
id, err := entriesStream.ReadString()
if err != nil {
return nil, err
}
if _, ok := mapping[id]; ok {
return nil, errors.New(fmt.Sprintf(
"Duplicate cfs entry id=%v in CFS: %v", id, entriesStream))
}
// log.Printf("Found entry: %v", id)
offset, err := entriesStream.ReadLong()
if err != nil {
return nil, err
}
length, err := entriesStream.ReadLong()
if err != nil {
return nil, err
}
mapping[id] = FileSlice{offset, length}
}
if d.version >= CFD_VERSION_CHECKSUM {
_, err = codec.CheckFooter(entriesStream)
} else {
err = codec.CheckEOF(entriesStream)
}
if err != nil {
return nil, err
}
} else {
// TODO remove once 3.x is not supported anymore
panic("not supported yet; will also be obsolete soon")
}
success = true
return mapping, nil
}
func (r *Lucene41PostingsReader) Close() error {
return util.Close(r.docIn, r.posIn, r.payIn)
}
func newLucene49NormsProducer(state SegmentReadState,
dataCodec, dataExtension, metaCodec, metaExtension string) (np *NormsProducer, err error) {
np = &NormsProducer{
Locker: new(sync.Mutex),
norms: make(map[int]*NormsEntry),
instances: make(map[int]NumericDocValues),
maxDoc: state.SegmentInfo.DocCount(),
ramBytesUsed: util.ShallowSizeOfInstance(reflect.TypeOf(np)),
}
metaName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension)
// read in the entries from the metadta file.
var in store.ChecksumIndexInput
if in, err = state.Dir.OpenChecksumInput(metaName, state.Context); err != nil {
return nil, err
}
if err = func() error {
var success = false
defer func() {
if success {
err = util.Close(in)
} else {
util.CloseWhileSuppressingError(in)
}
}()
if np.version, err = codec.CheckHeader(in, metaCodec, VERSION_START, VERSION_CURRENT); err != nil {
return err
}
if err = np.readFields(in, state.FieldInfos); err != nil {
return err
}
if _, err = codec.CheckFooter(in); err != nil {
return err
}
success = true
return nil
}(); err != nil {
return nil, err
}
dataName := util.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension)
if np.data, err = state.Dir.OpenInput(dataName, state.Context); err != nil {
return nil, err
}
var success = false
defer func() {
if !success {
util.CloseWhileSuppressingError(np.data)
}
}()
var version2 int32
if version2, err = codec.CheckHeader(np.data, dataCodec, VERSION_START, VERSION_CURRENT); err != nil {
return nil, err
}
if version2 != np.version {
return nil, errors.New("Format versions mismatch")
}
// NOTE: data file is too costly to verify checksum against all the
// bytes on open, but fo rnow we at least verify proper structure
// of the checksum footer: which looks for FOOTER_MATIC +
// algorithmID. This is cheap and can detect some forms of
// corruption such as file trucation.
if _, err = codec.RetrieveChecksum(np.data); err != nil {
return nil, err
}
success = true
return np, nil
}
// Sole constructor
func newCompressingStoredFieldsReader(d store.Directory,
si *model.SegmentInfo, segmentSuffix string,
fn model.FieldInfos, ctx store.IOContext, formatName string,
compressionMode compressing.CompressionMode) (r *CompressingStoredFieldsReader, err error) {
r = &CompressingStoredFieldsReader{}
r.compressionMode = compressionMode
segment := si.Name
r.fieldInfos = fn
r.numDocs = si.DocCount()
var indexStream store.IndexInput
success := false
defer func() {
if !success {
log.Println("Failed to initialize CompressionStoredFieldsReader.")
if err != nil {
log.Print(err)
}
util.Close(r, indexStream)
}
}()
// Load the index into memory
indexStreamFN := util.SegmentFileName(segment, segmentSuffix, lucene40.FIELDS_INDEX_EXTENSION)
indexStream, err = d.OpenInput(indexStreamFN, ctx)
if err != nil {
return nil, err
}
codecNameIdx := formatName + CODEC_SFX_IDX
codec.CheckHeader(indexStream, codecNameIdx, CODEC_SFX_VERSION_START, CODEC_SFX_VERSION_CURRENT)
if int64(codec.HeaderLength(codecNameIdx)) != indexStream.FilePointer() {
panic("assert fail")
}
r.indexReader, err = newCompressingStoredFieldsIndexReader(indexStream, si)
if err != nil {
return nil, err
}
err = indexStream.Close()
if err != nil {
return nil, err
}
indexStream = nil
// Open the data file and read metadata
fieldsStreamFN := util.SegmentFileName(segment, segmentSuffix, lucene40.FIELDS_EXTENSION)
r.fieldsStream, err = d.OpenInput(fieldsStreamFN, ctx)
if err != nil {
return nil, err
}
codecNameDat := formatName + CODEC_SFX_DAT
codec.CheckHeader(r.fieldsStream, codecNameDat, CODEC_SFX_VERSION_START, CODEC_SFX_VERSION_CURRENT)
if int64(codec.HeaderLength(codecNameDat)) != r.fieldsStream.FilePointer() {
panic("assert fail")
}
n, err := r.fieldsStream.ReadVInt()
if err != nil {
return nil, err
}
r.packedIntsVersion = int(n)
r.decompressor = compressionMode.NewDecompressor()
r.bytes = make([]byte, 0)
success = true
return r, nil
}