/*
A utility for writing the SEGMENTS_GEN file to a Directory.
NOTE: this is an internal utility which is kept public so that it's
accessible by code from other packages. You should avoid calling this
method unless you're absolutely sure what you're doing!
*/
func writeSegmentsGen(dir store.Directory, generation int64) {
if err := func() (err error) {
var genOutput store.IndexOutput
genOutput, err = dir.CreateOutput(INDEX_FILENAME_SEGMENTS_GEN, store.IO_CONTEXT_READONCE)
if err != nil {
return err
}
defer func() {
err = mergeError(err, genOutput.Close())
err = mergeError(err, dir.Sync([]string{INDEX_FILENAME_SEGMENTS_GEN}))
}()
if err = genOutput.WriteInt(FORMAT_SEGMENTS_GEN_CURRENT); err == nil {
if err = genOutput.WriteLong(generation); err == nil {
if err = genOutput.WriteLong(generation); err == nil {
err = codec.WriteFooter(genOutput)
}
}
}
return err
}(); err != nil {
// It's OK if we fail to write this file since it's used only as
// one of the retry fallbacks.
dir.DeleteFile(INDEX_FILENAME_SEGMENTS_GEN)
// Ignore error; this file is only used in a retry fallback on init
}
}
/*
Commit live docs (writes new _X_N.del files) and field update (writes
new _X_N.del files) to the directory; returns true if it wrote any
file and false if there were no new deletes or updates to write:
*/
func (rld *ReadersAndUpdates) writeLiveDocs(dir store.Directory) (bool, error) {
panic("not implemented yet")
rld.Lock()
defer rld.Unlock()
log.Printf("rld.writeLiveDocs seg=%v pendingDelCount=%v", rld.info, rld._pendingDeleteCount)
if rld._pendingDeleteCount != 0 {
// We have new deletes
assert(rld._liveDocs.Length() == rld.info.Info.DocCount())
// Do this so we can delete any created files on error; this
// saves all codecs from having to do it:
trackingDir := store.NewTrackingDirectoryWrapper(dir)
// We can write directly to the actual name (vs to a .tmp &
// renaming it) becaues the file is not live until segments file
// is written:
var success = false
defer func() {
if !success {
// Advance only the nextWriteDelGen so that a 2nd attempt to
// write will write to a new file
rld.info.AdvanceNextWriteDelGen()
// Delete any partially created files(s):
trackingDir.EachCreatedFiles(func(filename string) {
dir.DeleteFile(filename) // ignore error
})
}
}()
err := rld.info.Info.Codec().(Codec).LiveDocsFormat().WriteLiveDocs(rld._liveDocs.(util.MutableBits),
trackingDir, rld.info, rld._pendingDeleteCount, store.IO_CONTEXT_DEFAULT)
if err != nil {
return false, err
}
success = true
// If we hit an error in the line above (e.g. disk full) then
// info's delGen remains pointing to the previous (successfully
// written) del docs:
rld.info.AdvanceDelGen()
rld.info.SetDelCount(rld.info.DelCount() + rld._pendingDeleteCount)
assert(rld.info.DelCount() <= rld.info.Info.DocCount())
rld._pendingDeleteCount = 0
return true, nil
}
return false, nil
}
func (sis *SegmentInfos) finishCommit(dir store.Directory) (fileName string, err error) {
assert(dir != nil)
assert2(sis.pendingSegnOutput != nil, "prepareCommit was not called")
if err = func() error {
var success = false
defer func() {
if !success {
// Closes pendingSegnOutput & delets partial segments_N:
sis.rollbackCommit(dir)
} else {
err := func() error {
var success = false
defer func() {
if !success {
// Closes pendingSegnOutput & delets partial segments_N:
sis.rollbackCommit(dir)
} else {
sis.pendingSegnOutput = nil
}
}()
err := sis.pendingSegnOutput.Close()
success = err == nil
return err
}()
assertn(err == nil, "%v", err) // no shadow
}
}()
if err := codec.WriteFooter(sis.pendingSegnOutput); err != nil {
return err
}
success = true
return nil
}(); err != nil {
return
}
// NOTE: if we crash here, we have left a segments_N file in the
// directory in a possibly corrupt state (if some bytes made it to
// stable storage and others didn't). But, the segments_N file
// includes checksum at the end, which should catch this case. So
// when a reader tries to read it, it will return an index corrupt
// error, which should cause the retry logic in SegmentInfos to
// kick in and load the last good (previous) segments_N-1 file.
fileName = util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)
if err = func() error {
var success = false
defer func() {
if !success {
dir.DeleteFile(fileName)
// suppress error so we keep returning the original error
}
}()
err := dir.Sync([]string{fileName})
success = err == nil
return err
}(); err != nil {
return
}
sis.lastGeneration = sis.generation
writeSegmentsGen(dir, sis.generation)
return
}
func (sis *SegmentInfos) write(directory store.Directory) (err error) {
segmentsFilename := sis.nextSegmentFilename()
// Always advance the generation on write:
if sis.generation == -1 {
sis.generation = 1
} else {
sis.generation++
}
var segnOutput store.IndexOutput
var success = false
// var upgradedSIFiles = make(map[string]bool)
defer func() {
if !success {
// We hit an error above; try to close the file but suppress
// any errors
util.CloseWhileSuppressingError(segnOutput)
// for filename, _ := range upgradedSIFiles {
// directory.DeleteFile(filename) // ignore error
// }
// Try not to leave a truncated segments_N fle in the index:
directory.DeleteFile(segmentsFilename) // ignore error
}
}()
if segnOutput, err = directory.CreateOutput(segmentsFilename, store.IO_CONTEXT_DEFAULT); err != nil {
return
}
if err = codec.WriteHeader(segnOutput, "segments", VERSION_49); err != nil {
return
}
if err = segnOutput.WriteLong(sis.version); err == nil {
if err = segnOutput.WriteInt(int32(sis.counter)); err == nil {
err = segnOutput.WriteInt(int32(len(sis.Segments)))
}
}
if err != nil {
return
}
for _, siPerCommit := range sis.Segments {
si := siPerCommit.Info
if err = segnOutput.WriteString(si.Name); err == nil {
if err = segnOutput.WriteString(si.Codec().(Codec).Name()); err == nil {
if err = segnOutput.WriteLong(siPerCommit.DelGen()); err == nil {
assert2(siPerCommit.DelCount() >= 0 && siPerCommit.DelCount() <= si.DocCount(),
"cannot write segment: invalid docCount segment=%v docCount=%v delCount=%v",
si.Name, si.DocCount(), siPerCommit.DelCount())
if err = segnOutput.WriteInt(int32(siPerCommit.DelCount())); err == nil {
if err = segnOutput.WriteLong(siPerCommit.FieldInfosGen()); err == nil {
if err = segnOutput.WriteLong(siPerCommit.DocValuesGen()); err == nil {
if err = segnOutput.WriteStringSet(siPerCommit.FieldInfosFiles()); err == nil {
dvUpdatesFiles := siPerCommit.DocValuesUpdatesFiles()
if err = segnOutput.WriteInt(int32(len(dvUpdatesFiles))); err == nil {
for k, v := range dvUpdatesFiles {
if err = segnOutput.WriteInt(int32(k)); err != nil {
break
}
if err = segnOutput.WriteStringSet(v); err != nil {
break
}
}
}
}
}
}
}
}
}
}
if err != nil {
return
}
assert(si.Dir == directory)
// If this segment is pre-4.x, perform a one-time "upgrade" to
// write the .si file for it:
if version := si.Version(); len(version) == 0 || !version.OnOrAfter(util.VERSION_4_0) {
panic("not implemented yet")
}
}
if err = segnOutput.WriteStringStringMap(sis.userData); err != nil {
return
}
sis.pendingSegnOutput = segnOutput
success = true
return nil
}
func (sis *SegmentInfos) write(directory store.Directory) error {
segmentsFilename := sis.nextSegmentFilename()
// Always advance the generation on write:
if sis.generation == -1 {
sis.generation = 1
} else {
sis.generation++
}
var segnOutput *store.ChecksumIndexOutput
var success = false
var upgradedSIFiles = make(map[string]bool)
defer func() {
if !success {
// We hit an error above; try to close the file but suppress
// any errors
util.CloseWhileSuppressingError(segnOutput)
for filename, _ := range upgradedSIFiles {
directory.DeleteFile(filename) // ignore error
}
// Try not to leave a truncated segments_N fle in the index:
directory.DeleteFile(segmentsFilename) // ignore error
}
}()
out, err := directory.CreateOutput(segmentsFilename, store.IO_CONTEXT_DEFAULT)
if err != nil {
return err
}
segnOutput = store.NewChecksumIndexOutput(out)
err = codec.WriteHeader(segnOutput, "segments", VERSION_40)
if err != nil {
return err
}
err = segnOutput.WriteLong(sis.version)
if err == nil {
err = segnOutput.WriteInt(int32(sis.counter))
if err == nil {
err = segnOutput.WriteInt(int32(len(sis.Segments)))
}
}
if err != nil {
return err
}
for _, siPerCommit := range sis.Segments {
si := siPerCommit.info
err = segnOutput.WriteString(si.Name)
if err == nil {
err = segnOutput.WriteString(si.Codec().(Codec).Name())
if err == nil {
err = segnOutput.WriteLong(siPerCommit.delGen)
if err == nil {
err = segnOutput.WriteInt(int32(siPerCommit.delCount))
}
}
}
if err != nil {
return err
}
assert(si.Dir == directory)
assert(siPerCommit.delCount <= si.DocCount())
// If this segment is pre-4.x, perform a one-time "upgrade" to
// write the .si file for it:
if version := si.Version(); version == "" || versionLess(version, "4.0") {
panic("not implemented yet")
}
}
err = segnOutput.WriteStringStringMap(sis.userData)
if err != nil {
return err
}
sis.pendingSegnOutput = segnOutput
success = true
return nil
}