/* Get the next segments_N filename that will be written. */
func (sis *SegmentInfos) nextSegmentFilename() string {
var nextGeneration int64
if sis.generation == -1 {
nextGeneration = 1
} else {
nextGeneration = sis.generation + 1
}
return util.FileNameFromGeneration(util.SEGMENTS, "", nextGeneration)
}
func (format *Lucene40LiveDocsFormat) WriteLiveDocs(bits util.MutableBits,
dir store.Directory, info *SegmentCommitInfo, newDelCount int,
ctx store.IOContext) error {
filename := util.FileNameFromGeneration(info.Info.Name, DELETES_EXTENSION, info.NextDelGen())
liveDocs := bits.(*BitVector)
assert(liveDocs.Count() == info.Info.DocCount()-info.DelCount()-newDelCount)
assert(liveDocs.Length() == info.Info.DocCount())
return liveDocs.Write(dir, filename, ctx)
}
func (sis *SegmentInfos) rollbackCommit(dir store.Directory) {
if sis.pendingSegnOutput != nil {
// Suppress so we keep throwing the original error in our caller
util.CloseWhileSuppressingError(sis.pendingSegnOutput)
sis.pendingSegnOutput = nil
// Must carefully compute filename from "generation" since
// lastGeneration isn't incremented:
segmentFilename := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)
// Suppress so we keep throwing the original error in our caller
util.DeleteFilesIgnoringErrors(dir, segmentFilename)
}
}
func (sis *SegmentInfos) finishCommit(dir store.Directory) (fileName string, err error) {
assert(dir != nil)
assert2(sis.pendingSegnOutput != nil, "prepareCommit was not called")
if err = func() error {
var success = false
defer func() {
if !success {
// Closes pendingSegnOutput & delets partial segments_N:
sis.rollbackCommit(dir)
} else {
err := func() error {
var success = false
defer func() {
if !success {
// Closes pendingSegnOutput & delets partial segments_N:
sis.rollbackCommit(dir)
} else {
sis.pendingSegnOutput = nil
}
}()
err := sis.pendingSegnOutput.Close()
success = err == nil
return err
}()
assertn(err == nil, "%v", err) // no shadow
}
}()
if err := codec.WriteFooter(sis.pendingSegnOutput); err != nil {
return err
}
success = true
return nil
}(); err != nil {
return
}
// NOTE: if we crash here, we have left a segments_N file in the
// directory in a possibly corrupt state (if some bytes made it to
// stable storage and others didn't). But, the segments_N file
// includes checksum at the end, which should catch this case. So
// when a reader tries to read it, it will return an index corrupt
// error, which should cause the retry logic in SegmentInfos to
// kick in and load the last good (previous) segments_N-1 file.
fileName = util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)
if err = func() error {
var success = false
defer func() {
if !success {
dir.DeleteFile(fileName)
// suppress error so we keep returning the original error
}
}()
err := dir.Sync([]string{fileName})
success = err == nil
return err
}(); err != nil {
return
}
sis.lastGeneration = sis.generation
writeSegmentsGen(dir, sis.generation)
return
}
func (sis *SegmentInfos) SegmentsFileName() string {
return util.FileNameFromGeneration(util.SEGMENTS, "", sis.lastGeneration)
}
// TODO support IndexCommit
func (fsf *FindSegmentsFile) run(commit IndexCommit) (interface{}, error) {
// fmt.Println("Finding segments file...")
if commit != nil {
if fsf.directory != commit.Directory() {
return nil, errors.New("the specified commit does not match the specified Directory")
}
return fsf.doBody(commit.SegmentsFileName())
}
lastGen := int64(-1)
gen := int64(0)
genLookaheadCount := 0
var exc error
retryCount := 0
useFirstMethod := true
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try the first two in parallel (when
// useFirstMethod is true), and fall back to the third
// when necessary.
for {
// fmt.Println("Trying...")
if useFirstMethod {
// fmt.Println("Trying first method...")
// List the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents (NOTE: NFS clients often have such stale
// caching):
genA := int64(-1)
files, err := fsf.directory.ListAll()
if err != nil {
return nil, err
}
if files != nil {
genA = LastCommitGeneration(files)
}
// message("directory listing genA=%v", genA)
// Also open segments.gen and read its
// contents. Then we take the larger of the two
// gens. This way, if either approach is hitting
// a stale cache (NFS) we have a better chance of
// getting the right generation.
genB := int64(-1)
genInput, err := fsf.directory.OpenChecksumInput(INDEX_FILENAME_SEGMENTS_GEN, store.IO_CONTEXT_READ)
if err != nil {
message("segments.gen open: %v", err)
} else {
defer genInput.Close()
// fmt.Println("Reading segments info...")
var version int32
if version, err = genInput.ReadInt(); err != nil {
return nil, err
}
// fmt.Printf("Version: %v\n", version)
if version == FORMAT_SEGMENTS_GEN_47 || version == FORMAT_SEGMENTS_GEN_CURRENT {
// fmt.Println("Version is current.")
var gen0, gen1 int64
if gen0, err = genInput.ReadLong(); err != nil {
return nil, err
}
if gen1, err = genInput.ReadLong(); err != nil {
return nil, err
}
message("fallback check: %v; %v", gen0, gen1)
if version == FORMAT_SEGMENTS_GEN_CHECKSUM {
if _, err = codec.CheckFooter(genInput); err != nil {
return nil, err
}
} else {
if err = codec.CheckEOF(genInput); err != nil {
return nil, err
}
}
if gen0 == gen1 {
// The file is consistent.
genB = gen0
}
} else {
return nil, codec.NewIndexFormatTooNewError(genInput, version,
FORMAT_SEGMENTS_GEN_CURRENT, FORMAT_SEGMENTS_GEN_CURRENT)
}
}
message("%v check: genB=%v", INDEX_FILENAME_SEGMENTS_GEN, genB)
// Pick the larger of the two gen's:
gen = genA
if genB > gen {
gen = genB
}
if gen == -1 {
// Neither approach found a generation
return nil, errors.New(fmt.Sprintf("no segments* file found in %v: files: %#v", fsf.directory, files))
}
}
if useFirstMethod && lastGen == gen && retryCount >= 2 {
// Give up on first method -- this is 3rd cycle on
// listing directory and checking gen file to
// attempt to locate the segments file.
useFirstMethod = false
}
// Second method: since both directory cache and
// file contents cache seem to be stale, just
// advance the generation.
if !useFirstMethod {
if genLookaheadCount < fsf.defaultGenLookaheadCount {
gen++
genLookaheadCount++
message("look ahead increment gen to %v", gen)
} else {
// All attempts have failed -- throw first exc:
return nil, exc
}
} else if lastGen == gen {
// This means we're about to try the same
// segments_N last tried.
retryCount++
} else {
// Segment file has advanced since our last loop
// (we made "progress"), so reset retryCount:
retryCount = 0
}
lastGen = gen
segmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen)
// fmt.Printf("SegmentFileName: %v\n", segmentFileName)
var v interface{}
var err error
if v, err = fsf.doBody(segmentFileName); err == nil {
message("success on %v", segmentFileName)
return v, nil
}
// Save the original root cause:
if exc == nil {
exc = err
}
message("primary Exception on '%v': %v; will retry: retryCount = %v; gen = %v",
segmentFileName, err, retryCount, gen)
if gen > 1 && useFirstMethod && retryCount == 1 {
// This is our second time trying this same segments
// file (because retryCount is 1), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
prevSegmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen-1)
if prevExists := fsf.directory.FileExists(prevSegmentFileName); prevExists {
message("fallback to prior segment file '%v'", prevSegmentFileName)
if v, err = fsf.doBody(prevSegmentFileName); err != nil {
message("secondary Exception on '%v': %v; will retry", prevSegmentFileName, err)
} else {
message("success on fallback %v", prevSegmentFileName)
return v, nil
}
}
}
}
}
func (format *Lucene40LiveDocsFormat) Files(info *SegmentCommitInfo) []string {
if info.HasDeletions() {
return []string{util.FileNameFromGeneration(info.Info.Name, DELETES_EXTENSION, info.DelGen())}
}
return []string{}
}