func (sis SegmentInfos) SegmentsFileName() string { return util.FileNameFromGeneration(util.SEGMENTS, "", sis.lastGeneration) }
// TODO support IndexCommit func (fsf *FindSegmentsFile) run() (obj interface{}, err error) { log.Print("Finding segments file...") // if commit != nil { // if fsf.directory != commit.Directory { // return nil, errors.New("the specified commit does not match the specified Directory") // } // return fsf.doBody(commit.SegmentsFileName) // } lastGen := int64(-1) gen := int64(0) genLookaheadCount := 0 var exc error retryCount := 0 useFirstMethod := true // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means a commit was in process and has finished, in // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. // We have three methods for determining the current // generation. We try the first two in parallel (when // useFirstMethod is true), and fall back to the third // when necessary. for { log.Print("Trying...") if useFirstMethod { log.Print("Trying first method...") // List the directory and use the highest // segments_N file. This method works well as long // as there is no stale caching on the directory // contents (NOTE: NFS clients often have such stale // caching): genA := int64(-1) files, err := fsf.directory.ListAll() if err != nil { return nil, err } if files != nil { genA = LastCommitGeneration(files) } // TODO support info stream // if fsf.infoStream != nil { // message("directory listing genA=" + genA) // } log.Printf("directory listing genA=%v", genA) // Also open segments.gen and read its // contents. Then we take the larger of the two // gens. This way, if either approach is hitting // a stale cache (NFS) we have a better chance of // getting the right generation. genB := int64(-1) genInput, err := fsf.directory.OpenInput(INDEX_FILENAME_SEGMENTS_GEN, store.IO_CONTEXT_READ) if err != nil { // if fsf.infoStream != nil { log.Printf("segments.gen open: %v", err) // } } else { defer genInput.Close() log.Print("Reading segments info...") version, err := genInput.ReadInt() if err != nil { return nil, err } log.Printf("Version: %v", version) if version == FORMAT_SEGMENTS_GEN_CURRENT { log.Print("Version is current.") gen0, err := genInput.ReadLong() if err != nil { return nil, err } gen1, err := genInput.ReadLong() if err != nil { return nil, err } else { // if fsf.infoStream != nil { log.Printf("fallback check: %v; %v", gen0, gen1) // } if gen0 == gen1 { // The file is consistent. genB = gen0 } } } else { return nil, codec.NewIndexFormatTooNewError(genInput, version, FORMAT_SEGMENTS_GEN_CURRENT, FORMAT_SEGMENTS_GEN_CURRENT) } } // if fsf.infoStream != nil { log.Printf("%v check: genB=%v", INDEX_FILENAME_SEGMENTS_GEN, genB) // } // Pick the larger of the two gen's: gen = genA if genB > gen { gen = genB } if gen == -1 { // Neither approach found a generation return nil, errors.New(fmt.Sprintf("no segments* file found in %v: files: %#v", fsf.directory, files)) } } if useFirstMethod && lastGen == gen && retryCount >= 2 { // Give up on first method -- this is 3rd cycle on // listing directory and checking gen file to // attempt to locate the segments file. useFirstMethod = false } // Second method: since both directory cache and // file contents cache seem to be stale, just // advance the generation. if !useFirstMethod { if genLookaheadCount < fsf.defaultGenLookaheadCount { gen++ genLookaheadCount++ // if fsf.infoStream != nil { log.Printf("look ahead increment gen to %v", gen) // } } else { // All attempts have failed -- throw first exc: return nil, exc } } else if lastGen == gen { // This means we're about to try the same // segments_N last tried. retryCount++ } else { // Segment file has advanced since our last loop // (we made "progress"), so reset retryCount: retryCount = 0 } lastGen = gen segmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen) log.Printf("SegmentFileName: %v", segmentFileName) v, err := fsf.doBody(segmentFileName) if err != nil { // Save the original root cause: if exc == nil { exc = err } // if fsf.infoStream != nil { log.Printf("primary Exception on '%v': %v; will retry: retryCount = %v; gen = %v", segmentFileName, err, retryCount, gen) // } if gen > 1 && useFirstMethod && retryCount == 1 { // This is our second time trying this same segments // file (because retryCount is 1), and, there is // possibly a segments_(N-1) (because gen > 1). // So, check if the segments_(N-1) exists and // try it if so: prevSegmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen-1) if prevExists := fsf.directory.FileExists(prevSegmentFileName); prevExists { // if fsf.infoStream != nil { log.Printf("fallback to prior segment file '%v'", prevSegmentFileName) // } v, err = fsf.doBody(prevSegmentFileName) if err != nil { // if fsf.infoStream != nil { log.Printf("secondary Exception on '%v': %v; will retry", prevSegmentFileName, err) //} } else { // if fsf.infoStream != nil { log.Printf("success on fallback %v", prevSegmentFileName) //} return v, nil } } } } else { // if fsf.infoStream != nil { log.Printf("success on %v", segmentFileName) // } return v, nil } } }