Exemple #1
0
/* Get the next segments_N filename that will be written. */
func (sis *SegmentInfos) nextSegmentFilename() string {
	var nextGeneration int64
	if sis.generation == -1 {
		nextGeneration = 1
	} else {
		nextGeneration = sis.generation + 1
	}
	return util.FileNameFromGeneration(util.SEGMENTS, "", nextGeneration)
}
Exemple #2
0
func (format *Lucene40LiveDocsFormat) WriteLiveDocs(bits util.MutableBits,
	dir store.Directory, info *SegmentCommitInfo, newDelCount int,
	ctx store.IOContext) error {

	filename := util.FileNameFromGeneration(info.Info.Name, DELETES_EXTENSION, info.NextDelGen())
	liveDocs := bits.(*BitVector)
	assert(liveDocs.Count() == info.Info.DocCount()-info.DelCount()-newDelCount)
	assert(liveDocs.Length() == info.Info.DocCount())
	return liveDocs.Write(dir, filename, ctx)
}
Exemple #3
0
func (sis *SegmentInfos) rollbackCommit(dir store.Directory) {
	if sis.pendingSegnOutput != nil {
		// Suppress so we keep throwing the original error in our caller
		util.CloseWhileSuppressingError(sis.pendingSegnOutput)
		sis.pendingSegnOutput = nil

		// Must carefully compute filename from "generation" since
		// lastGeneration isn't incremented:
		segmentFilename := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)

		// Suppress so we keep throwing the original error in our caller
		util.DeleteFilesIgnoringErrors(dir, segmentFilename)
	}
}
Exemple #4
0
func (sis *SegmentInfos) finishCommit(dir store.Directory) (fileName string, err error) {
	assert(dir != nil)
	assert2(sis.pendingSegnOutput != nil, "prepareCommit was not called")
	if err = func() error {
		var success = false
		defer func() {
			if !success {
				// Closes pendingSegnOutput & delets partial segments_N:
				sis.rollbackCommit(dir)
			} else {
				err := func() error {
					var success = false
					defer func() {
						if !success {
							// Closes pendingSegnOutput & delets partial segments_N:
							sis.rollbackCommit(dir)
						} else {
							sis.pendingSegnOutput = nil
						}
					}()

					err := sis.pendingSegnOutput.Close()
					success = err == nil
					return err
				}()
				assertn(err == nil, "%v", err) // no shadow
			}
		}()

		if err := codec.WriteFooter(sis.pendingSegnOutput); err != nil {
			return err
		}
		success = true
		return nil
	}(); err != nil {
		return
	}

	// NOTE: if we crash here, we have left a segments_N file in the
	// directory in a possibly corrupt state (if some bytes made it to
	// stable storage and others didn't). But, the segments_N file
	// includes checksum at the end, which should catch this case. So
	// when a reader tries to read it, it will return an index corrupt
	// error, which should cause the retry logic in SegmentInfos to
	// kick in and load the last good (previous) segments_N-1 file.

	fileName = util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", sis.generation)
	if err = func() error {
		var success = false
		defer func() {
			if !success {
				dir.DeleteFile(fileName)
				// suppress error so we keep returning the original error
			}
		}()

		err := dir.Sync([]string{fileName})
		success = err == nil
		return err
	}(); err != nil {
		return
	}

	sis.lastGeneration = sis.generation
	writeSegmentsGen(dir, sis.generation)
	return
}
Exemple #5
0
func (sis *SegmentInfos) SegmentsFileName() string {
	return util.FileNameFromGeneration(util.SEGMENTS, "", sis.lastGeneration)
}
Exemple #6
0
// TODO support IndexCommit
func (fsf *FindSegmentsFile) run(commit IndexCommit) (interface{}, error) {
	// fmt.Println("Finding segments file...")
	if commit != nil {
		if fsf.directory != commit.Directory() {
			return nil, errors.New("the specified commit does not match the specified Directory")
		}
		return fsf.doBody(commit.SegmentsFileName())
	}

	lastGen := int64(-1)
	gen := int64(0)
	genLookaheadCount := 0
	var exc error
	retryCount := 0

	useFirstMethod := true

	// Loop until we succeed in calling doBody() without
	// hitting an IOException.  An IOException most likely
	// means a commit was in process and has finished, in
	// the time it took us to load the now-old infos files
	// (and segments files).  It's also possible it's a
	// true error (corrupt index).  To distinguish these,
	// on each retry we must see "forward progress" on
	// which generation we are trying to load.  If we
	// don't, then the original error is real and we throw
	// it.

	// We have three methods for determining the current
	// generation.  We try the first two in parallel (when
	// useFirstMethod is true), and fall back to the third
	// when necessary.

	for {
		// fmt.Println("Trying...")
		if useFirstMethod {
			// fmt.Println("Trying first method...")
			// List the directory and use the highest
			// segments_N file.  This method works well as long
			// as there is no stale caching on the directory
			// contents (NOTE: NFS clients often have such stale
			// caching):
			genA := int64(-1)

			files, err := fsf.directory.ListAll()
			if err != nil {
				return nil, err
			}
			if files != nil {
				genA = LastCommitGeneration(files)
			}

			// message("directory listing genA=%v", genA)

			// Also open segments.gen and read its
			// contents.  Then we take the larger of the two
			// gens.  This way, if either approach is hitting
			// a stale cache (NFS) we have a better chance of
			// getting the right generation.
			genB := int64(-1)
			genInput, err := fsf.directory.OpenChecksumInput(INDEX_FILENAME_SEGMENTS_GEN, store.IO_CONTEXT_READ)
			if err != nil {
				message("segments.gen open: %v", err)
			} else {
				defer genInput.Close()
				// fmt.Println("Reading segments info...")

				var version int32
				if version, err = genInput.ReadInt(); err != nil {
					return nil, err
				}
				// fmt.Printf("Version: %v\n", version)
				if version == FORMAT_SEGMENTS_GEN_47 || version == FORMAT_SEGMENTS_GEN_CURRENT {
					// fmt.Println("Version is current.")
					var gen0, gen1 int64
					if gen0, err = genInput.ReadLong(); err != nil {
						return nil, err
					}
					if gen1, err = genInput.ReadLong(); err != nil {
						return nil, err
					}
					message("fallback check: %v; %v", gen0, gen1)
					if version == FORMAT_SEGMENTS_GEN_CHECKSUM {
						if _, err = codec.CheckFooter(genInput); err != nil {
							return nil, err
						}
					} else {
						if err = codec.CheckEOF(genInput); err != nil {
							return nil, err
						}
					}
					if gen0 == gen1 {
						// The file is consistent.
						genB = gen0
					}
				} else {
					return nil, codec.NewIndexFormatTooNewError(genInput, version,
						FORMAT_SEGMENTS_GEN_CURRENT, FORMAT_SEGMENTS_GEN_CURRENT)
				}
			}

			message("%v check: genB=%v", INDEX_FILENAME_SEGMENTS_GEN, genB)

			// Pick the larger of the two gen's:
			gen = genA
			if genB > gen {
				gen = genB
			}

			if gen == -1 {
				// Neither approach found a generation
				return nil, errors.New(fmt.Sprintf("no segments* file found in %v: files: %#v", fsf.directory, files))
			}
		}

		if useFirstMethod && lastGen == gen && retryCount >= 2 {
			// Give up on first method -- this is 3rd cycle on
			// listing directory and checking gen file to
			// attempt to locate the segments file.
			useFirstMethod = false
		}

		// Second method: since both directory cache and
		// file contents cache seem to be stale, just
		// advance the generation.
		if !useFirstMethod {
			if genLookaheadCount < fsf.defaultGenLookaheadCount {
				gen++
				genLookaheadCount++
				message("look ahead increment gen to %v", gen)
			} else {
				// All attempts have failed -- throw first exc:
				return nil, exc
			}
		} else if lastGen == gen {
			// This means we're about to try the same
			// segments_N last tried.
			retryCount++
		} else {
			// Segment file has advanced since our last loop
			// (we made "progress"), so reset retryCount:
			retryCount = 0
		}

		lastGen = gen
		segmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen)
		// fmt.Printf("SegmentFileName: %v\n", segmentFileName)

		var v interface{}
		var err error
		if v, err = fsf.doBody(segmentFileName); err == nil {
			message("success on %v", segmentFileName)
			return v, nil
		}
		// Save the original root cause:
		if exc == nil {
			exc = err
		}

		message("primary Exception on '%v': %v; will retry: retryCount = %v; gen = %v",
			segmentFileName, err, retryCount, gen)

		if gen > 1 && useFirstMethod && retryCount == 1 {
			// This is our second time trying this same segments
			// file (because retryCount is 1), and, there is
			// possibly a segments_(N-1) (because gen > 1).
			// So, check if the segments_(N-1) exists and
			// try it if so:
			prevSegmentFileName := util.FileNameFromGeneration(INDEX_FILENAME_SEGMENTS, "", gen-1)

			if prevExists := fsf.directory.FileExists(prevSegmentFileName); prevExists {
				message("fallback to prior segment file '%v'", prevSegmentFileName)
				if v, err = fsf.doBody(prevSegmentFileName); err != nil {
					message("secondary Exception on '%v': %v; will retry", prevSegmentFileName, err)
				} else {
					message("success on fallback %v", prevSegmentFileName)
					return v, nil
				}
			}
		}
	}
}
Exemple #7
0
func (format *Lucene40LiveDocsFormat) Files(info *SegmentCommitInfo) []string {
	if info.HasDeletions() {
		return []string{util.FileNameFromGeneration(info.Info.Name, DELETES_EXTENSION, info.DelGen())}
	}
	return []string{}
}