func DetectDataDir(dirpath string) (DataDirVersion, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
if os.IsNotExist(err) {
err = nil
}
// Error reading the directory
return DataDirUnknown, err
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.Contains("member") {
ver, err := DetectDataDir(path.Join(dirpath, "member"))
if ver == DataDir2_0 {
return DataDir2_0_1, nil
}
return ver, err
}
if nameSet.ContainsAll([]string{"snap", "wal"}) {
// .../wal cannot be empty to exist.
walnames, err := fileutil.ReadDir(path.Join(dirpath, "wal"))
if err == nil && len(walnames) > 0 {
return DataDir2_0, nil
}
}
if nameSet.ContainsAll([]string{"proxy"}) {
return DataDir2_0Proxy, nil
}
return DataDirUnknown, nil
}
// identifyDataDirOrDie returns the type of the data dir.
// Dies if the datadir is invalid.
func identifyDataDirOrDie(dir string) dirType {
names, err := fileutil.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return dirEmpty
}
plog.Fatalf("error listing data dir: %s", dir)
}
var m, p bool
for _, name := range names {
switch dirType(name) {
case dirMember:
m = true
case dirProxy:
p = true
default:
plog.Warningf("found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)", name, dir)
}
}
if m && p {
plog.Fatal("invalid datadir. Both member and proxy directories exist.")
}
if m {
return dirMember
}
if p {
return dirProxy
}
return dirEmpty
}
func makeMemberDir(dir string) error {
membdir := path.Join(dir, "member")
_, err := os.Stat(membdir)
switch {
case err == nil:
return nil
case !os.IsNotExist(err):
return err
}
if err := os.MkdirAll(membdir, 0700); err != nil {
return err
}
v1Files := types.NewUnsafeSet("conf", "log", "snapshot")
v2Files := types.NewUnsafeSet("wal", "snap")
names, err := fileutil.ReadDir(dir)
if err != nil {
return err
}
for _, name := range names {
switch {
case v1Files.Contains(name):
// Link it to the subdir and keep the v1 file at the original
// location, so v0.4 etcd can still bootstrap if the upgrade
// failed.
if err := os.Symlink(path.Join(dir, name), path.Join(membdir, name)); err != nil {
return err
}
case v2Files.Contains(name):
if err := os.Rename(path.Join(dir, name), path.Join(membdir, name)); err != nil {
return err
}
}
}
return nil
}
func Exist(dirpath string) bool {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return false
}
return len(names) != 0
}
func migrateSnapshots(legacySnapDir, snapDir string) error {
// use temporary snaphot directory so initialization appears atomic
tmpdirpath := filepath.Clean(snapDir) + ".tmp"
if fileutil.Exist(tmpdirpath) {
if err := os.RemoveAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not remove temporary snapshot directory")
}
}
if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not create temporary snapshot directory")
}
snapshotNames, err := fileutil.ReadDir(legacySnapDir)
if err != nil {
return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir)
}
for _, fname := range snapshotNames {
err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname))
if err != nil {
return errors.Wrap(err, "error linking snapshot file")
}
}
if err := os.Rename(tmpdirpath, snapDir); err != nil {
return err
}
return nil
}
func DetectVersion(dirpath string) (WalVersion, error) {
if _, err := os.Stat(dirpath); os.IsNotExist(err) {
return WALNotExist, nil
}
names, err := fileutil.ReadDir(dirpath)
if err != nil {
// Error reading the directory
return WALNotExist, err
}
if len(names) == 0 {
// Empty WAL directory
return WALNotExist, nil
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.ContainsAll([]string{"snap", "wal"}) {
// .../wal cannot be empty to exist.
if Exist(path.Join(dirpath, "wal")) {
return WALv0_5, nil
}
}
if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
return WALv0_4, nil
}
return WALUnknown, nil
}
func checkVersion(dataDir string) (version, error) {
names, err := fileutil.ReadDir(dataDir)
if err != nil {
if os.IsNotExist(err) {
err = nil
}
return empty, err
}
if len(names) == 0 {
return empty, nil
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.ContainsAll([]string{"member"}) {
return v2_0, nil
}
if nameSet.ContainsAll([]string{"proxy"}) {
return v2_0Proxy, nil
}
if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
return v0_4, nil
}
if nameSet.ContainsAll([]string{"standby_info"}) {
return v0_4, nil
}
return unknown, fmt.Errorf("failed to check version")
}
func migrateWALs(legacyWALDir, walDir string) error {
// keep temporary wal directory so WAL initialization appears atomic
tmpdirpath := filepath.Clean(walDir) + ".tmp"
if fileutil.Exist(tmpdirpath) {
if err := os.RemoveAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not remove temporary WAL directory")
}
}
if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not create temporary WAL directory")
}
walNames, err := fileutil.ReadDir(legacyWALDir)
if err != nil {
return errors.Wrapf(err, "could not list WAL directory %s", legacyWALDir)
}
for _, fname := range walNames {
_, err := copyFile(filepath.Join(legacyWALDir, fname), filepath.Join(tmpdirpath, fname), 0600)
if err != nil {
return errors.Wrap(err, "error copying WAL file")
}
}
if err := os.Rename(tmpdirpath, walDir); err != nil {
return err
}
return nil
}
func readWalNames(dirpath string) ([]string, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
wnames := checkWalNames(names)
if len(wnames) == 0 {
return nil, ErrFileNotFound
}
return wnames, nil
}
// openLast opens the last wal file for read and write.
func openLast(dirpath string) (*os.File, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
last := path.Join(dirpath, names[len(names)-1])
return os.OpenFile(last, os.O_RDWR, 0)
}
// OpenAtIndex opens the WAL at the given index.
// The index SHOULD have been previously committed to the WAL, or the following
// ReadAll will fail.
// The returned WAL is ready to read and the first record will be the given
// index. The WAL cannot be appended to before reading out all of its
// previous records.
func OpenAtIndex(dirpath string, index uint64) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
sort.Sort(sort.StringSlice(names))
nameIndex, ok := searchIndex(names, index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files for reading
rcs := make([]io.ReadCloser, 0)
for _, name := range names[nameIndex:] {
f, err := os.Open(path.Join(dirpath, name))
if err != nil {
return nil, err
}
rcs = append(rcs, f)
}
rc := MultiReadCloser(rcs...)
// open the lastest wal file for appending
seq, _, err := parseWalName(names[len(names)-1])
if err != nil {
rc.Close()
return nil, err
}
last := path.Join(dirpath, names[len(names)-1])
f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
rc.Close()
return nil, err
}
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
ri: index,
decoder: newDecoder(rc),
f: f,
seq: seq,
}
return w, nil
}
// getSnapFilePath returns the file path for the snapshot with given index.
// If the snapshot does not exist, it returns error.
func (ss *snapshotStore) getSnapFilePath(index uint64) (string, error) {
fns, err := fileutil.ReadDir(ss.dir)
if err != nil {
return "", err
}
wfn := fmt.Sprintf("%016x.db", index)
for _, fn := range fns {
if fn == wfn {
return path.Join(ss.dir, fn), nil
}
}
return "", fmt.Errorf("snapshot file doesn't exist")
}
// DBFilePath returns the file path for the snapshot of the database with
// given id. If the snapshot does not exist, it returns error.
func (s *Snapshotter) DBFilePath(id uint64) (string, error) {
fns, err := fileutil.ReadDir(s.dir)
if err != nil {
return "", err
}
wfn := fmt.Sprintf("%016x.snap.db", id)
for _, fn := range fns {
if fn == wfn {
return path.Join(s.dir, fn), nil
}
}
return "", fmt.Errorf("snap: snapshot file doesn't exist")
}
func DetectVersion(dirpath string) WalVersion {
names, err := fileutil.ReadDir(dirpath)
if err != nil || len(names) == 0 {
return WALNotExist
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.ContainsAll([]string{"snap", "wal"}) {
// .../wal cannot be empty to exist.
if Exist(path.Join(dirpath, "wal")) {
return WALv0_5
}
return WALNotExist
}
if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
return WALv0_4
}
return WALUnknown
}
func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
nameIndex, ok := searchIndex(names, snap.Index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files for reading
rcs := make([]io.ReadCloser, 0)
ls := make([]fileutil.Lock, 0)
for _, name := range names[nameIndex:] {
f, err := os.Open(path.Join(dirpath, name))
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
err = l.TryLock()
if err != nil {
if write {
return nil, err
}
}
rcs = append(rcs, f)
ls = append(ls, l)
}
rc := MultiReadCloser(rcs...)
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
start: snap,
decoder: newDecoder(rc),
locks: ls,
}
if write {
// open the lastest wal file for appending
seq, _, err := parseWalName(names[len(names)-1])
if err != nil {
rc.Close()
return nil, err
}
last := path.Join(dirpath, names[len(names)-1])
f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
rc.Close()
return nil, err
}
err = fileutil.Preallocate(f, segmentSizeBytes)
if err != nil {
rc.Close()
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return nil, err
}
w.f = f
w.seq = seq
}
return w, nil
}
func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
nameIndex, ok := searchIndex(names, snap.Index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files
rcs := make([]io.ReadCloser, 0)
rs := make([]io.Reader, 0)
ls := make([]*fileutil.LockedFile, 0)
for _, name := range names[nameIndex:] {
p := path.Join(dirpath, name)
if write {
l, err := fileutil.TryLockFile(p, os.O_RDWR, 0600)
if err != nil {
closeAll(rcs...)
return nil, err
}
ls = append(ls, l)
rcs = append(rcs, l)
} else {
rf, err := os.OpenFile(p, os.O_RDONLY, 0600)
if err != nil {
closeAll(rcs...)
return nil, err
}
ls = append(ls, nil)
rcs = append(rcs, rf)
}
rs = append(rs, rcs[len(rcs)-1])
}
closer := func() error { return closeAll(rcs...) }
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
start: snap,
decoder: newDecoder(rs...),
readClose: closer,
locks: ls,
}
if write {
// write reuses the file descriptors from read; don't close so
// WAL can append without dropping the file lock
w.readClose = nil
if _, _, err := parseWalName(path.Base(w.tail().Name())); err != nil {
closer()
return nil, err
}
// don't resize file for preallocation in case tail is corrupted
if err := fileutil.Preallocate(w.tail().File, segmentSizeBytes, false); err != nil {
closer()
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return nil, err
}
w.fp = newFilePipeline(w.dir, segmentSizeBytes)
}
return w, nil
}
func openAtIndex(dirpath string, index uint64, all bool) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
sort.Sort(sort.StringSlice(names))
nameIndex, ok := searchIndex(names, index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files for reading
rcs := make([]io.ReadCloser, 0)
ls := make([]fileutil.Lock, 0)
for _, name := range names[nameIndex:] {
f, err := os.Open(path.Join(dirpath, name))
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
err = l.TryLock()
if err != nil {
if all {
return nil, err
} else {
log.Printf("wal: opened all the files until %s, since it is still in use by an etcd server", name)
break
}
}
rcs = append(rcs, f)
ls = append(ls, l)
}
rc := MultiReadCloser(rcs...)
// open the lastest wal file for appending
seq, _, err := parseWalName(names[len(names)-1])
if err != nil {
rc.Close()
return nil, err
}
last := path.Join(dirpath, names[len(names)-1])
f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
rc.Close()
return nil, err
}
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
ri: index,
decoder: newDecoder(rc),
f: f,
seq: seq,
locks: ls,
}
return w, nil
}