// writeFakeRaftData writes the given snapshot and some generated WAL data to given "snap" and "wal" directories
func writeFakeRaftData(t *testing.T, stateDir string, snapshot *raftpb.Snapshot, wf storage.WALFactory, sf storage.SnapFactory) {
snapDir := filepath.Join(stateDir, "raft", "snap-v3-encrypted")
walDir := filepath.Join(stateDir, "raft", "wal-v3-encrypted")
require.NoError(t, os.MkdirAll(snapDir, 0755))
wsn := walpb.Snapshot{}
if snapshot != nil {
require.NoError(t, sf.New(snapDir).SaveSnap(*snapshot))
wsn.Index = snapshot.Metadata.Index
wsn.Term = snapshot.Metadata.Term
}
var entries []raftpb.Entry
for i := wsn.Index + 1; i < wsn.Index+6; i++ {
entries = append(entries, raftpb.Entry{
Term: wsn.Term + 1,
Index: i,
Data: []byte(fmt.Sprintf("v3Entry %d", i)),
})
}
walWriter, err := wf.Create(walDir, []byte("v3metadata"))
require.NoError(t, err)
require.NoError(t, walWriter.SaveSnapshot(wsn))
require.NoError(t, walWriter.Save(raftpb.HardState{}, entries))
require.NoError(t, walWriter.Close())
}
func decryptRaftData(swarmdir, outdir, unlockKey string) error {
krw, err := getKRW(swarmdir, unlockKey)
if err != nil {
return err
}
deks, err := getDEKData(krw)
if err != nil {
return err
}
_, d := encryption.Defaults(deks.CurrentDEK)
if deks.PendingDEK == nil {
_, d2 := encryption.Defaults(deks.PendingDEK)
d = storage.MultiDecrypter{d, d2}
}
snapDir := filepath.Join(outdir, "snap-decrypted")
if err := moveDirAside(snapDir); err != nil {
return err
}
if err := storage.MigrateSnapshot(
filepath.Join(swarmdir, "raft", "snap-v3-encrypted"), snapDir,
storage.NewSnapFactory(encryption.NoopCrypter, d), storage.OriginalSnap); err != nil {
return err
}
var walsnap walpb.Snapshot
snap, err := storage.OriginalSnap.New(snapDir).Load()
if err != nil && !os.IsNotExist(err) {
return err
}
if snap != nil {
walsnap.Index = snap.Metadata.Index
walsnap.Term = snap.Metadata.Term
}
walDir := filepath.Join(outdir, "wal-decrypted")
if err := moveDirAside(walDir); err != nil {
return err
}
return storage.MigrateWALs(context.Background(),
filepath.Join(swarmdir, "raft", "wal-v3-encrypted"), walDir,
storage.NewWALFactory(encryption.NoopCrypter, d), storage.OriginalWAL, walsnap)
}
func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) {
var (
walsnap walpb.Snapshot
metadata []byte
st raftpb.HardState
ents []raftpb.Entry
)
if snapshot != nil {
walsnap.Index = snapshot.Metadata.Index
walsnap.Term = snapshot.Metadata.Term
}
repaired := false
for {
if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil {
return fmt.Errorf("open WAL error: %v", err)
}
if metadata, st, ents, err = n.wal.ReadAll(); err != nil {
if err := n.wal.Close(); err != nil {
return err
}
// we can only repair ErrUnexpectedEOF and we never repair twice.
if repaired || err != io.ErrUnexpectedEOF {
return fmt.Errorf("read WAL error (%v) and cannot be repaired", err)
}
if !wal.Repair(n.walDir()) {
return fmt.Errorf("WAL error (%v) cannot be repaired", err)
}
log.G(ctx).Infof("repaired WAL error (%v)", err)
repaired = true
continue
}
break
}
defer func() {
if err != nil {
if walErr := n.wal.Close(); walErr != nil {
n.Config.Logger.Errorf("error closing raft WAL: %v", walErr)
}
}
}()
var raftNode api.RaftMember
if err := raftNode.Unmarshal(metadata); err != nil {
return fmt.Errorf("error unmarshalling WAL metadata: %v", err)
}
n.Config.ID = raftNode.RaftID
// All members that are no longer part of the cluster must be added to
// the removed list right away, so that we don't try to connect to them
// before processing the configuration change entries, which could make
// us get stuck.
for _, ent := range ents {
if ent.Index <= st.Commit && ent.Type == raftpb.EntryConfChange {
var cc raftpb.ConfChange
if err := cc.Unmarshal(ent.Data); err != nil {
return fmt.Errorf("error unmarshalling config change: %v", err)
}
if cc.Type == raftpb.ConfChangeRemoveNode {
n.cluster.RemoveMember(cc.NodeID)
}
}
}
if forceNewCluster {
// discard the previously uncommitted entries
for i, ent := range ents {
if ent.Index > st.Commit {
log.G(context.Background()).Infof("discarding %d uncommitted WAL entries ", len(ents)-i)
ents = ents[:i]
break
}
}
// force append the configuration change entries
toAppEnts := createConfigChangeEnts(getIDs(snapshot, ents), uint64(n.Config.ID), st.Term, st.Commit)
// All members that are being removed as part of the
// force-new-cluster process must be added to the
// removed list right away, so that we don't try to
// connect to them before processing the configuration
// change entries, which could make us get stuck.
for _, ccEnt := range toAppEnts {
if ccEnt.Type == raftpb.EntryConfChange {
var cc raftpb.ConfChange
if err := cc.Unmarshal(ccEnt.Data); err != nil {
return fmt.Errorf("error unmarshalling force-new-cluster config change: %v", err)
}
if cc.Type == raftpb.ConfChangeRemoveNode {
n.cluster.RemoveMember(cc.NodeID)
}
}
}
ents = append(ents, toAppEnts...)
// force commit newly appended entries
err := n.wal.Save(st, toAppEnts)
if err != nil {
log.G(context.Background()).Fatalf("%v", err)
}
if len(toAppEnts) != 0 {
st.Commit = toAppEnts[len(toAppEnts)-1].Index
}
}
if snapshot != nil {
if err := n.raftStore.ApplySnapshot(*snapshot); err != nil {
return err
}
}
if err := n.raftStore.SetHardState(st); err != nil {
return err
}
if err := n.raftStore.Append(ents); err != nil {
return err
}
return nil
}
func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) {
var (
walsnap walpb.Snapshot
metadata []byte
st raftpb.HardState
ents []raftpb.Entry
)
if snapshot != nil {
walsnap.Index = snapshot.Metadata.Index
walsnap.Term = snapshot.Metadata.Term
}
repaired := false
for {
if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil {
return fmt.Errorf("open wal error: %v", err)
}
if metadata, st, ents, err = n.wal.ReadAll(); err != nil {
if err := n.wal.Close(); err != nil {
return err
}
// we can only repair ErrUnexpectedEOF and we never repair twice.
if repaired || err != io.ErrUnexpectedEOF {
return fmt.Errorf("read wal error (%v) and cannot be repaired", err)
}
if !wal.Repair(n.walDir()) {
return fmt.Errorf("WAL error (%v) cannot be repaired", err)
}
log.G(ctx).Infof("repaired WAL error (%v)", err)
repaired = true
continue
}
break
}
defer func() {
if err != nil {
if walErr := n.wal.Close(); walErr != nil {
n.Config.Logger.Errorf("error closing raft WAL: %v", walErr)
}
}
}()
var raftNode api.RaftMember
if err := raftNode.Unmarshal(metadata); err != nil {
return fmt.Errorf("error unmarshalling wal metadata: %v", err)
}
n.Config.ID = raftNode.RaftID
if forceNewCluster {
// discard the previously uncommitted entries
for i, ent := range ents {
if ent.Index > st.Commit {
log.G(context.Background()).Infof("discarding %d uncommitted WAL entries ", len(ents)-i)
ents = ents[:i]
break
}
}
// force append the configuration change entries
toAppEnts := createConfigChangeEnts(getIDs(snapshot, ents), uint64(n.Config.ID), st.Term, st.Commit)
ents = append(ents, toAppEnts...)
// force commit newly appended entries
err := n.wal.Save(st, toAppEnts)
if err != nil {
log.G(context.Background()).Fatalf("%v", err)
}
if len(toAppEnts) != 0 {
st.Commit = toAppEnts[len(toAppEnts)-1].Index
}
}
if snapshot != nil {
if err := n.raftStore.ApplySnapshot(*snapshot); err != nil {
return err
}
}
if err := n.raftStore.SetHardState(st); err != nil {
return err
}
if err := n.raftStore.Append(ents); err != nil {
return err
}
return nil
}
// BootstrapFromDisk creates a new snapshotter and wal, and also reads the latest snapshot and WALs from disk
func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncryptionKeys ...[]byte) (*raftpb.Snapshot, WALData, error) {
e.encoderMu.Lock()
defer e.encoderMu.Unlock()
walDir := e.walDir()
snapDir := e.snapDir()
encrypter, decrypter := encryption.Defaults(e.EncryptionKey)
if oldEncryptionKeys != nil {
decrypters := []encryption.Decrypter{decrypter}
for _, key := range oldEncryptionKeys {
_, d := encryption.Defaults(key)
decrypters = append(decrypters, d)
}
decrypter = MultiDecrypter(decrypters)
}
snapFactory := NewSnapFactory(encrypter, decrypter)
if !fileutil.Exist(snapDir) {
// If snapshots created by the etcd-v2 code exist, or by swarmkit development version,
// read the latest snapshot and write it encoded to the new path. The new path
// prevents etc-v2 creating snapshots that are visible to us, but not encoded and
// out of sync with our WALs, after a downgrade.
for _, dirs := range versionedWALSnapDirs[1:] {
legacySnapDir := filepath.Join(e.StateDir, dirs.snap)
if fileutil.Exist(legacySnapDir) {
if err := MigrateSnapshot(legacySnapDir, snapDir, OriginalSnap, snapFactory); err != nil {
return nil, WALData{}, err
}
break
}
}
}
// ensure the new directory exists
if err := os.MkdirAll(snapDir, 0700); err != nil {
return nil, WALData{}, errors.Wrap(err, "failed to create snapshot directory")
}
var (
snapshotter Snapshotter
walObj WAL
err error
)
// Create a snapshotter and load snapshot data
snapshotter = snapFactory.New(snapDir)
snapshot, err := snapshotter.Load()
if err != nil && err != snap.ErrNoSnapshot {
return nil, WALData{}, err
}
walFactory := NewWALFactory(encrypter, decrypter)
var walsnap walpb.Snapshot
if snapshot != nil {
walsnap.Index = snapshot.Metadata.Index
walsnap.Term = snapshot.Metadata.Term
}
if !wal.Exist(walDir) {
var walExists bool
// If wals created by the etcd-v2 wal code exist, read the latest ones based
// on this snapshot and encode them to wals in the new path to avoid adding
// backwards-incompatible entries to those files.
for _, dirs := range versionedWALSnapDirs[1:] {
legacyWALDir := filepath.Join(e.StateDir, dirs.wal)
if !wal.Exist(legacyWALDir) {
continue
}
if err = MigrateWALs(ctx, legacyWALDir, walDir, OriginalWAL, walFactory, walsnap); err != nil {
return nil, WALData{}, err
}
walExists = true
break
}
if !walExists {
return nil, WALData{}, ErrNoWAL
}
}
walObj, waldata, err := ReadRepairWAL(ctx, walDir, walsnap, walFactory)
if err != nil {
return nil, WALData{}, err
}
e.snapshotter = snapshotter
e.wal = walObj
return snapshot, waldata, nil
}