func (n *Node) restoreFromSnapshot(data []byte, forceNewCluster bool) error {
var snapshot api.Snapshot
if err := snapshot.Unmarshal(data); err != nil {
return err
}
if snapshot.Version != api.Snapshot_V0 {
return fmt.Errorf("unrecognized snapshot version %d", snapshot.Version)
}
if err := n.memoryStore.Restore(&snapshot.Store); err != nil {
return err
}
n.cluster.Clear()
if !forceNewCluster {
for _, member := range snapshot.Membership.Members {
if err := n.registerNode(&api.RaftMember{RaftID: member.RaftID, NodeID: member.NodeID, Addr: member.Addr}); err != nil {
return err
}
}
}
for _, removedMember := range snapshot.Membership.Removed {
n.cluster.RemoveMember(removedMember)
}
return nil
}
func (n *Node) clusterSnapshot(data []byte) (api.ClusterSnapshot, error) {
var snapshot api.Snapshot
if err := snapshot.Unmarshal(data); err != nil {
return snapshot.Membership, err
}
if snapshot.Version != api.Snapshot_V0 {
return snapshot.Membership, fmt.Errorf("unrecognized snapshot version %d", snapshot.Version)
}
if err := n.memoryStore.Restore(&snapshot.Store); err != nil {
return snapshot.Membership, err
}
return snapshot.Membership, nil
}
func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
snapshot := api.Snapshot{Version: api.Snapshot_V0}
for _, member := range n.cluster.Members() {
snapshot.Membership.Members = append(snapshot.Membership.Members,
&api.RaftMember{
NodeID: member.NodeID,
RaftID: member.RaftID,
Addr: member.Addr,
})
}
snapshot.Membership.Removed = n.cluster.Removed()
viewStarted := make(chan struct{})
n.asyncTasks.Add(1)
n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot
go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) {
defer func() {
n.asyncTasks.Done()
n.snapshotInProgress <- snapshotMeta
}()
var err error
n.memoryStore.View(func(tx store.ReadTx) {
close(viewStarted)
var storeSnapshot *api.StoreSnapshot
storeSnapshot, err = n.memoryStore.Save(tx)
snapshot.Store = *storeSnapshot
})
if err != nil {
log.G(ctx).WithError(err).Error("failed to read snapshot from store")
return
}
d, err := snapshot.Marshal()
if err != nil {
log.G(ctx).WithError(err).Error("failed to marshal snapshot")
return
}
snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
if err == nil {
if err := n.raftLogger.SaveSnapshot(snap); err != nil {
log.G(ctx).WithError(err).Error("failed to save snapshot")
return
}
snapshotMeta = snap.Metadata
if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
if err != nil && err != raft.ErrCompacted {
log.G(ctx).WithError(err).Error("failed to compact snapshot")
}
}
} else if err != raft.ErrSnapOutOfDate {
log.G(ctx).WithError(err).Error("failed to create snapshot")
}
}(n.appliedIndex, n.snapshotMeta)
// Wait for the goroutine to establish a read transaction, to make
// sure it sees the state as of this moment.
<-viewStarted
}
func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
snapshot := api.Snapshot{Version: api.Snapshot_V0}
for _, member := range n.cluster.Members() {
snapshot.Membership.Members = append(snapshot.Membership.Members,
&api.RaftMember{
NodeID: member.NodeID,
RaftID: member.RaftID,
Addr: member.Addr,
})
}
snapshot.Membership.Removed = n.cluster.Removed()
// maybe start rotation
n.rotationQueued = false
var newEncryptionKeys *EncryptionKeys
if n.keyRotator.NeedsRotation() {
keys := n.keyRotator.GetKeys()
if keys.PendingDEK != nil {
n.raftLogger.RotateEncryptionKey(keys.PendingDEK)
newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK}
}
}
viewStarted := make(chan struct{})
n.asyncTasks.Add(1)
n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot
go func(appliedIndex, snapshotIndex uint64) {
defer func() {
n.asyncTasks.Done()
n.snapshotInProgress <- snapshotIndex
}()
var err error
n.memoryStore.View(func(tx store.ReadTx) {
close(viewStarted)
var storeSnapshot *api.StoreSnapshot
storeSnapshot, err = n.memoryStore.Save(tx)
snapshot.Store = *storeSnapshot
})
if err != nil {
log.G(ctx).WithError(err).Error("failed to read snapshot from store")
return
}
d, err := snapshot.Marshal()
if err != nil {
log.G(ctx).WithError(err).Error("failed to marshal snapshot")
return
}
snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
if err == nil {
if err := n.raftLogger.SaveSnapshot(snap); err != nil {
log.G(ctx).WithError(err).Error("failed to save snapshot")
return
}
snapshotIndex = appliedIndex
if newEncryptionKeys != nil {
// this means we tried to rotate - so finish the rotation
if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil {
log.G(ctx).WithError(err).Error(
"failed to update encryption keys after a rotation - will wait for the next snapshot")
}
}
if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
if err != nil && err != raft.ErrCompacted {
log.G(ctx).WithError(err).Error("failed to compact snapshot")
}
}
if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs")
}
} else if err != raft.ErrSnapOutOfDate {
log.G(ctx).WithError(err).Error("failed to create snapshot")
}
}(n.appliedIndex, n.snapshotIndex)
// Wait for the goroutine to establish a read transaction, to make
// sure it sees the state as of this moment.
<-viewStarted
}
