func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
snapshot := api.Snapshot{Version: api.Snapshot_V0}
for _, member := range n.cluster.Members() {
snapshot.Membership.Members = append(snapshot.Membership.Members,
&api.RaftMember{
NodeID: member.NodeID,
RaftID: member.RaftID,
Addr: member.Addr,
})
}
snapshot.Membership.Removed = n.cluster.Removed()
viewStarted := make(chan struct{})
n.asyncTasks.Add(1)
n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot
go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) {
defer func() {
n.asyncTasks.Done()
n.snapshotInProgress <- snapshotMeta
}()
var err error
n.memoryStore.View(func(tx store.ReadTx) {
close(viewStarted)
var storeSnapshot *api.StoreSnapshot
storeSnapshot, err = n.memoryStore.Save(tx)
snapshot.Store = *storeSnapshot
})
if err != nil {
log.G(ctx).WithError(err).Error("failed to read snapshot from store")
return
}
d, err := snapshot.Marshal()
if err != nil {
log.G(ctx).WithError(err).Error("failed to marshal snapshot")
return
}
snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
if err == nil {
if err := n.raftLogger.SaveSnapshot(snap); err != nil {
log.G(ctx).WithError(err).Error("failed to save snapshot")
return
}
snapshotMeta = snap.Metadata
if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
if err != nil && err != raft.ErrCompacted {
log.G(ctx).WithError(err).Error("failed to compact snapshot")
}
}
} else if err != raft.ErrSnapOutOfDate {
log.G(ctx).WithError(err).Error("failed to create snapshot")
}
}(n.appliedIndex, n.snapshotMeta)
// Wait for the goroutine to establish a read transaction, to make
// sure it sees the state as of this moment.
<-viewStarted
}
func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
snapshot := api.Snapshot{Version: api.Snapshot_V0}
for _, member := range n.cluster.Members() {
snapshot.Membership.Members = append(snapshot.Membership.Members,
&api.RaftMember{
NodeID: member.NodeID,
RaftID: member.RaftID,
Addr: member.Addr,
})
}
snapshot.Membership.Removed = n.cluster.Removed()
// maybe start rotation
n.rotationQueued = false
var newEncryptionKeys *EncryptionKeys
if n.keyRotator.NeedsRotation() {
keys := n.keyRotator.GetKeys()
if keys.PendingDEK != nil {
n.raftLogger.RotateEncryptionKey(keys.PendingDEK)
newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK}
}
}
viewStarted := make(chan struct{})
n.asyncTasks.Add(1)
n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot
go func(appliedIndex, snapshotIndex uint64) {
defer func() {
n.asyncTasks.Done()
n.snapshotInProgress <- snapshotIndex
}()
var err error
n.memoryStore.View(func(tx store.ReadTx) {
close(viewStarted)
var storeSnapshot *api.StoreSnapshot
storeSnapshot, err = n.memoryStore.Save(tx)
snapshot.Store = *storeSnapshot
})
if err != nil {
log.G(ctx).WithError(err).Error("failed to read snapshot from store")
return
}
d, err := snapshot.Marshal()
if err != nil {
log.G(ctx).WithError(err).Error("failed to marshal snapshot")
return
}
snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
if err == nil {
if err := n.raftLogger.SaveSnapshot(snap); err != nil {
log.G(ctx).WithError(err).Error("failed to save snapshot")
return
}
snapshotIndex = appliedIndex
if newEncryptionKeys != nil {
// this means we tried to rotate - so finish the rotation
if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil {
log.G(ctx).WithError(err).Error(
"failed to update encryption keys after a rotation - will wait for the next snapshot")
}
}
if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
if err != nil && err != raft.ErrCompacted {
log.G(ctx).WithError(err).Error("failed to compact snapshot")
}
}
if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs")
}
} else if err != raft.ErrSnapOutOfDate {
log.G(ctx).WithError(err).Error("failed to create snapshot")
}
}(n.appliedIndex, n.snapshotIndex)
// Wait for the goroutine to establish a read transaction, to make
// sure it sees the state as of this moment.
<-viewStarted
}