func snapshot(fsm raft.FSM, fss raft.SnapshotStore, numLogs uint64) error {
snapshot, err := fsm.Snapshot()
if err != nil {
return fmt.Errorf("Unexpected error in fsm.Snapshot(): %v", err)
}
robustsnap, ok := snapshot.(*robustSnapshot)
if !ok {
return fmt.Errorf("fsm.Snapshot() return value is not a robustSnapshot")
}
if robustsnap.lastIndex != numLogs {
return fmt.Errorf("snapshot does not retain the last message, got: %d, want: %d", robustsnap.lastIndex, numLogs)
}
sink, err := fss.Create(numLogs, 1, []byte{})
if err != nil {
return fmt.Errorf("fss.Create: %v", err)
}
if err := snapshot.Persist(sink); err != nil {
return fmt.Errorf("Unexpected error in snapshot.Persist(): %v", err)
}
sink.Close()
return nil
}
func restore(fsm raft.FSM, fss raft.SnapshotStore, numLogs uint64) error {
snapshots, err := fss.List()
if err != nil {
return fmt.Errorf("fss.List(): %v", err)
}
// snapshots[0] is the most recent snapshot
snapshotId := snapshots[0].ID
_, readcloser, err := fss.Open(snapshotId)
if err != nil {
return fmt.Errorf("fss.Open(%s): %v", snapshotId, err)
}
if err := fsm.Restore(readcloser); err != nil {
return fmt.Errorf("fsm.Restore(): %v", err)
}
first, _ := fsm.(*FSM).ircstore.FirstIndex()
last, _ := fsm.(*FSM).ircstore.LastIndex()
if last-first >= numLogs {
return fmt.Errorf("Compaction did not decrease log size. got: %d, want: < %d", last-first, numLogs)
}
return nil
}
func canary(fsm raft.FSM, statePath string) {
// Create a snapshot (only creates metadata) and persist it (does the
// actual compaction). Afterwards we have access to |rs.parsed| (all
// raft log entries, but parsed) and |rs.del| (all messages which were
// just compacted).
log.Printf("Compacting before dumping state\n")
snapshot, err := fsm.Snapshot()
if err != nil {
log.Fatalf("fsm.Snapshot(): %v\n", err)
}
rs, ok := snapshot.(*robustSnapshot)
if !ok {
log.Fatalf("snapshot is not a robustSnapshot")
}
sink, err := raft.NewDiscardSnapshotStore().Create(rs.lastIndex, 1, []byte{})
if err != nil {
log.Fatalf("DiscardSnapshotStore.Create(): %v\n", err)
}
if err := snapshot.Persist(sink); err != nil {
log.Fatalf("snapshot.Persist(): %v\n", err)
}
// Dump the in-memory state into a file, to be read by robustirc-canary.
f, err := os.Create(statePath)
if err != nil {
log.Fatal(err)
}
defer f.Close()
log.Printf("Dumping state for robustirc-canary into %q\n", statePath)
enc := json.NewEncoder(f)
// Sort the keys to iterate through |rs.parsed| in deterministic order.
keys := make([]uint64, 0, len(rs.parsed))
for idx := range rs.parsed {
keys = append(keys, idx)
}
sort.Sort(uint64Slice(keys))
for _, idx := range keys {
nmsg := rs.parsed[idx]
// TODO: come up with pseudo-values for createsession/deletesession
if nmsg.Type != types.RobustIRCFromClient {
continue
}
ircmsg := irc.ParseMessage(nmsg.Data)
if ircmsg.Command == irc.PING || ircmsg.Command == irc.PONG {
continue
}
vmsgs, _ := ircServer.Get(nmsg.Id)
cm := canaryMessageState{
Id: idx,
Session: nmsg.Session.Id,
Input: util.PrivacyFilterIrcmsg(ircmsg).String(),
Output: make([]canaryMessageOutput, len(vmsgs)),
Compacted: rs.del[idx],
}
for idx, vmsg := range vmsgs {
ifc := make(map[string]bool)
for k, v := range vmsg.InterestingFor {
ifc["0x"+strconv.FormatInt(k, 16)] = v
}
cm.Output[idx] = canaryMessageOutput{
Text: util.PrivacyFilterIrcmsg(irc.ParseMessage(vmsg.Data)).String(),
InterestingFor: ifc,
}
}
if err := enc.Encode(&cm); err != nil {
log.Fatal(err)
}
}
}
func canary(fsm raft.FSM, statePath string) {
// Create a snapshot (only creates metadata) and persist it (does the
// actual compaction). Afterwards we have access to |rs.parsed| (all
// raft log entries, but parsed) and |rs.del| (all messages which were
// just compacted).
log.Printf("Compacting before dumping state\n")
fsm.(*FSM).skipDeletionForCanary = true
snapshot, err := fsm.Snapshot()
if err != nil {
log.Fatalf("fsm.Snapshot(): %v\n", err)
}
rs, ok := snapshot.(*robustSnapshot)
if !ok {
log.Fatalf("snapshot is not a robustSnapshot")
}
sink, err := raft.NewDiscardSnapshotStore().Create(rs.lastIndex, 1, []byte{})
if err != nil {
log.Fatalf("DiscardSnapshotStore.Create(): %v\n", err)
}
if err := snapshot.Persist(sink); err != nil {
log.Fatalf("snapshot.Persist(): %v\n", err)
}
sink.Close()
// Dump the in-memory state into a file, to be read by robustirc-canary.
f, err := os.Create(statePath)
if err != nil {
log.Fatal(err)
}
defer f.Close()
log.Printf("Dumping state for robustirc-canary into %q\n", statePath)
enc := json.NewEncoder(f)
iterator := rs.store.GetBulkIterator(rs.firstIndex, rs.lastIndex+1)
defer iterator.Release()
available := iterator.First()
for available {
var nlog raft.Log
if err := iterator.Error(); err != nil {
glog.Errorf("Error while iterating through the log: %v", err)
available = iterator.Next()
continue
}
idx := binary.BigEndian.Uint64(iterator.Key())
value := iterator.Value()
if err := json.Unmarshal(value, &nlog); err != nil {
glog.Errorf("Skipping log entry %d because of a JSON unmarshaling error: %v", idx, err)
continue
}
available = iterator.Next()
// TODO: compact raft messages as well, so that peer changes are not kept forever
if nlog.Type != raft.LogCommand {
continue
}
nmsg := types.NewRobustMessageFromBytes(nlog.Data)
if time.Unix(0, nmsg.Id.Id).Before(rs.compactionEnd) {
continue
}
// TODO: come up with pseudo-values for createsession/deletesession
if nmsg.Type != types.RobustIRCFromClient {
continue
}
ircmsg := irc.ParseMessage(nmsg.Data)
if ircmsg.Command == irc.PING || ircmsg.Command == irc.PONG {
continue
}
vmsgs, _ := ircServer.Get(nmsg.Id)
cm := canaryMessageState{
Id: idx,
Session: nmsg.Session.Id,
Input: util.PrivacyFilterIrcmsg(ircmsg).String(),
Output: make([]canaryMessageOutput, len(vmsgs)),
Compacted: false,
}
for idx, vmsg := range vmsgs {
ifc := make(map[string]bool)
for k, v := range vmsg.InterestingFor {
ifc["0x"+strconv.FormatInt(k, 16)] = v
}
cm.Output[idx] = canaryMessageOutput{
Text: util.PrivacyFilterIrcmsg(irc.ParseMessage(vmsg.Data)).String(),
InterestingFor: ifc,
}
}
if err := enc.Encode(&cm); err != nil {
log.Fatal(err)
}
}
}