コード例 #1
0
ファイル: compaction_test.go プロジェクト: aftran/robustirc
func snapshot(fsm raft.FSM, fss raft.SnapshotStore, numLogs uint64) error {
	snapshot, err := fsm.Snapshot()
	if err != nil {
		return fmt.Errorf("Unexpected error in fsm.Snapshot(): %v", err)
	}

	robustsnap, ok := snapshot.(*robustSnapshot)
	if !ok {
		return fmt.Errorf("fsm.Snapshot() return value is not a robustSnapshot")
	}
	if robustsnap.lastIndex != numLogs {
		return fmt.Errorf("snapshot does not retain the last message, got: %d, want: %d", robustsnap.lastIndex, numLogs)
	}

	sink, err := fss.Create(numLogs, 1, []byte{})
	if err != nil {
		return fmt.Errorf("fss.Create: %v", err)
	}

	if err := snapshot.Persist(sink); err != nil {
		return fmt.Errorf("Unexpected error in snapshot.Persist(): %v", err)
	}
	sink.Close()
	return nil
}
コード例 #2
0
ファイル: compaction_test.go プロジェクト: aftran/robustirc
func restore(fsm raft.FSM, fss raft.SnapshotStore, numLogs uint64) error {
	snapshots, err := fss.List()
	if err != nil {
		return fmt.Errorf("fss.List(): %v", err)
	}
	// snapshots[0] is the most recent snapshot
	snapshotId := snapshots[0].ID
	_, readcloser, err := fss.Open(snapshotId)
	if err != nil {
		return fmt.Errorf("fss.Open(%s): %v", snapshotId, err)
	}

	if err := fsm.Restore(readcloser); err != nil {
		return fmt.Errorf("fsm.Restore(): %v", err)
	}

	first, _ := fsm.(*FSM).ircstore.FirstIndex()
	last, _ := fsm.(*FSM).ircstore.LastIndex()

	if last-first >= numLogs {
		return fmt.Errorf("Compaction did not decrease log size. got: %d, want: < %d", last-first, numLogs)
	}
	return nil
}
コード例 #3
0
ファイル: canary.go プロジェクト: dopuskh3/robustirc
func canary(fsm raft.FSM, statePath string) {
	// Create a snapshot (only creates metadata) and persist it (does the
	// actual compaction). Afterwards we have access to |rs.parsed| (all
	// raft log entries, but parsed) and |rs.del| (all messages which were
	// just compacted).
	log.Printf("Compacting before dumping state\n")

	snapshot, err := fsm.Snapshot()
	if err != nil {
		log.Fatalf("fsm.Snapshot(): %v\n", err)
	}

	rs, ok := snapshot.(*robustSnapshot)
	if !ok {
		log.Fatalf("snapshot is not a robustSnapshot")
	}

	sink, err := raft.NewDiscardSnapshotStore().Create(rs.lastIndex, 1, []byte{})
	if err != nil {
		log.Fatalf("DiscardSnapshotStore.Create(): %v\n", err)
	}

	if err := snapshot.Persist(sink); err != nil {
		log.Fatalf("snapshot.Persist(): %v\n", err)
	}

	// Dump the in-memory state into a file, to be read by robustirc-canary.
	f, err := os.Create(statePath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	log.Printf("Dumping state for robustirc-canary into %q\n", statePath)

	enc := json.NewEncoder(f)

	// Sort the keys to iterate through |rs.parsed| in deterministic order.
	keys := make([]uint64, 0, len(rs.parsed))
	for idx := range rs.parsed {
		keys = append(keys, idx)
	}
	sort.Sort(uint64Slice(keys))

	for _, idx := range keys {
		nmsg := rs.parsed[idx]
		// TODO: come up with pseudo-values for createsession/deletesession
		if nmsg.Type != types.RobustIRCFromClient {
			continue
		}
		ircmsg := irc.ParseMessage(nmsg.Data)
		if ircmsg.Command == irc.PING || ircmsg.Command == irc.PONG {
			continue
		}
		vmsgs, _ := ircServer.Get(nmsg.Id)
		cm := canaryMessageState{
			Id:        idx,
			Session:   nmsg.Session.Id,
			Input:     util.PrivacyFilterIrcmsg(ircmsg).String(),
			Output:    make([]canaryMessageOutput, len(vmsgs)),
			Compacted: rs.del[idx],
		}
		for idx, vmsg := range vmsgs {
			ifc := make(map[string]bool)
			for k, v := range vmsg.InterestingFor {
				ifc["0x"+strconv.FormatInt(k, 16)] = v
			}
			cm.Output[idx] = canaryMessageOutput{
				Text:           util.PrivacyFilterIrcmsg(irc.ParseMessage(vmsg.Data)).String(),
				InterestingFor: ifc,
			}
		}
		if err := enc.Encode(&cm); err != nil {
			log.Fatal(err)
		}
	}
}
コード例 #4
0
ファイル: canary.go プロジェクト: aftran/robustirc
func canary(fsm raft.FSM, statePath string) {
	// Create a snapshot (only creates metadata) and persist it (does the
	// actual compaction). Afterwards we have access to |rs.parsed| (all
	// raft log entries, but parsed) and |rs.del| (all messages which were
	// just compacted).
	log.Printf("Compacting before dumping state\n")

	fsm.(*FSM).skipDeletionForCanary = true

	snapshot, err := fsm.Snapshot()
	if err != nil {
		log.Fatalf("fsm.Snapshot(): %v\n", err)
	}

	rs, ok := snapshot.(*robustSnapshot)
	if !ok {
		log.Fatalf("snapshot is not a robustSnapshot")
	}

	sink, err := raft.NewDiscardSnapshotStore().Create(rs.lastIndex, 1, []byte{})
	if err != nil {
		log.Fatalf("DiscardSnapshotStore.Create(): %v\n", err)
	}

	if err := snapshot.Persist(sink); err != nil {
		log.Fatalf("snapshot.Persist(): %v\n", err)
	}

	sink.Close()

	// Dump the in-memory state into a file, to be read by robustirc-canary.
	f, err := os.Create(statePath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	log.Printf("Dumping state for robustirc-canary into %q\n", statePath)

	enc := json.NewEncoder(f)

	iterator := rs.store.GetBulkIterator(rs.firstIndex, rs.lastIndex+1)
	defer iterator.Release()
	available := iterator.First()
	for available {
		var nlog raft.Log
		if err := iterator.Error(); err != nil {
			glog.Errorf("Error while iterating through the log: %v", err)
			available = iterator.Next()
			continue
		}
		idx := binary.BigEndian.Uint64(iterator.Key())
		value := iterator.Value()
		if err := json.Unmarshal(value, &nlog); err != nil {
			glog.Errorf("Skipping log entry %d because of a JSON unmarshaling error: %v", idx, err)
			continue
		}
		available = iterator.Next()

		// TODO: compact raft messages as well, so that peer changes are not kept forever
		if nlog.Type != raft.LogCommand {
			continue
		}

		nmsg := types.NewRobustMessageFromBytes(nlog.Data)
		if time.Unix(0, nmsg.Id.Id).Before(rs.compactionEnd) {
			continue
		}

		// TODO: come up with pseudo-values for createsession/deletesession
		if nmsg.Type != types.RobustIRCFromClient {
			continue
		}
		ircmsg := irc.ParseMessage(nmsg.Data)
		if ircmsg.Command == irc.PING || ircmsg.Command == irc.PONG {
			continue
		}
		vmsgs, _ := ircServer.Get(nmsg.Id)
		cm := canaryMessageState{
			Id:        idx,
			Session:   nmsg.Session.Id,
			Input:     util.PrivacyFilterIrcmsg(ircmsg).String(),
			Output:    make([]canaryMessageOutput, len(vmsgs)),
			Compacted: false,
		}
		for idx, vmsg := range vmsgs {
			ifc := make(map[string]bool)
			for k, v := range vmsg.InterestingFor {
				ifc["0x"+strconv.FormatInt(k, 16)] = v
			}
			cm.Output[idx] = canaryMessageOutput{
				Text:           util.PrivacyFilterIrcmsg(irc.ParseMessage(vmsg.Data)).String(),
				InterestingFor: ifc,
			}
		}
		if err := enc.Encode(&cm); err != nil {
			log.Fatal(err)
		}
	}
}