예제 #1
0
func (this *WriteAheadLog) Initialize(opts *Options,
	dir, name string) (status error) {

	var logger log.Logger
	if this.Logger != nil {
		logger = this.Logger.NewLogger("local-wal:%s", name)
	} else {
		simpleLog := log.SimpleFileLog{}
		if err := simpleLog.Initialize("/dev/stderr"); err != nil {
			return err
		}
		logger = simpleLog.NewLogger("local-wal:%s", name)
	}

	if err := opts.Validate(); err != nil {
		logger.Errorf("bad wal options: %v", err)
		return err
	}

	lwal := WriteAheadLog{
		Logger: logger,
		opts:   *opts,

		dir:  dir,
		name: name,

		beginCheckpointOffset: -1,

		changeFileMap: make(map[int64]*os.File),
		recovererMap:  make(map[string]wal.Recoverer),
	}
	defer func() {
		if status != nil {
			if err := lwal.Close(); err != nil {
				this.Errorf("could not close temporary wal: %v", err)
			}
		}
	}()

	*this = lwal
	this.writingChangesCond.L = &this.mutex
	this.writingCheckpointsCond.L = &this.mutex
	return nil
}
예제 #2
0
func TestRepeatRecoverAppendChanges(test *testing.T) {
	filePath := "/tmp/test_repeat_recover_append_changes.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-repeat-recover-append")

	tmpDir, errTemp := ioutil.TempDir("", "TestWALChanges")
	if errTemp != nil {
		test.Fatalf("could not create temporary directory: %v", errTemp)
	}
	defer func() {
		if !test.Failed() {
			os.RemoveAll(tmpDir)
		}
	}()

	opts := &Options{}
	opts.MaxReadSize = 4096
	opts.MaxWriteSize = 4096
	opts.MaxReadDirNames = 1024
	opts.MaxFileSize = 1024 * 1024
	opts.FileMode = os.FileMode(0600)

	// Recovers current wal state and adds one change record.

	recoverANDappend := func(change string) []string {
		lwal := WriteAheadLog{Logger: logger}
		errInit := lwal.Initialize(opts, tmpDir, "test")
		if errInit != nil {
			test.Errorf("could not initialize local wal: %v", errInit)
			return nil
		}
		defer func() {
			if err := lwal.Close(); err != nil {
				test.Errorf("could not close the wal: %v", err)
			}
		}()

		state := walState{Logger: lwal}
		if err := lwal.Recover(&state); err != nil {
			test.Errorf("could not recover the wal: %v", err)
			return nil
		}
		test.Logf("restored %d checkpoints and %d changes",
			len(state.checkpointList), len(state.changeList))

		// Take a checkpoint for every 100 records -- this will merge changeList
		// into the checkpointList.
		if len(state.changeList)%100 == 0 {
			if err := lwal.BeginCheckpoint(); err != nil {
				test.Errorf("could not begin checkpoint: %v", err)
				return nil
			}
			for _, record := range state.checkpointList {
				if err := lwal.AppendCheckpointRecord("", []byte(record)); err != nil {
					test.Errorf("could not append checkpoint record: %v", err)
					return nil
				}
			}
			for _, record := range state.changeList {
				if err := lwal.AppendCheckpointRecord("", []byte(record)); err != nil {
					test.Errorf("could not append checkpoint record: %v", err)
					return nil
				}
			}
			if err := lwal.EndCheckpoint(true /* commit */); err != nil {
				test.Errorf("could not commit checkpoint: %v", err)
				return nil
			}
		}

		// After recovery, append one record.
		if _, err := lwal.SyncChangeRecord("", []byte(change)); err != nil {
			test.Errorf("could not append change [%s]: %v", change, err)
			return nil
		}

		// Merge checkpoint records and change records to construct all records
		// recovered.
		recoveredList := state.checkpointList
		for _, record := range state.changeList {
			recoveredList = append(recoveredList, record)
		}
		return recoveredList
	}

	// Use a seeded random steam to create records with arbitrary data and verify
	// them against the same random steam for correctness.
	seed := time.Now().UnixNano()
	if *userSeed != 0 {
		seed = *userSeed
	}
	var inputStream dataStream
	inputStream.Initialize(seed, *maxRecordSize)

	// Log the seed so that we can use it to reproduce failures later.
	test.Logf("using seed %d", seed)

	var inputList []string
	for ii := 0; ii < *numTestRecords; ii++ {
		input := inputStream.RandomString()
		inputList = append(inputList, input)

		recoveredList := recoverANDappend(input)
		numRecovered := len(recoveredList)
		if numRecovered != ii {
			test.Errorf("recovery should've received %d entries, but got %d", ii,
				numRecovered)
			return
		}
		// Test that all records are in expected format.
		for jj, record := range recoveredList {
			if record != inputList[jj] {
				test.Errorf("change record %d recovered as [%s] which is unexpected",
					jj, record)
			}
		}
		test.Logf("test for %d wal records has passed", ii)
	}
}
예제 #3
0
func TestLoopbackMessages(test *testing.T) {
	filePath := "/tmp/test_msg_simple_loopback_messages.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-msg-simple")
	logger.Infof("starting new test")

	opts := &Options{
		MaxWriteTimeout:     20 * time.Millisecond,
		ResponseQueueSize:   1,
		SendQueueSize:       1,
		NegotiationTimeout:  20 * time.Millisecond,
		SendRetryTimeout:    10 * time.Millisecond,
		MaxDispatchRequests: 1,
	}
	if err := opts.Validate(); err != nil {
		test.Errorf("could not validate messenger options: %v", err)
		return
	}

	msn1 := &Messenger{Logger: logger}
	if err := msn1.Initialize(opts, "msn1"); err != nil {
		test.Errorf("could not initialize messenger msn1: %v", err)
		return
	}
	defer func() {
		if err := msn1.Close(); err != nil {
			test.Errorf("could not close messenger msn1: %v", err)
		}
	}()

	if err := msn1.Start(); err != nil {
		test.Errorf("could not start messenger msn1: %v", err)
		return
	}
	defer func() {
		if err := msn1.Stop(); err != nil {
			test.Errorf("could not stop messenger msn1: %v", err)
		}
	}()

	server := &RPCServer{Logger: logger, Messenger: msn1}

	if err := msn1.RegisterClass("msn1", server, "a", "b", "c"); err != nil {
		test.Errorf("could not export rpc services: %v", err)
		return
	}
	defer func() {
		if err := msn1.UnregisterClass("msn1", "a", "b", "c"); err != nil {
			test.Errorf("could not unregister rpc services: %v", err)
		}
	}()

	start := time.Now()
	for ii := 0; ii < 1000; ii++ {
		reqHeader := msn1.NewRequest("msn1", "", "a", time.Second)
		if err := msn1.Send("msn1", reqHeader, []byte("request")); err != nil {
			test.Errorf("could not send request to self: %v", err)
			return
		}
		if _, _, err := msn1.Receive(reqHeader); err != nil {
			test.Errorf("could not receive response from self: %v", err)
			return
		}
	}
	duration := time.Since(start)
	test.Logf("average loopback round-trip time is %v", duration/1000)
}
예제 #4
0
func TestNetworkMessaging(test *testing.T) {
	filePath := "/tmp/test_msg_simple_network_messaging.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-msg-simple")
	logger.Infof("starting new test")

	opts := &Options{
		MaxWriteTimeout:        20 * time.Millisecond,
		ResponseQueueSize:      1,
		SendQueueSize:          1,
		NegotiationTimeout:     20 * time.Millisecond,
		SendRetryTimeout:       10 * time.Millisecond,
		MaxDispatchRequests:    1,
		DispatchRequestTimeout: time.Millisecond,
	}
	if err := opts.Validate(); err != nil {
		test.Errorf("could not validate messenger options: %v", err)
		return
	}

	msn1 := &Messenger{Logger: logger}
	if err := msn1.Initialize(opts, "msn1"); err != nil {
		test.Errorf("could not initialize messenger msn1: %v", err)
		return
	}
	defer func() {
		if err := msn1.Close(); err != nil {
			test.Errorf("could not close messenger msn1: %v", err)
		}
	}()

	msn2 := &Messenger{Logger: logger}
	if err := msn2.Initialize(opts, "msn2"); err != nil {
		test.Errorf("could not initialize messenger msn2: %v", err)
		return
	}
	defer func() {
		if err := msn2.Close(); err != nil {
			test.Errorf("could not close messenger msn2: %v", err)
		}
	}()

	if err := msn1.Start(); err != nil {
		test.Errorf("could not start messenger msn1: %v", err)
		return
	}
	defer func() {
		if err := msn1.Stop(); err != nil {
			test.Errorf("could not stop messenger msn1: %v", err)
		}
	}()

	if err := msn2.Start(); err != nil {
		test.Errorf("could not start messenger msn2: %v", err)
		return
	}
	defer func() {
		if err := msn2.Stop(); err != nil {
			test.Errorf("could not stop messenger msn2: %v", err)
		}
	}()

	msn1Address := "tcp://127.0.0.1:10000"
	if err := msn1.AddListenerAddress(msn1Address); err != nil {
		test.Errorf("could not add listener address %s to msn1: %v", msn1Address,
			err)
		return
	}
	msn1AddressList := msn1.ListenerAddressList()

	msn2Address := "tcp://127.0.0.1:20000"
	if err := msn2.AddListenerAddress(msn2Address); err != nil {
		test.Errorf("could not add listener address %s to msn2: %v", msn2Address,
			err)
		return
	}
	msn2AddressList := msn2.ListenerAddressList()

	if err := msn1.AddPeerAddress("msn2", msn2AddressList); err != nil {
		test.Errorf("could not add msn2 addresses to msn1: %v", err)
		return
	}

	if err := msn2.AddPeerAddress("msn1", msn1AddressList); err != nil {
		test.Errorf("could not add msn1 addresses to msn2: %v", err)
		return
	}

	// Register a service on msn1.
	msn1server := &RPCServer{Logger: logger, Messenger: msn1}

	if err := msn1.RegisterClass("class", msn1server, "ping"); err != nil {
		test.Errorf("could not export rpc services: %v", err)
		return
	}
	defer func() {
		if err := msn1.UnregisterClass("class", "ping"); err != nil {
			test.Errorf("could not unregister rpc services: %v", err)
		}
	}()

	const count = 1000
	start := time.Now()
	for ii := 0; ii < count; ii++ {
		reqHeader := msn2.NewRequest("class", "", "ping", time.Second)
		reqData := []byte("request-data")
		if err := msn2.Send("msn1", reqHeader, reqData); err != nil {
			test.Errorf("could not send request to msn1: %v", err)
			return
		}
		resHeader, _, errRecv := msn2.Receive(reqHeader)
		if errRecv != nil {
			test.Errorf("could not receive response from msn1: %v", errRecv)
			return
		}
		logger.Infof("[%s] -> [%s]", reqHeader, resHeader)
		if err := msn2.CloseMessage(reqHeader); err != nil {
			test.Errorf("could not close request %s: %v", reqHeader, err)
			return
		}
	}
	duration := time.Since(start)
	test.Logf("average loopback network round-trip time is %v", duration/count)
}
예제 #5
0
func TestBasicController(test *testing.T) {
	filePath := "/tmp/test_basic_controller.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-basic-controller")
	logger.Infof("starting new controller test")

	controller := &BasicController{}
	controller.Initialize(logger)
	defer func() {
		if err := controller.Close(); err != nil {
			test.Errorf("could not close the controller: %v", err)
			return
		}

		lock, errLock := controller.LockAll()
		if !errs.IsClosed(errLock) {
			test.Errorf("controller issued lock %v after it is closed", lock)
			return
		}

		// Lock and Unlock work even after a Close. Safety is not expected.
		foobar := controller.ReadLock("foo", "bar")
		foobar.Unlock()
	}()

	lock1, errLock1 := controller.LockAll()
	if errLock1 != nil {
		test.Errorf("could not acquire lock1: %v", errLock1)
		return
	}

	lock2, errLock2 := controller.TimedLockAll(time.Millisecond)
	if !errs.IsTimeout(errLock2) {
		test.Errorf("second lock %v is issued while lock1 %v is active",
			lock2, lock1)
		return
	}
	lock1.Unlock()

	lock3, errLock3 := controller.TimedLock(time.Millisecond, "a")
	if errLock3 != nil {
		test.Errorf("could not acquire lock3: %v", errLock3)
		return
	}

	lock4, errLock4 := controller.TimedLock(time.Millisecond, "b")
	if errLock4 != nil {
		test.Errorf("could not acquire lock4: %v", errLock4)
		return
	}

	lock5, errLock5 := controller.TimedLockAll(time.Millisecond)
	if errLock5 == nil {
		test.Errorf("lock all lock %v issue while locks %v and %v are active",
			lock5, lock3, lock4)
		return
	}

	lock3.Unlock()
	lock4.Unlock()

	foo := controller.ReadLock("foo")
	bar := controller.ReadLock("bar")
	bar.Unlock("bar")
	foo.Unlock("foo")

	baz := controller.ReadLock("baz")
	baz.Unlock()

	test.Logf("returning")
}
예제 #6
0
func TestResourceController(test *testing.T) {
	filePath := "/tmp/test_resource_controller.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-resource-controller")
	logger.Infof("starting new test")

	controller := &ResourceController{}
	controller.Initialize(logger)

	foo := controller.LockResources("foo")
	bar := controller.LockResources("bar")

	timeoutCh := time.After(time.Millisecond)
	if _, err := controller.TimeLockResources(timeoutCh, "foo"); err == nil {
		test.Errorf("resource foo is issued multiple times")
		return
	}

	timeoutCh = time.After(time.Millisecond)
	if _, err := controller.TimeLockAll(timeoutCh); err == nil {
		test.Errorf("lock all issued when foo and bar are busy")
		return
	}

	bar.Unlock("bar")
	foo.Unlock("foo")

	all := controller.LockAll()
	all.Unlock()

	baz := controller.LockResources("baz")
	baz.Unlock()

	baz2 := controller.LockResources("baz")
	baz2.Unlock()

	// Allow multiple readers on a resource.

	a1 := controller.LockResources("", "aa")
	a2 := controller.LockResources("", "aa")

	timeoutCh = time.After(time.Millisecond)
	if _, err := controller.TimeLockAll(timeoutCh); err == nil {
		test.Errorf("lock all issued when two readers are sharing aa")
		return
	}
	timeoutCh = time.After(time.Millisecond)
	if _, err := controller.TimeLockResources(timeoutCh, "aa"); err == nil {
		test.Errorf("exclusive lock is issued when two readers have aa")
		return
	}
	a1.Unlock()
	a2.Unlock()

	// A close on the timeout channel must unlock the waiters.

	b1 := controller.LockAll()
	timeoutCh2 := make(chan time.Time)
	close(timeoutCh2)
	if _, err := controller.TimeLockAll(timeoutCh2); !errs.IsTimeout(err) {
		test.Errorf("closing timeout channel did not unblock the lock")
		return
	}
	b1.Unlock()
}
예제 #7
0
파일: election_test.go 프로젝트: bvk/ascent
func TestAlgoElection(test *testing.T) {
	runtime.GOMAXPROCS(4)

	filePath := "/tmp/test_algo_election.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-algo-election")
	logger.Infof("starting new test")

	tmpDir, errTemp := ioutil.TempDir("", "TestAlgoElectionWAL")
	if errTemp != nil {
		test.Fatalf("could not create temporary directory: %v", errTemp)
	}
	defer func() {
		if !test.Failed() {
			os.RemoveAll(tmpDir)
		}
	}()

	walOpts := &fswal.Options{
		MaxReadSize:     4096,
		MaxWriteSize:    4096,
		MaxReadDirNames: 1024,
		MaxFileSize:     1024 * 1024,
		FileMode:        os.FileMode(0600),
	}
	if err := walOpts.Validate(); err != nil {
		test.Errorf("could not validate fswal options: %v", err)
		return
	}

	msnOpts := &simple.Options{
		MaxWriteTimeout:        20 * time.Millisecond,
		ResponseQueueSize:      1024,
		SendQueueSize:          1024,
		NegotiationTimeout:     20 * time.Millisecond,
		SendRetryTimeout:       10 * time.Millisecond,
		MaxDispatchRequests:    1024,
		DispatchRequestTimeout: time.Millisecond,
	}
	if err := msnOpts.Validate(); err != nil {
		test.Errorf("could not validate messenger options: %v", err)
		return
	}

	electionOpts := &Options{
		MaxElectionHistory: 10,
		PaxosOptions: classic.Options{
			ProposeRetryInterval:    time.Millisecond,
			NumExtraPhase1Acceptors: 1,
			LearnTimeout:            time.Second,
			LearnRetryInterval:      time.Millisecond,
		},
	}
	if err := electionOpts.Validate(); err != nil {
		test.Errorf("could not validate election options: %v", err)
		return
	}

	type Agent struct {
		log.Logger

		name        string
		addressList []string
		chosen      []byte

		wal      *fswal.WriteAheadLog
		msn      *simple.Messenger
		election *Election
	}

	newAgent := func(name string) *Agent {
		agent := &Agent{}

		wal := &fswal.WriteAheadLog{Logger: logger}
		if err := wal.Initialize(walOpts, tmpDir, name); err != nil {
			test.Errorf("could not create wal for %s: %v", name, err)
			return nil
		}

		msn := &simple.Messenger{Logger: logger}
		if err := msn.Initialize(msnOpts, name); err != nil {
			test.Errorf("could not initialize messenger for %s: %v", name, err)
			return nil
		}

		election := &Election{Logger: logger}
		errInit := election.Initialize(electionOpts, "algo/election", "test", msn,
			wal)
		if errInit != nil {
			test.Errorf("could not initialize election instance for %s: %v", name,
				errInit)
			return nil
		}

		rpcList := ElectionRPCList()
		errRegister := msn.RegisterClass("algo/election", election, rpcList...)
		if errRegister != nil {
			test.Errorf("could not export paxos instance rpcs: %v", errRegister)
			return nil
		}

		agent.Logger = msn
		agent.name = name
		agent.msn = msn
		agent.wal = wal
		agent.election = election
		return agent
	}
	agent1 := newAgent("one")
	agent2 := newAgent("two")
	agent3 := newAgent("three")
	other := newAgent("other")

	startAgent := func(agent *Agent) {
		if err := agent.msn.Start(); err != nil {
			test.Errorf("could not start messenger on %s: %v", agent.name, err)
			return
		}

		msnAddress := "tcp://127.0.0.1:0"
		if err := agent.msn.AddListenerAddress(msnAddress); err != nil {
			test.Errorf("could not add listener address %s to %s: %v", msnAddress,
				agent.name, err)
			return
		}
		agent.addressList = agent.msn.ListenerAddressList()
	}
	startAgent(agent1)
	startAgent(agent2)
	startAgent(agent3)
	startAgent(other)

	connectAgents := func(this *Agent, rest ...*Agent) {
		for _, other := range rest {
			errAdd := this.msn.AddPeerAddress(other.name, other.addressList)
			if errAdd != nil {
				test.Errorf("could not add peer %s to %s: %v", this.name, other.name,
					errAdd)
				return
			}
		}
	}
	connectAgents(agent1, agent2, agent3, other)
	connectAgents(agent2, agent1, agent3, other)
	connectAgents(agent3, agent1, agent2, other)
	connectAgents(other, agent1, agent2, agent3)

	configureAgents := func(this *Agent, committee []string) {
		errConfig := this.election.Configure(committee)
		if errConfig != nil {
			test.Errorf("could not configure paxos on %s: %v", this.name,
				errConfig)
			return
		}
	}
	committee := []string{agent1.name, agent2.name, agent3.name}
	configureAgents(agent1, committee)
	configureAgents(agent2, committee)
	configureAgents(agent3, committee)
	configureAgents(other, committee)

	doneCh := make(chan bool)
	monitor := func(client *Agent) {
		defer func() {
			doneCh <- true
		}()

		leader, round, errRefresh := client.election.RefreshLeader(time.Second)
		if errRefresh != nil {
			test.Errorf("could not refresh election status: %v", errRefresh)
			return
		}
		test.Logf("last known election leader is %s for round %d", leader,
			round)

		if leader == "" {
			newLeader, newRound, errElect := client.election.ElectLeader(time.Second)
			if errElect != nil {
				test.Errorf("could not elect new leader: %v", errElect)
				return
			}
			test.Logf("new leader %s is elected for round %d", newLeader,
				newRound)
			leader, round = newLeader, newRound
		}
	}

	go monitor(agent1)
	go monitor(agent2)
	go monitor(agent3)
	go monitor(other)
	<-doneCh
	<-doneCh
	<-doneCh
	<-doneCh

	closeAgent := func(agent *Agent) {
		rpcList := ElectionRPCList()
		errUnregister := agent.msn.UnregisterClass("algo/election", rpcList...)
		if errUnregister != nil {
			test.Errorf("could not unregister election exports: %v", errUnregister)
			return
		}

		if err := agent.election.Close(); err != nil {
			test.Errorf("could not close paxos on %s: %v", agent.name, err)
			return
		}
		if err := agent.msn.Stop(); err != nil {
			test.Errorf("could not stop messenger on %s: %v", agent.name, err)
			return
		}
		if err := agent.msn.Close(); err != nil {
			test.Errorf("could not close messenger on %s: %v", agent.name, err)
			return
		}
		if err := agent.wal.Close(); err != nil {
			test.Errorf("could not close wal on %s: %v", agent.name, err)
			return
		}
	}
	_ = closeAgent
	// closeAgent(agent1)
	// closeAgent(agent2)
	// closeAgent(agent3)
	// closeAgent(other)
}
예제 #8
0
파일: paxos_test.go 프로젝트: bvk/ascent
func TestClassicPaxosConsensus(test *testing.T) {
	runtime.GOMAXPROCS(4)

	filePath := "/tmp/test_paxos_classic_consensus.log"
	simpleLog := log.SimpleFileLog{}
	if err := simpleLog.Initialize(filePath); err != nil {
		test.Fatalf("could not initialize log backend: %v", err)
		return
	}
	logger := simpleLog.NewLogger("test-paxos-classic")
	logger.Infof("starting new test")

	tmpDir, errTemp := ioutil.TempDir("", "TestWALChanges")
	if errTemp != nil {
		test.Fatalf("could not create temporary directory: %v", errTemp)
	}
	defer func() {
		if !test.Failed() {
			os.RemoveAll(tmpDir)
		}
	}()

	walOpts := &fswal.Options{
		MaxReadSize:     4096,
		MaxWriteSize:    4096,
		MaxReadDirNames: 1024,
		MaxFileSize:     1024 * 1024,
		FileMode:        os.FileMode(0600),
	}

	msnOpts := &simple.Options{
		MaxWriteTimeout:        20 * time.Millisecond,
		ResponseQueueSize:      1024,
		SendQueueSize:          1024,
		NegotiationTimeout:     20 * time.Millisecond,
		SendRetryTimeout:       10 * time.Millisecond,
		MaxDispatchRequests:    10,
		DispatchRequestTimeout: time.Millisecond,
	}
	if err := msnOpts.Validate(); err != nil {
		test.Errorf("could not validate messenger options: %v", err)
		return
	}

	paxosOpts := &Options{
		ProposeRetryInterval:    time.Millisecond,
		NumExtraPhase1Acceptors: 1,
		LearnTimeout:            10 * time.Millisecond,
		LearnRetryInterval:      time.Millisecond,
	}

	type Agent struct {
		name        string
		addressList []string
		chosen      []byte

		wal   *fswal.WriteAheadLog
		msn   *simple.Messenger
		paxos *Paxos
	}

	newAgent := func(name string) *Agent {
		agent := &Agent{}

		wal1 := &fswal.WriteAheadLog{Logger: logger}
		if err := wal1.Initialize(walOpts, tmpDir, name); err != nil {
			test.Errorf("could not create wal for %s: %v", name, err)
			return nil
		}

		msn1 := &simple.Messenger{Logger: logger}
		if err := msn1.Initialize(msnOpts, name); err != nil {
			test.Errorf("could not initialize messenger for %s: %v", name, err)
			return nil
		}

		paxos1 := &Paxos{Logger: logger}
		errInit := paxos1.Initialize(paxosOpts, "paxos/classic", "test", msn1,
			wal1)
		if errInit != nil {
			test.Errorf("could not initialize paxos1 instance for %s: %v", name,
				errInit)
			return nil
		}

		rpcList := PaxosRPCList()
		errRegister := msn1.RegisterClass("paxos/classic", paxos1, rpcList...)
		if errRegister != nil {
			test.Errorf("could not export paxos instance rpcs: %v", errRegister)
			return nil
		}

		agent.name = name
		agent.msn = msn1
		agent.wal = wal1
		agent.paxos = paxos1
		return agent
	}
	agent1 := newAgent("one")
	agent2 := newAgent("two")
	agent3 := newAgent("three")

	startAgent := func(agent *Agent) {
		if err := agent.msn.Start(); err != nil {
			test.Errorf("could not start messenger on %s: %v", agent.name, err)
			return
		}

		msn1Address := "tcp://127.0.0.1:0"
		if err := agent.msn.AddListenerAddress(msn1Address); err != nil {
			test.Errorf("could not add listener address %s to %s: %v", msn1Address,
				agent.name, err)
			return
		}
		agent.addressList = agent.msn.ListenerAddressList()
	}
	startAgent(agent1)
	startAgent(agent2)
	startAgent(agent3)

	connectAgents := func(this *Agent, rest ...*Agent) {
		for _, other := range rest {
			errAdd := this.msn.AddPeerAddress(other.name, other.addressList)
			if errAdd != nil {
				test.Errorf("could not add peer %s to %s: %v", this.name, other.name,
					errAdd)
				return
			}
		}
	}
	connectAgents(agent1, agent2, agent3)
	connectAgents(agent2, agent1, agent3)
	connectAgents(agent3, agent1, agent2)

	configureAgents := func(this *Agent,
		proposers, acceptors, learners []string) {
		errConfig := this.paxos.Configure(proposers, acceptors, learners)
		if errConfig != nil {
			test.Errorf("could not configure paxos on %s: %v", this.name,
				errConfig)
			return
		}
	}
	agents := []string{agent1.name, agent2.name, agent3.name}
	configureAgents(agent1, agents, agents, agents)
	configureAgents(agent2, agents, agents, agents)
	configureAgents(agent3, agents, agents, agents)

	doneCh := make(chan bool)
	propose := func(client *Agent, value string) {
		defer func() {
			doneCh <- true
		}()

		start := time.Now()
		chosen, errProp := client.paxos.Propose([]byte(value), time.Second)
		if errProp != nil {
			test.Errorf("could not propose value %s: %v", value, errProp)
			return
		}

		test.Logf("classic paxos consensus took %v time to choose %s for %s",
			time.Since(start), chosen, client.name)
		client.chosen = chosen
	}
	go propose(agent1, "agent1")
	go propose(agent2, "agent2")
	go propose(agent3, "agent3")
	<-doneCh
	<-doneCh
	<-doneCh

	if bytes.Compare(agent1.chosen, agent2.chosen) != 0 ||
		bytes.Compare(agent2.chosen, agent3.chosen) != 0 ||
		bytes.Compare(agent3.chosen, agent1.chosen) != 0 {
		test.Errorf("different values are chosen, which is wrong")
		return
	}

	closeAgent := func(agent *Agent) {
		rpcList := PaxosRPCList()
		errUnregister := agent.msn.UnregisterClass("paxos/classic", rpcList...)
		if errUnregister != nil {
			test.Errorf("could not unregister paxos instance exports: %v",
				errUnregister)
			return
		}

		if err := agent.paxos.Close(); err != nil {
			test.Errorf("could not close paxos on %s: %v", agent.name, err)
			return
		}
		if err := agent.msn.Stop(); err != nil {
			test.Errorf("could not stop messenger on %s: %v", agent.name, err)
			return
		}
		if err := agent.msn.Close(); err != nil {
			test.Errorf("could not close messenger on %s: %v", agent.name, err)
			return
		}
		if err := agent.wal.Close(); err != nil {
			test.Errorf("could not close wal on %s: %v", agent.name, err)
			return
		}
	}
	closeAgent(agent1)
	closeAgent(agent2)
	closeAgent(agent3)
}