func setup(t *testing.T, tag string, ngroups int, nreplicas int, unreliable bool) *tCluster {
runtime.GOMAXPROCS(4)
// compile ../main/diskvd.go
cmd := exec.Command("go", "build", "diskvd.go")
cmd.Dir = "../main"
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
t.Fatalf("could not compile ../main/diskvd.go: %v", err)
}
const nmasters = 3
tc := &tCluster{}
tc.t = t
tc.unreliable = unreliable
tc.dir = "/var/tmp/824-"
tc.dir += strconv.Itoa(os.Getuid()) + "/"
os.Mkdir(tc.dir, 0777)
tc.dir += "lab5-" + tag + "/"
os.RemoveAll(tc.dir)
os.Mkdir(tc.dir, 0777)
tc.masters = make([]*shardmaster.ShardMaster, nmasters)
tc.masterports = make([]string, nmasters)
for i := 0; i < nmasters; i++ {
tc.masterports[i] = tc.newport()
}
log.SetOutput(ioutil.Discard) // suppress method errors &c
for i := 0; i < nmasters; i++ {
tc.masters[i] = shardmaster.StartServer(tc.masterports, i)
}
log.SetOutput(os.Stdout) // re-enable error output.
tc.mck = tc.shardclerk()
tc.groups = make([]*tGroup, ngroups)
for i := 0; i < ngroups; i++ {
g := &tGroup{}
tc.groups[i] = g
g.gid = int64(i + 100)
g.servers = make([]*tServer, nreplicas)
for j := 0; j < nreplicas; j++ {
g.servers[j] = &tServer{}
g.servers[j].port = tc.newport()
g.servers[j].dir = tc.dir + randstring(12)
if err := os.Mkdir(g.servers[j].dir, 0777); err != nil {
t.Fatalf("Mkdir(%v): %v", g.servers[j].dir, err)
}
}
for j := 0; j < nreplicas; j++ {
tc.start1(i, j)
}
}
// return smh, gids, ha, sa, clean
return tc
}
// Set up and start servers for shardkv groups and shardmaster
func setup(tag string, unreliable bool, numGroups int, numReplicas int) ([]string, []int64, [][]string, [][]*ShardKV, func(), func()) {
runtime.GOMAXPROCS(4)
const numMasters = 3
var smServers []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, numMasters)
var smPorts []string = make([]string, numMasters)
// Start shardmaster servers
for i := 0; i < numMasters; i++ {
smPorts[i] = makePort(tag+"m", i)
}
for i := 0; i < numMasters; i++ {
smServers[i] = shardmaster.StartServer(smPorts, i, false)
}
gids := make([]int64, numGroups) // each group ID
kvPorts := make([][]string, numGroups) // ShardKV ports, [group][replica]
kvServers := make([][]*ShardKV, numGroups) // ShardKVs
// Start numReplicas shardkv servers for each of numGroups groups
for i := 0; i < numGroups; i++ {
gids[i] = int64(i + 100)
kvServers[i] = make([]*ShardKV, numReplicas)
kvPorts[i] = make([]string, numReplicas)
for j := 0; j < numReplicas; j++ {
kvPorts[i][j] = makePort(tag+"s", (i*numReplicas)+j)
}
for j := 0; j < numReplicas; j++ {
kvServers[i][j] = StartServer(gids[i], smPorts, kvPorts[i], j, false)
kvServers[i][j].unreliable = unreliable
}
}
clean := func() { shardkvCleanup(kvServers, false); smCleanup(smServers, false) }
cleanSaveDisk := func() { shardkvCleanup(kvServers, true); smCleanup(smServers, true) }
return smPorts, gids, kvPorts, kvServers, clean, cleanSaveDisk
}
// Start all servers/clients on local machine.
// Returns a system and a clean function
func localSetup(name string, args SetupArgs) (System, func()) {
runtime.GOMAXPROCS(args.NumProcesses)
// Start shardmasters
smHosts := make([]string, args.NumMasters)
sms := make([]*shardmaster.ShardMaster, args.NumMasters)
for i := 0; i < args.NumMasters; i++ {
smHosts[i] = port(name+"m", i)
sms[i] = shardmaster.StartServer(smHosts, i)
}
smClerk := shardmaster.MakeClerk(smHosts)
// Start ShardKV servers
gids := make([]int64, args.NumGroups)
skvHosts := make([][]string, args.NumGroups)
skvs := make([][]*shardkv.ShardKV, args.NumGroups)
for i := 0; i < args.NumGroups; i++ {
gids[i] = int64(100 + i)
skvHosts[i] = make([]string, args.NumReplicas)
skvs[i] = make([]*shardkv.ShardKV, args.NumReplicas)
for j := 0; j < args.NumReplicas; j++ {
skvHosts[i][j] = port(name+"s", i*args.NumReplicas+j)
skvs[i][j] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], j)
}
smClerk.Join(gids[i], skvHosts[i])
}
// Start clients
clerks := make([]*shardkv.Clerk, args.NumClients)
for i := 0; i < args.NumClients; i++ {
clerks[i] = shardkv.MakeClerk(smHosts)
}
system := System{
ShardMasterHosts: smHosts,
ShardMasters: sms,
ShardMasterClerk: smClerk,
GroupIDs: gids,
ShardKVHosts: skvHosts,
ShardKVs: skvs,
Clerks: clerks,
}
clean := func() {
for i := 0; i < args.NumMasters; i++ {
sms[i].Kill()
}
for i := 0; i < args.NumGroups; i++ {
for j := 0; j < args.NumReplicas; j++ {
skvs[i][j].Kill()
}
}
}
return system, clean
}
func setup(tag string, unreliable bool, sudden bool) ([]string, []int64, [][]string, [][]*ShardKV, func(), []string) {
runtime.GOMAXPROCS(4)
const nmasters = 3
var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
var smh []string = make([]string, nmasters)
// defer mcleanup(sma)
for i := 0; i < nmasters; i++ {
smh[i] = port(tag+"m", i)
}
for i := 0; i < nmasters; i++ {
sma[i] = shardmaster.StartServer(smh, i)
}
var mba []*messagebroker.MBServer = make([]*messagebroker.MBServer, nmasters)
var mbh []string = make([]string, nmasters)
for i := 0; i < nmasters; i++ {
mbh[i] = port(tag+"messagebroker", i)
}
for i := 0; i < nmasters; i++ {
mba[i] = messagebroker.StartServer(mbh, i)
}
const ngroups = 3 // replica groups
const nreplicas = 3 // servers per group
gids := make([]int64, ngroups) // each group ID
ha := make([][]string, ngroups) // ShardKV ports, [group][replica]
sa := make([][]*ShardKV, ngroups) // ShardKVs
// defer cleanup(sa)
for i := 0; i < ngroups; i++ {
gids[i] = int64(i + 100)
sa[i] = make([]*ShardKV, nreplicas)
ha[i] = make([]string, nreplicas)
for j := 0; j < nreplicas; j++ {
ha[i][j] = port(tag+"s", (i*nreplicas)+j)
}
for j := 0; j < nreplicas; j++ {
// if sudden {
// var procAttr os.ProcAttr
// procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
// process, err := os.StartProcess(os.Args[1], os.Args[1:], &procAttr)
// if err != nil {
// println("start process failed:" + err.String())
// return
// }
// _, err = process.Wait(0)
// } else {
sa[i][j] = StartServer(gids[i], smh, ha[i], j, mbh)
sa[i][j].unreliable = unreliable
// }
}
}
clean := func() { cleanup(sa); mcleanup(sma); mbcleanup(mba) }
return smh, gids, ha, sa, clean, mbh
}
func setupb(tag string, unreliable bool, sudden bool) ([]string, []int64, [][]string, [][]*ShardKV, func(), []string) {
runtime.GOMAXPROCS(4)
const nmasters = 3
var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
var smh []string = make([]string, nmasters)
// defer mcleanup(sma)
for i := 0; i < nmasters; i++ {
smh[i] = port(tag+"m", i)
}
for i := 0; i < nmasters; i++ {
sma[i] = shardmaster.StartServer(smh, i)
}
var mba []*messagebroker.MBServer = make([]*messagebroker.MBServer, nmasters)
var mbh []string = make([]string, nmasters)
for i := 0; i < nmasters; i++ {
mbh[i] = port(tag+"messagebroker", i)
}
for i := 0; i < nmasters; i++ {
mba[i] = messagebroker.StartServer(mbh, i)
}
const ngroups = 3 // replica groups
const nreplicas = 3 // servers per group
gids := make([]int64, ngroups) // each group ID
ha := make([][]string, ngroups) // ShardKV ports, [group][replica]
sa := make([][]*ShardKV, ngroups) // ShardKVs
// defer cleanup(sa)
for i := 0; i < ngroups; i++ {
gids[i] = int64(i + 100)
sa[i] = make([]*ShardKV, nreplicas)
ha[i] = make([]string, nreplicas)
for j := 0; j < nreplicas; j++ {
ha[i][j] = port(tag+"s", (i*nreplicas)+j)
}
for j := 0; j < nreplicas; j++ {
sa[i][j] = StartServer(gids[i], smh, ha[i], j, mbh)
sa[i][j].unreliable = unreliable
}
}
clean := func() { cleanup(sa); mcleanup(sma); mbcleanup(mba) }
return smh, gids, ha, sa, clean, mbh
}
func (cfg *config) StartMasterServer(i int) {
// ClientEnds to talk to other master replicas.
ends := make([]*labrpc.ClientEnd, cfg.nmasters)
for j := 0; j < cfg.nmasters; j++ {
endname := randstring(20)
ends[j] = cfg.net.MakeEnd(endname)
cfg.net.Connect(endname, cfg.mastername(j))
cfg.net.Enable(endname, true)
}
p := raft.MakePersister()
cfg.masterservers[i] = shardmaster.StartServer(ends, i, p)
msvc := labrpc.MakeService(cfg.masterservers[i])
rfsvc := labrpc.MakeService(cfg.masterservers[i].Raft())
srv := labrpc.MakeServer()
srv.AddService(msvc)
srv.AddService(rfsvc)
cfg.net.AddServer(cfg.mastername(i), srv)
}
func setup(t *testing.T, tag string, unreliable bool) *tCluster {
runtime.GOMAXPROCS(4)
const nmasters = 3
const ngroups = 3 // replica groups
const nreplicas = 3 // servers per group
tc := &tCluster{}
tc.t = t
tc.masters = make([]*shardmaster.ShardMaster, nmasters)
tc.masterports = make([]string, nmasters)
for i := 0; i < nmasters; i++ {
tc.masterports[i] = port(tag+"m", i)
}
for i := 0; i < nmasters; i++ {
tc.masters[i] = shardmaster.StartServer(tc.masterports, i)
}
tc.mck = tc.shardclerk()
tc.groups = make([]*tGroup, ngroups)
for i := 0; i < ngroups; i++ {
tc.groups[i] = &tGroup{}
tc.groups[i].gid = int64(i + 100)
tc.groups[i].servers = make([]*ShardKV, nreplicas)
tc.groups[i].ports = make([]string, nreplicas)
for j := 0; j < nreplicas; j++ {
tc.groups[i].ports[j] = port(tag+"s", (i*nreplicas)+j)
}
for j := 0; j < nreplicas; j++ {
tc.start1(i, j, unreliable)
}
}
// return smh, gids, ha, sa, clean
return tc
}
func setupmem(tag string, unreliable bool) ([]string, []int64, [][]string, [][]*ShardKV, func()) {
runtime.GOMAXPROCS(4)
const nmasters = 3
var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
var smh []string = make([]string, nmasters)
// defer mcleanup(sma)
for i := 0; i < nmasters; i++ {
smh[i] = port(tag+"m", i)
}
for i := 0; i < nmasters; i++ {
sma[i] = shardmaster.StartServer(smh, i)
}
//
const ngroups = 3 // replica groups
const nreplicas = 3 // servers per group
gids := make([]int64, ngroups) // each group ID
ha := make([][]string, ngroups) // ShardKV ports, [group][replica]
sa := make([][]*ShardKV, ngroups) // ShardKVs
// defer cleanup(sa)
for i := 0; i < ngroups; i++ {
gids[i] = int64(i + 100)
sa[i] = make([]*ShardKV, nreplicas)
ha[i] = make([]string, nreplicas)
for j := 0; j < nreplicas; j++ {
ha[i][j] = port(tag+"s", (i*nreplicas)+j)
}
for j := 0; j < nreplicas; j++ {
sa[i][j] = StartServer(gids[i], smh, ha[i], j)
sa[i][j].SetMemory(200)
sa[i][j].storage.Clear()
sa[i][j].unreliable = unreliable
}
}
clean := func() { cleanup(sa); mcleanup(sma) }
return smh, gids, ha, sa, clean
}
func TestNetwork(t *testing.T) {
runtime.GOMAXPROCS(4)
paxos.Network = "tcp" // set connection types to tcp
shardmaster.Network = "tcp"
shardkv.Network = "tcp"
// Start shardmster
fmt.Printf("Starting shardmaster...\n")
smHosts := make([]string, N)
for i := 0; i < N; i++ {
smHosts[i] = fmt.Sprintf("%s:%d", SERVERS[i], PORT0)
}
sm := shardmaster.StartServer(smHosts, Index)
// Start ShardKV server
fmt.Printf("Starting shard server...\n")
gids := make([]int64, N)
skvHosts := make([][]string, N)
skvs := make([]*shardkv.ShardKV, N)
for i := 0; i < N; i++ {
gids[i] = int64(100 + i)
skvHosts[i] = make([]string, N)
for j := 0; j < N; j++ {
skvHosts[i][j] = fmt.Sprintf("%s:%d", SERVERS[j], PORT0+1+i)
}
skvs[i] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], Index)
}
// Start shardmaster clerk, if this is the first machine.
fmt.Printf("Starting shardmaster clerk...\n")
if Index == 0 {
smClerk := shardmaster.MakeClerk(smHosts)
for i := 0; i < N; i++ {
smClerk.Join(gids[i], skvHosts[i])
}
}
// Start client
fmt.Printf("Starting client...\n")
client := shardkv.MakeClerk(smHosts)
// Make requests
time.Sleep(5000 * time.Millisecond)
startTime := time.Now().UnixNano()
for i := 0; i < NumOperations; i++ {
client.Put("a", "x")
client.Get("a")
}
endTime := time.Now().UnixNano()
latency := elapsed(startTime, endTime) / float64(NumOperations)
fmt.Printf("Total num operations: %d\n", NumOperations)
fmt.Printf("Latency: %.3f s\n", latency)
fmt.Printf("Note: a put() and get() query make up one operation.\n")
// Wait for every to finish, then clean up
fmt.Printf("Cleaning up...\n")
time.Sleep(10000 * time.Millisecond)
sm.Kill()
for i := 0; i < N; i++ {
skvs[i].Kill()
}
}
func main() {
var npaxos = flag.Int("npaxos", 3, "number of paxos instances")
var ngroups = flag.Int("ngroups", 3, "number of shard groups")
var nmasters = flag.Int("nmasters", 3, "number of shardmasters per shard group")
var nreplicas = flag.Int("nreplicas", 3, "number of kvshard replicas per group")
var clean = flag.Bool("clean", false, "clean the db")
flag.Parse()
args := flag.Args()
if len(args) < 1 {
fmt.Printf("Not enough arguments, must specify program type:\n" +
" paxos|shardmaster|shardkv\n")
os.Exit(1)
}
switch args[0] {
case "paxos":
fmt.Println("Attempting to start paxos server...")
if *clean {
cleanDB("paxos")
}
peers := test.GetPaxos(*npaxos)
me := whoami(peers)
if me == -1 {
fmt.Println("Host didn't find own IP in peer list! Exiting!")
os.Exit(1)
}
paxos.Make(peers, me, nil, network, "somedbtag")
fmt.Println("peers: ", peers)
fmt.Println("me: ", me)
fmt.Printf("Started paxos server.\n")
case "shardmaster":
fmt.Println("Attempting to start shardmaster server...")
if *clean {
cleanDB("shardmaster")
}
peers, _ := test.GetShardmasters(*nmasters, *ngroups)
me := whoami(peers)
if me == -1 {
fmt.Println("Host didn't find own IP in peer list! Exiting!")
os.Exit(1)
}
shardmaster.StartServer(peers, me, network)
fmt.Println("peers: ", peers)
fmt.Println("me: ", me)
fmt.Printf("Started shardmaster.\n")
case "shardkv":
fmt.Println("------------------------------------------")
fmt.Println(getIP(), "attempting to start shardkv server...")
//---------Default Shardmaster Servers------------
//group 100: 10.0.0.101
//group 101: 10.0.0.102
//group 102: 10.0.0.103
//-----------Default ShardKV Servers--------------
//group 100: 10.0.0.104, 10.0.0.105, 10.0.0.106
//group 101: 10.0.0.107, 10.0.0.108, 10.0.0.109
//group 102: 10.0.0.110, 10.0.0.111, 10.0.0.112
metapeers, masters, _ := test.GetShardkvs(*nreplicas, *nmasters, *ngroups)
fmt.Printf("peers: %v\n", metapeers)
me := whoami(masters)
if me != -1 {
fmt.Println("Starting shardmaster instead.")
if *clean {
cleanDB("shardmaster")
}
shardmaster.StartServer(masters, me, network)
fmt.Printf("peers: %v\n", masters)
fmt.Printf("me: %d\n", me)
fmt.Println("Success!")
select {}
}
var gid int64
peers := make([]string, *ngroups)
for i, v := range metapeers {
peers = v
me = whoami(v)
gid = int64(100 + i)
if me != -1 {
break
}
}
if me == -1 {
fmt.Printf("Exiting! Host didn't find own IP %s in peer list: %v! "+
"or masters: %v\n",
getIP(), metapeers, masters)
os.Exit(1)
}
if *clean {
cleanDB("shardkv")
}
fmt.Println("masters:", masters)
fmt.Printf("peers: %v\n", peers)
fmt.Printf("me: %d, gid :%d\n", me, gid)
shardkv.StartServer(gid, masters, peers, me, network)
default:
fmt.Printf("Invalid program type, choose one:" +
" paxos|shardmaster|shardkv")
os.Exit(1)
}
select {}
}