// Start all servers/clients on local machine.
// Returns a system and a clean function
func localSetup(name string, args SetupArgs) (System, func()) {
runtime.GOMAXPROCS(args.NumProcesses)
// Start shardmasters
smHosts := make([]string, args.NumMasters)
sms := make([]*shardmaster.ShardMaster, args.NumMasters)
for i := 0; i < args.NumMasters; i++ {
smHosts[i] = port(name+"m", i)
sms[i] = shardmaster.StartServer(smHosts, i)
}
smClerk := shardmaster.MakeClerk(smHosts)
// Start ShardKV servers
gids := make([]int64, args.NumGroups)
skvHosts := make([][]string, args.NumGroups)
skvs := make([][]*shardkv.ShardKV, args.NumGroups)
for i := 0; i < args.NumGroups; i++ {
gids[i] = int64(100 + i)
skvHosts[i] = make([]string, args.NumReplicas)
skvs[i] = make([]*shardkv.ShardKV, args.NumReplicas)
for j := 0; j < args.NumReplicas; j++ {
skvHosts[i][j] = port(name+"s", i*args.NumReplicas+j)
skvs[i][j] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], j)
}
smClerk.Join(gids[i], skvHosts[i])
}
// Start clients
clerks := make([]*shardkv.Clerk, args.NumClients)
for i := 0; i < args.NumClients; i++ {
clerks[i] = shardkv.MakeClerk(smHosts)
}
system := System{
ShardMasterHosts: smHosts,
ShardMasters: sms,
ShardMasterClerk: smClerk,
GroupIDs: gids,
ShardKVHosts: skvHosts,
ShardKVs: skvs,
Clerks: clerks,
}
clean := func() {
for i := 0; i < args.NumMasters; i++ {
sms[i].Kill()
}
for i := 0; i < args.NumGroups; i++ {
for j := 0; j < args.NumReplicas; j++ {
skvs[i][j].Kill()
}
}
}
return system, clean
}
func setup(tag string, unreliable bool, sudden bool) ([]string, []int64, [][]string, [][]*shardkv.ShardKV, func(), []string) {
runtime.GOMAXPROCS(4)
const nmasters = 3
var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
var smh []string = make([]string, nmasters)
// defer mcleanup(sma)
for i := 0; i < nmasters; i++ {
smh[i] = port(tag+"m", i)
}
for i := 0; i < nmasters; i++ {
sma[i] = shardmaster.StartServer(smh, i)
}
var mba []*messagebroker.MBServer = make([]*messagebroker.MBServer, nmasters)
var mbh []string = make([]string, nmasters)
for i := 0; i < nmasters; i++ {
mbh[i] = port(tag+"messagebroker", i)
}
for i := 0; i < nmasters; i++ {
mba[i] = messagebroker.StartServer(mbh, i)
}
const ngroups = 3 // replica groups
const nreplicas = 3 // servers per group
gids := make([]int64, ngroups) // each group ID
ha := make([][]string, ngroups) // ShardKV ports, [group][replica]
sa := make([][]*shardkv.ShardKV, ngroups) // ShardKVs
// defer cleanup(sa)
for i := 0; i < ngroups; i++ {
gids[i] = int64(i + 100)
sa[i] = make([]*shardkv.ShardKV, nreplicas)
ha[i] = make([]string, nreplicas)
for j := 0; j < nreplicas; j++ {
ha[i][j] = port(tag+"s", (i*nreplicas)+j)
}
for j := 0; j < nreplicas; j++ {
sa[i][j] = shardkv.StartServer(gids[i], smh, ha[i], j, mbh)
}
}
clean := func() { cleanup(sa); mcleanup(sma); mbcleanup(mba) }
return smh, gids, ha, sa, clean, mbh
}
func TestNetwork(t *testing.T) {
runtime.GOMAXPROCS(4)
paxos.Network = "tcp" // set connection types to tcp
shardmaster.Network = "tcp"
shardkv.Network = "tcp"
// Start shardmster
fmt.Printf("Starting shardmaster...\n")
smHosts := make([]string, N)
for i := 0; i < N; i++ {
smHosts[i] = fmt.Sprintf("%s:%d", SERVERS[i], PORT0)
}
sm := shardmaster.StartServer(smHosts, Index)
// Start ShardKV server
fmt.Printf("Starting shard server...\n")
gids := make([]int64, N)
skvHosts := make([][]string, N)
skvs := make([]*shardkv.ShardKV, N)
for i := 0; i < N; i++ {
gids[i] = int64(100 + i)
skvHosts[i] = make([]string, N)
for j := 0; j < N; j++ {
skvHosts[i][j] = fmt.Sprintf("%s:%d", SERVERS[j], PORT0+1+i)
}
skvs[i] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], Index)
}
// Start shardmaster clerk, if this is the first machine.
fmt.Printf("Starting shardmaster clerk...\n")
if Index == 0 {
smClerk := shardmaster.MakeClerk(smHosts)
for i := 0; i < N; i++ {
smClerk.Join(gids[i], skvHosts[i])
}
}
// Start client
fmt.Printf("Starting client...\n")
client := shardkv.MakeClerk(smHosts)
// Make requests
time.Sleep(5000 * time.Millisecond)
startTime := time.Now().UnixNano()
for i := 0; i < NumOperations; i++ {
client.Put("a", "x")
client.Get("a")
}
endTime := time.Now().UnixNano()
latency := elapsed(startTime, endTime) / float64(NumOperations)
fmt.Printf("Total num operations: %d\n", NumOperations)
fmt.Printf("Latency: %.3f s\n", latency)
fmt.Printf("Note: a put() and get() query make up one operation.\n")
// Wait for every to finish, then clean up
fmt.Printf("Cleaning up...\n")
time.Sleep(10000 * time.Millisecond)
sm.Kill()
for i := 0; i < N; i++ {
skvs[i].Kill()
}
}
func main() {
var npaxos = flag.Int("npaxos", 3, "number of paxos instances")
var ngroups = flag.Int("ngroups", 3, "number of shard groups")
var nmasters = flag.Int("nmasters", 3, "number of shardmasters per shard group")
var nreplicas = flag.Int("nreplicas", 3, "number of kvshard replicas per group")
var clean = flag.Bool("clean", false, "clean the db")
flag.Parse()
args := flag.Args()
if len(args) < 1 {
fmt.Printf("Not enough arguments, must specify program type:\n" +
" paxos|shardmaster|shardkv\n")
os.Exit(1)
}
switch args[0] {
case "paxos":
fmt.Println("Attempting to start paxos server...")
if *clean {
cleanDB("paxos")
}
peers := test.GetPaxos(*npaxos)
me := whoami(peers)
if me == -1 {
fmt.Println("Host didn't find own IP in peer list! Exiting!")
os.Exit(1)
}
paxos.Make(peers, me, nil, network, "somedbtag")
fmt.Println("peers: ", peers)
fmt.Println("me: ", me)
fmt.Printf("Started paxos server.\n")
case "shardmaster":
fmt.Println("Attempting to start shardmaster server...")
if *clean {
cleanDB("shardmaster")
}
peers, _ := test.GetShardmasters(*nmasters, *ngroups)
me := whoami(peers)
if me == -1 {
fmt.Println("Host didn't find own IP in peer list! Exiting!")
os.Exit(1)
}
shardmaster.StartServer(peers, me, network)
fmt.Println("peers: ", peers)
fmt.Println("me: ", me)
fmt.Printf("Started shardmaster.\n")
case "shardkv":
fmt.Println("------------------------------------------")
fmt.Println(getIP(), "attempting to start shardkv server...")
//---------Default Shardmaster Servers------------
//group 100: 10.0.0.101
//group 101: 10.0.0.102
//group 102: 10.0.0.103
//-----------Default ShardKV Servers--------------
//group 100: 10.0.0.104, 10.0.0.105, 10.0.0.106
//group 101: 10.0.0.107, 10.0.0.108, 10.0.0.109
//group 102: 10.0.0.110, 10.0.0.111, 10.0.0.112
metapeers, masters, _ := test.GetShardkvs(*nreplicas, *nmasters, *ngroups)
fmt.Printf("peers: %v\n", metapeers)
me := whoami(masters)
if me != -1 {
fmt.Println("Starting shardmaster instead.")
if *clean {
cleanDB("shardmaster")
}
shardmaster.StartServer(masters, me, network)
fmt.Printf("peers: %v\n", masters)
fmt.Printf("me: %d\n", me)
fmt.Println("Success!")
select {}
}
var gid int64
peers := make([]string, *ngroups)
for i, v := range metapeers {
peers = v
me = whoami(v)
gid = int64(100 + i)
if me != -1 {
break
}
}
if me == -1 {
fmt.Printf("Exiting! Host didn't find own IP %s in peer list: %v! "+
"or masters: %v\n",
getIP(), metapeers, masters)
os.Exit(1)
}
if *clean {
cleanDB("shardkv")
}
fmt.Println("masters:", masters)
fmt.Printf("peers: %v\n", peers)
fmt.Printf("me: %d, gid :%d\n", me, gid)
shardkv.StartServer(gid, masters, peers, me, network)
default:
fmt.Printf("Invalid program type, choose one:" +
" paxos|shardmaster|shardkv")
os.Exit(1)
}
select {}
}