func setup(t *testing.T, tag string, ngroups int, nreplicas int, unreliable bool) *tCluster {
	runtime.GOMAXPROCS(4)

	// compile ../main/diskvd.go
	cmd := exec.Command("go", "build", "diskvd.go")
	cmd.Dir = "../main"
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		t.Fatalf("could not compile ../main/diskvd.go: %v", err)
	}

	const nmasters = 3

	tc := &tCluster{}
	tc.t = t
	tc.unreliable = unreliable

	tc.dir = "/var/tmp/824-"
	tc.dir += strconv.Itoa(os.Getuid()) + "/"
	os.Mkdir(tc.dir, 0777)
	tc.dir += "lab5-" + tag + "/"
	os.RemoveAll(tc.dir)
	os.Mkdir(tc.dir, 0777)

	tc.masters = make([]*shardmaster.ShardMaster, nmasters)
	tc.masterports = make([]string, nmasters)
	for i := 0; i < nmasters; i++ {
		tc.masterports[i] = tc.newport()
	}
	log.SetOutput(ioutil.Discard) // suppress method errors &c
	for i := 0; i < nmasters; i++ {
		tc.masters[i] = shardmaster.StartServer(tc.masterports, i)
	}
	log.SetOutput(os.Stdout) // re-enable error output.
	tc.mck = tc.shardclerk()

	tc.groups = make([]*tGroup, ngroups)

	for i := 0; i < ngroups; i++ {
		g := &tGroup{}
		tc.groups[i] = g
		g.gid = int64(i + 100)
		g.servers = make([]*tServer, nreplicas)
		for j := 0; j < nreplicas; j++ {
			g.servers[j] = &tServer{}
			g.servers[j].port = tc.newport()
			g.servers[j].dir = tc.dir + randstring(12)
			if err := os.Mkdir(g.servers[j].dir, 0777); err != nil {
				t.Fatalf("Mkdir(%v): %v", g.servers[j].dir, err)
			}
		}
		for j := 0; j < nreplicas; j++ {
			tc.start1(i, j)
		}
	}

	// return smh, gids, ha, sa, clean
	return tc
}
Exemple #2
0
// Set up and start servers for shardkv groups and shardmaster
func setup(tag string, unreliable bool, numGroups int, numReplicas int) ([]string, []int64, [][]string, [][]*ShardKV, func(), func()) {
	runtime.GOMAXPROCS(4)

	const numMasters = 3
	var smServers []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, numMasters)
	var smPorts []string = make([]string, numMasters)
	// Start shardmaster servers
	for i := 0; i < numMasters; i++ {
		smPorts[i] = makePort(tag+"m", i)
	}
	for i := 0; i < numMasters; i++ {
		smServers[i] = shardmaster.StartServer(smPorts, i, false)
	}

	gids := make([]int64, numGroups)           // each group ID
	kvPorts := make([][]string, numGroups)     // ShardKV ports, [group][replica]
	kvServers := make([][]*ShardKV, numGroups) // ShardKVs
	// Start numReplicas shardkv servers for each of numGroups groups
	for i := 0; i < numGroups; i++ {
		gids[i] = int64(i + 100)
		kvServers[i] = make([]*ShardKV, numReplicas)
		kvPorts[i] = make([]string, numReplicas)
		for j := 0; j < numReplicas; j++ {
			kvPorts[i][j] = makePort(tag+"s", (i*numReplicas)+j)
		}
		for j := 0; j < numReplicas; j++ {
			kvServers[i][j] = StartServer(gids[i], smPorts, kvPorts[i], j, false)
			kvServers[i][j].unreliable = unreliable
		}
	}

	clean := func() { shardkvCleanup(kvServers, false); smCleanup(smServers, false) }
	cleanSaveDisk := func() { shardkvCleanup(kvServers, true); smCleanup(smServers, true) }
	return smPorts, gids, kvPorts, kvServers, clean, cleanSaveDisk
}
Exemple #3
0
// Start all servers/clients on local machine.
// Returns a system and a clean function
func localSetup(name string, args SetupArgs) (System, func()) {
	runtime.GOMAXPROCS(args.NumProcesses)

	// Start shardmasters
	smHosts := make([]string, args.NumMasters)
	sms := make([]*shardmaster.ShardMaster, args.NumMasters)
	for i := 0; i < args.NumMasters; i++ {
		smHosts[i] = port(name+"m", i)
		sms[i] = shardmaster.StartServer(smHosts, i)
	}
	smClerk := shardmaster.MakeClerk(smHosts)

	// Start ShardKV servers
	gids := make([]int64, args.NumGroups)
	skvHosts := make([][]string, args.NumGroups)
	skvs := make([][]*shardkv.ShardKV, args.NumGroups)
	for i := 0; i < args.NumGroups; i++ {
		gids[i] = int64(100 + i)
		skvHosts[i] = make([]string, args.NumReplicas)
		skvs[i] = make([]*shardkv.ShardKV, args.NumReplicas)
		for j := 0; j < args.NumReplicas; j++ {
			skvHosts[i][j] = port(name+"s", i*args.NumReplicas+j)
			skvs[i][j] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], j)
		}
		smClerk.Join(gids[i], skvHosts[i])
	}

	// Start clients
	clerks := make([]*shardkv.Clerk, args.NumClients)
	for i := 0; i < args.NumClients; i++ {
		clerks[i] = shardkv.MakeClerk(smHosts)
	}

	system := System{
		ShardMasterHosts: smHosts,
		ShardMasters:     sms,
		ShardMasterClerk: smClerk,
		GroupIDs:         gids,
		ShardKVHosts:     skvHosts,
		ShardKVs:         skvs,
		Clerks:           clerks,
	}

	clean := func() {
		for i := 0; i < args.NumMasters; i++ {
			sms[i].Kill()
		}
		for i := 0; i < args.NumGroups; i++ {
			for j := 0; j < args.NumReplicas; j++ {
				skvs[i][j].Kill()
			}
		}
	}

	return system, clean
}
Exemple #4
0
func setup(tag string, unreliable bool, sudden bool) ([]string, []int64, [][]string, [][]*ShardKV, func(), []string) {
  runtime.GOMAXPROCS(4)

  const nmasters = 3
  var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
  var smh []string = make([]string, nmasters)
  // defer mcleanup(sma)
  for i := 0; i < nmasters; i++ {
    smh[i] = port(tag+"m", i)
  }
  for i := 0; i < nmasters; i++ {
    sma[i] = shardmaster.StartServer(smh, i)
  }

  var mba []*messagebroker.MBServer = make([]*messagebroker.MBServer, nmasters)
  var mbh []string = make([]string, nmasters)
  for i := 0; i < nmasters; i++ {
    mbh[i] = port(tag+"messagebroker", i)
  }
  for i := 0; i < nmasters; i++ {
    mba[i] = messagebroker.StartServer(mbh, i)
  }

  const ngroups = 3                 // replica groups
  const nreplicas = 3               // servers per group
  gids := make([]int64, ngroups)    // each group ID
  ha := make([][]string, ngroups)   // ShardKV ports, [group][replica]
  sa := make([][]*ShardKV, ngroups) // ShardKVs
  // defer cleanup(sa)
  for i := 0; i < ngroups; i++ {
    gids[i] = int64(i + 100)
    sa[i] = make([]*ShardKV, nreplicas)
    ha[i] = make([]string, nreplicas)
    for j := 0; j < nreplicas; j++ {
      ha[i][j] = port(tag+"s", (i*nreplicas)+j)
    }
    for j := 0; j < nreplicas; j++ {
      // if sudden {
      //   var procAttr os.ProcAttr
      //   procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
      //   process, err := os.StartProcess(os.Args[1], os.Args[1:], &procAttr)
      //   if err != nil {
      //     println("start process failed:" + err.String())
      //     return
      //   }
      //   _, err = process.Wait(0)
      // } else {
      sa[i][j] = StartServer(gids[i], smh, ha[i], j, mbh)
      sa[i][j].unreliable = unreliable
      // }
    }
  }

  clean := func() { cleanup(sa); mcleanup(sma); mbcleanup(mba) }
  return smh, gids, ha, sa, clean, mbh
}
Exemple #5
0
func setupb(tag string, unreliable bool, sudden bool) ([]string, []int64, [][]string, [][]*ShardKV, func(), []string) {
	runtime.GOMAXPROCS(4)

	const nmasters = 3
	var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
	var smh []string = make([]string, nmasters)
	// defer mcleanup(sma)
	for i := 0; i < nmasters; i++ {
		smh[i] = port(tag+"m", i)
	}
	for i := 0; i < nmasters; i++ {
		sma[i] = shardmaster.StartServer(smh, i)
	}

	var mba []*messagebroker.MBServer = make([]*messagebroker.MBServer, nmasters)
	var mbh []string = make([]string, nmasters)
	for i := 0; i < nmasters; i++ {
		mbh[i] = port(tag+"messagebroker", i)
	}
	for i := 0; i < nmasters; i++ {
		mba[i] = messagebroker.StartServer(mbh, i)
	}

	const ngroups = 3                 // replica groups
	const nreplicas = 3               // servers per group
	gids := make([]int64, ngroups)    // each group ID
	ha := make([][]string, ngroups)   // ShardKV ports, [group][replica]
	sa := make([][]*ShardKV, ngroups) // ShardKVs
	// defer cleanup(sa)
	for i := 0; i < ngroups; i++ {
		gids[i] = int64(i + 100)
		sa[i] = make([]*ShardKV, nreplicas)
		ha[i] = make([]string, nreplicas)
		for j := 0; j < nreplicas; j++ {
			ha[i][j] = port(tag+"s", (i*nreplicas)+j)
		}
		for j := 0; j < nreplicas; j++ {
			sa[i][j] = StartServer(gids[i], smh, ha[i], j, mbh)
			sa[i][j].unreliable = unreliable
		}
	}

	clean := func() { cleanup(sa); mcleanup(sma); mbcleanup(mba) }
	return smh, gids, ha, sa, clean, mbh
}
Exemple #6
0
func (cfg *config) StartMasterServer(i int) {
	// ClientEnds to talk to other master replicas.
	ends := make([]*labrpc.ClientEnd, cfg.nmasters)
	for j := 0; j < cfg.nmasters; j++ {
		endname := randstring(20)
		ends[j] = cfg.net.MakeEnd(endname)
		cfg.net.Connect(endname, cfg.mastername(j))
		cfg.net.Enable(endname, true)
	}

	p := raft.MakePersister()

	cfg.masterservers[i] = shardmaster.StartServer(ends, i, p)

	msvc := labrpc.MakeService(cfg.masterservers[i])
	rfsvc := labrpc.MakeService(cfg.masterservers[i].Raft())
	srv := labrpc.MakeServer()
	srv.AddService(msvc)
	srv.AddService(rfsvc)
	cfg.net.AddServer(cfg.mastername(i), srv)
}
func setup(t *testing.T, tag string, unreliable bool) *tCluster {
	runtime.GOMAXPROCS(4)

	const nmasters = 3
	const ngroups = 3   // replica groups
	const nreplicas = 3 // servers per group

	tc := &tCluster{}
	tc.t = t
	tc.masters = make([]*shardmaster.ShardMaster, nmasters)
	tc.masterports = make([]string, nmasters)

	for i := 0; i < nmasters; i++ {
		tc.masterports[i] = port(tag+"m", i)
	}
	for i := 0; i < nmasters; i++ {
		tc.masters[i] = shardmaster.StartServer(tc.masterports, i)
	}
	tc.mck = tc.shardclerk()

	tc.groups = make([]*tGroup, ngroups)

	for i := 0; i < ngroups; i++ {
		tc.groups[i] = &tGroup{}
		tc.groups[i].gid = int64(i + 100)
		tc.groups[i].servers = make([]*ShardKV, nreplicas)
		tc.groups[i].ports = make([]string, nreplicas)
		for j := 0; j < nreplicas; j++ {
			tc.groups[i].ports[j] = port(tag+"s", (i*nreplicas)+j)
		}
		for j := 0; j < nreplicas; j++ {
			tc.start1(i, j, unreliable)
		}
	}

	// return smh, gids, ha, sa, clean
	return tc
}
func setupmem(tag string, unreliable bool) ([]string, []int64, [][]string, [][]*ShardKV, func()) {
	runtime.GOMAXPROCS(4)

	const nmasters = 3
	var sma []*shardmaster.ShardMaster = make([]*shardmaster.ShardMaster, nmasters)
	var smh []string = make([]string, nmasters)
	// defer mcleanup(sma)
	for i := 0; i < nmasters; i++ {
		smh[i] = port(tag+"m", i)
	}
	for i := 0; i < nmasters; i++ {
		sma[i] = shardmaster.StartServer(smh, i)
	}
	//
	const ngroups = 3                 // replica groups
	const nreplicas = 3               // servers per group
	gids := make([]int64, ngroups)    // each group ID
	ha := make([][]string, ngroups)   // ShardKV ports, [group][replica]
	sa := make([][]*ShardKV, ngroups) // ShardKVs
	// defer cleanup(sa)
	for i := 0; i < ngroups; i++ {
		gids[i] = int64(i + 100)
		sa[i] = make([]*ShardKV, nreplicas)
		ha[i] = make([]string, nreplicas)
		for j := 0; j < nreplicas; j++ {
			ha[i][j] = port(tag+"s", (i*nreplicas)+j)
		}
		for j := 0; j < nreplicas; j++ {
			sa[i][j] = StartServer(gids[i], smh, ha[i], j)
			sa[i][j].SetMemory(200)
			sa[i][j].storage.Clear()
			sa[i][j].unreliable = unreliable
		}
	}
	clean := func() { cleanup(sa); mcleanup(sma) }
	return smh, gids, ha, sa, clean
}
func TestNetwork(t *testing.T) {
	runtime.GOMAXPROCS(4)
	paxos.Network = "tcp" // set connection types to tcp
	shardmaster.Network = "tcp"
	shardkv.Network = "tcp"

	// Start shardmster
	fmt.Printf("Starting shardmaster...\n")
	smHosts := make([]string, N)
	for i := 0; i < N; i++ {
		smHosts[i] = fmt.Sprintf("%s:%d", SERVERS[i], PORT0)
	}
	sm := shardmaster.StartServer(smHosts, Index)

	// Start ShardKV server
	fmt.Printf("Starting shard server...\n")
	gids := make([]int64, N)
	skvHosts := make([][]string, N)
	skvs := make([]*shardkv.ShardKV, N)
	for i := 0; i < N; i++ {
		gids[i] = int64(100 + i)
		skvHosts[i] = make([]string, N)
		for j := 0; j < N; j++ {
			skvHosts[i][j] = fmt.Sprintf("%s:%d", SERVERS[j], PORT0+1+i)
		}
		skvs[i] = shardkv.StartServer(gids[i], smHosts, skvHosts[i], Index)
	}

	// Start shardmaster clerk, if this is the first machine.
	fmt.Printf("Starting shardmaster clerk...\n")
	if Index == 0 {
		smClerk := shardmaster.MakeClerk(smHosts)
		for i := 0; i < N; i++ {
			smClerk.Join(gids[i], skvHosts[i])
		}
	}

	// Start client
	fmt.Printf("Starting client...\n")
	client := shardkv.MakeClerk(smHosts)

	// Make requests
	time.Sleep(5000 * time.Millisecond)
	startTime := time.Now().UnixNano()
	for i := 0; i < NumOperations; i++ {
		client.Put("a", "x")
		client.Get("a")
	}
	endTime := time.Now().UnixNano()

	latency := elapsed(startTime, endTime) / float64(NumOperations)

	fmt.Printf("Total num operations: %d\n", NumOperations)
	fmt.Printf("Latency: %.3f s\n", latency)
	fmt.Printf("Note: a put() and get() query make up one operation.\n")

	// Wait for every to finish, then clean up
	fmt.Printf("Cleaning up...\n")
	time.Sleep(10000 * time.Millisecond)
	sm.Kill()
	for i := 0; i < N; i++ {
		skvs[i].Kill()
	}
}
Exemple #10
0
func main() {
	var npaxos = flag.Int("npaxos", 3, "number of paxos instances")
	var ngroups = flag.Int("ngroups", 3, "number of shard groups")
	var nmasters = flag.Int("nmasters", 3, "number of shardmasters per shard group")
	var nreplicas = flag.Int("nreplicas", 3, "number of kvshard replicas per group")
	var clean = flag.Bool("clean", false, "clean the db")

	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		fmt.Printf("Not enough arguments, must specify program type:\n" +
			"   paxos|shardmaster|shardkv\n")
		os.Exit(1)
	}

	switch args[0] {
	case "paxos":
		fmt.Println("Attempting to start paxos server...")
		if *clean {
			cleanDB("paxos")
		}
		peers := test.GetPaxos(*npaxos)
		me := whoami(peers)
		if me == -1 {
			fmt.Println("Host didn't find own IP in peer list! Exiting!")
			os.Exit(1)
		}
		paxos.Make(peers, me, nil, network, "somedbtag")
		fmt.Println("peers: ", peers)
		fmt.Println("me: ", me)
		fmt.Printf("Started paxos server.\n")

	case "shardmaster":
		fmt.Println("Attempting to start shardmaster server...")
		if *clean {
			cleanDB("shardmaster")
		}
		peers, _ := test.GetShardmasters(*nmasters, *ngroups)
		me := whoami(peers)
		if me == -1 {
			fmt.Println("Host didn't find own IP in peer list! Exiting!")
			os.Exit(1)
		}
		shardmaster.StartServer(peers, me, network)
		fmt.Println("peers: ", peers)
		fmt.Println("me: ", me)
		fmt.Printf("Started shardmaster.\n")

	case "shardkv":
		fmt.Println("------------------------------------------")
		fmt.Println(getIP(), "attempting to start shardkv server...")
		//---------Default Shardmaster Servers------------
		//group 100: 10.0.0.101
		//group 101: 10.0.0.102
		//group 102: 10.0.0.103
		//-----------Default ShardKV Servers--------------
		//group 100: 10.0.0.104, 10.0.0.105, 10.0.0.106
		//group 101: 10.0.0.107, 10.0.0.108, 10.0.0.109
		//group 102: 10.0.0.110, 10.0.0.111, 10.0.0.112
		metapeers, masters, _ := test.GetShardkvs(*nreplicas, *nmasters, *ngroups)
		fmt.Printf("peers: %v\n", metapeers)

		me := whoami(masters)
		if me != -1 {
			fmt.Println("Starting shardmaster instead.")
			if *clean {
				cleanDB("shardmaster")
			}
			shardmaster.StartServer(masters, me, network)
			fmt.Printf("peers: %v\n", masters)
			fmt.Printf("me: %d\n", me)
			fmt.Println("Success!")
			select {}
		}
		var gid int64
		peers := make([]string, *ngroups)
		for i, v := range metapeers {
			peers = v
			me = whoami(v)
			gid = int64(100 + i)
			if me != -1 {
				break
			}
		}
		if me == -1 {
			fmt.Printf("Exiting! Host didn't find own IP %s in peer list: %v! "+
				"or masters: %v\n",
				getIP(), metapeers, masters)
			os.Exit(1)
		}
		if *clean {
			cleanDB("shardkv")
		}
		fmt.Println("masters:", masters)
		fmt.Printf("peers: %v\n", peers)
		fmt.Printf("me: %d, gid :%d\n", me, gid)
		shardkv.StartServer(gid, masters, peers, me, network)

	default:
		fmt.Printf("Invalid program type, choose one:" +
			"   paxos|shardmaster|shardkv")
		os.Exit(1)
	}
	select {}
}