Example #1
0
func main() {
	log.SetFlags(log.Lshortfile)
	flag.Parse()
	log.Println("RUNNING DEPLOY2DETER WITH RATE:", rate)
	os.MkdirAll("remote", 0777)
	var wg sync.WaitGroup
	// start building the necessary packages
	packages := []string{"../logserver", "../timeclient", "../exec", "../forkexec", "../deter"}
	for _, p := range packages {
		wg.Add(1)
		if p == "../deter" {
			go func(p string) {
				defer wg.Done()
				// the users node has a 386 FreeBSD architecture
				err := cliutils.Build(p, "386", "freebsd")
				if err != nil {
					log.Fatal(err)
				}
			}(p)
			continue
		}
		go func(p string) {
			defer wg.Done()
			// deter has an amd64, linux architecture
			err := cliutils.Build(p, "amd64", "linux")
			if err != nil {
				log.Fatal(err)
			}
		}(p)
	}
	// killssh processes on users
	cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net", "killall ssh scp deter 2>/dev/null 1>/dev/null")

	// parse the hosts.txt file to create a separate list (and file)
	// of physical nodes and virtual nodes. Such that each host on line i, in phys.txt
	// corresponds to each host on line i, in virt.txt.
	physVirt, err := cliutils.ReadLines("hosts.txt")
	phys := make([]string, 0, len(physVirt)/2)
	virt := make([]string, 0, len(physVirt)/2)
	for i := 0; i < len(physVirt); i += 2 {
		phys = append(phys, physVirt[i])
		virt = append(virt, physVirt[i+1])
	}
	nloggers := 3
	// only use the number of machines that we need
	phys = phys[:nmachs+nloggers]
	virt = virt[:nmachs+nloggers]
	physOut := strings.Join(phys, "\n")
	virtOut := strings.Join(virt, "\n")

	// phys.txt and virt.txt only contain the number of machines that we need
	err = ioutil.WriteFile("remote/phys.txt", []byte(physOut), 0666)
	if err != nil {
		log.Fatal("failed to write physical nodes file", err)
	}

	err = ioutil.WriteFile("remote/virt.txt", []byte(virtOut), 0666)
	if err != nil {
		log.Fatal("failed to write virtual nodes file", err)
	}

	masterLogger := phys[0]
	// slaveLogger1 := phys[1]
	// slaveLogger2 := phys[2]
	virt = virt[3:]
	phys = phys[3:]
	t, hostnames, depth, err := graphs.TreeFromList(virt, hpn, bf)
	log.Println("DEPTH:", depth)
	log.Println("TOTAL HOSTS:", len(hostnames))

	// wait for the build to finish
	wg.Wait()

	// copy the logserver directory to the current directory
	err = exec.Command("rsync", "-au", "../logserver", "remote/").Run()
	if err != nil {
		log.Fatal("error rsyncing logserver directory into remote directory:", err)
	}
	err = exec.Command("rsync", "-au", "remote/phys.txt", "remote/virt.txt", "remote/logserver/").Run()
	if err != nil {
		log.Fatal("error rsyncing phys, virt, and remote/logserver:", err)
	}
	err = os.Rename("logserver", "remote/logserver/logserver")
	if err != nil {
		log.Fatal("error renaming logserver:", err)
	}

	b, err := json.Marshal(t)
	if err != nil {
		log.Fatal("unable to generate tree from list")
	}
	err = ioutil.WriteFile("remote/logserver/cfg.json", b, 0660)
	if err != nil {
		log.Fatal("unable to write configuration file")
	}

	// NOTE: now remote/logserver is ready for transfer
	// it has logserver/ folder, binary, and cfg.json, and phys.txt, virt.txt

	// generate the configuration file from the tree
	cf := config.ConfigFromTree(t, hostnames)
	cfb, err := json.Marshal(cf)
	err = ioutil.WriteFile("remote/cfg.json", cfb, 0666)
	if err != nil {
		log.Fatal(err)
	}

	// scp the files that we need over to the boss node
	files := []string{"timeclient", "exec", "forkexec", "deter"}
	for _, f := range files {
		cmd := exec.Command("rsync", "-au", f, "remote/")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err := cmd.Run()
		if err != nil {
			log.Fatal("error unable to rsync file into remote directory:", err)
		}
	}
	err = cliutils.Rsync("dvisher", "users.isi.deterlab.net", "remote", "")
	if err != nil {
		log.Fatal(err)
	}
	killssh := exec.Command("pkill", "-f", "ssh -t -t")
	killssh.Stdout = os.Stdout
	killssh.Stderr = os.Stderr
	err = killssh.Run()
	if err != nil {
		log.Print(err)
	}

	// setup port forwarding for viewing log server
	// ssh -L 8080:pcXXX:80 [email protected]
	// ssh [email protected] -L 8118:somenode.experiment.YourClass.isi.deterlab.net:80
	fmt.Println("setup port forwarding for master logger: ", masterLogger)
	cmd := exec.Command(
		"ssh",
		"-t",
		"-t",
		"*****@*****.**",
		"-L",
		"8080:"+masterLogger+":10000")
	err = cmd.Start()
	if err != nil {
		log.Fatal("failed to setup portforwarding for logging server")
	}
	log.Println("runnning deter with nmsgs:", nmsgs)
	// run the deter lab boss nodes process
	// it will be responsible for forwarding the files and running the individual
	// timestamping servers
	log.Fatal(cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net",
		"GOMAXPROCS=8 remote/deter -nmsgs="+strconv.Itoa(nmsgs)+
			" -hpn="+strconv.Itoa(hpn)+
			" -bf="+strconv.Itoa(bf)+
			" -rate="+strconv.Itoa(rate)+
			" -rounds="+strconv.Itoa(rounds)+
			" -debug="+strconv.FormatBool(debug)+
			" -failures="+strconv.Itoa(failures)+
			" -rfail="+strconv.Itoa(rFail)+
			" -ffail="+strconv.Itoa(fFail)+
			" -test_connect="+strconv.FormatBool(testConnect)+
			" -app="+app+
			" -suite="+suite+
			" -kill="+strconv.FormatBool(kill)))
}
Example #2
0
func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)
	fmt.Println("running deter with nmsgs:", nmsgs, rate, rounds)

	virt, err := cliutils.ReadLines("remote/virt.txt")
	if err != nil {
		log.Fatal(err)
	}
	phys, err := cliutils.ReadLines("remote/phys.txt")
	if err != nil {
		log.Fatal(err)
	}
	vpmap := make(map[string]string)
	for i := range virt {
		vpmap[virt[i]] = phys[i]
	}
	// kill old processes
	var wg sync.WaitGroup
	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.SshRun("", h, "sudo killall exec logserver timeclient scp ssh 2>/dev/null >/dev/null")
			time.Sleep(1 * time.Second)
			cliutils.SshRun("", h, "sudo killall forkexec 2>/dev/null >/dev/null")
		}(h)
	}
	wg.Wait()

	if kill {
		return
	}

	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.Rsync("", h, "remote", "")
		}(h)
	}
	wg.Wait()

	nloggers := 3
	masterLogger := phys[0]
	slaveLogger1 := phys[1]
	slaveLogger2 := phys[2]
	loggers := []string{masterLogger, slaveLogger1, slaveLogger2}

	phys = phys[nloggers:]
	virt = virt[nloggers:]

	// Read in and parse the configuration file
	file, err := ioutil.ReadFile("remote/cfg.json")
	if err != nil {
		log.Fatal("deter.go: error reading configuration file: %v\n", err)
	}
	log.Println("cfg file:", string(file))
	var cf config.ConfigFile
	err = json.Unmarshal(file, &cf)
	if err != nil {
		log.Fatal("unable to unmarshal config.ConfigFile:", err)
	}

	hostnames := cf.Hosts

	depth := graphs.Depth(cf.Tree)
	var random_leaf string
	cf.Tree.TraverseTree(func(t *graphs.Tree) {
		if random_leaf != "" {
			return
		}
		if len(t.Children) == 0 {
			random_leaf = t.Name
		}
	})

	rootname = hostnames[0]

	log.Println("depth of tree:", depth)

	// mapping from physical node name to the timestamp servers that are running there
	// essentially a reverse mapping of vpmap except ports are also used
	physToServer := make(map[string][]string)
	for _, virt := range hostnames {
		v, _, _ := net.SplitHostPort(virt)
		p := vpmap[v]
		ss := physToServer[p]
		ss = append(ss, virt)
		physToServer[p] = ss
	}

	// start up the logging server on the final host at port 10000
	fmt.Println("starting up logserver")
	// start up the master logger
	loggerports := make([]string, len(loggers))
	for i, logger := range loggers {
		loggerport := logger + ":10000"
		loggerports[i] = loggerport
		// redirect to the master logger
		master := masterLogger + ":10000"
		// if this is the master logger than don't set the master to anything
		if loggerport == masterLogger+":10000" {
			master = ""
		}

		go cliutils.SshRunStdout("", logger, "cd remote/logserver; sudo ./logserver -addr="+loggerport+
			" -hosts="+strconv.Itoa(len(hostnames))+
			" -depth="+strconv.Itoa(depth)+
			" -bf="+bf+
			" -hpn="+hpn+
			" -nmsgs="+nmsgs+
			" -rate="+strconv.Itoa(rate)+
			" -master="+master)
	}

	// wait a little bit for the logserver to start up
	time.Sleep(5 * time.Second)
	fmt.Println("starting time clients")

	// start up one timeclient per physical machine
	// it requests timestamps from all the servers on that machine
	i := 0
	for p, ss := range physToServer {
		if len(ss) == 0 {
			continue
		}
		servers := strings.Join(ss, ",")
		go func(i int, p string) {
			_, err := cliutils.SshRun("", p, "cd remote; sudo ./timeclient -nmsgs="+nmsgs+
				" -name=client@"+p+
				" -server="+servers+
				" -logger="+loggerports[i]+
				" -debug="+debug+
				" -rate="+strconv.Itoa(rate))
			if err != nil {
				log.Println(err)
			}
		}(i, p)
		i = (i + 1) % len(loggerports)
	}
	rootwait := strconv.Itoa(10)
	for phys, virts := range physToServer {
		if len(virts) == 0 {
			continue
		}
		log.Println("starting timestamper")
		cmd := GenExecCmd(rFail, fFail, failures, phys, virts, loggerports[i], rootwait, random_leaf)
		i = (i + 1) % len(loggerports)
		wg.Add(1)
		//time.Sleep(500 * time.Millisecond)
		go func(phys, cmd string) {
			//log.Println("running on ", phys, cmd)
			defer wg.Done()
			err := cliutils.SshRunStdout("", phys, cmd)
			if err != nil {
				log.Fatal("ERROR STARTING TIMESTAMPER:", err)
			}
		}(phys, cmd)

	}
	// wait for the servers to finish before stopping
	wg.Wait()
	time.Sleep(10 * time.Minute)
}