func getDebugServers() []string {
// read in physical nodes and virtual nodes into global variables
phys, err := cliutils.ReadLines("phys.txt")
if err != nil {
log.Errorln(err)
}
virt, err := cliutils.ReadLines("virt.txt")
if err != nil {
log.Errorln(err)
}
// create mapping from virtual nodes to physical nodes
vpmap := make(map[string]string)
for i := range phys {
vpmap[virt[i]] = phys[i]
}
// now read in the hosttree to get a list of servers
cfg, e := ioutil.ReadFile("cfg.json")
if e != nil {
log.Fatal("Error Reading Configuration File:", e)
}
var tree graphs.Tree
json.Unmarshal(cfg, &tree)
debugServers := make([]string, 0, len(virt))
tree.TraverseTree(func(t *graphs.Tree) {
h, p, err := net.SplitHostPort(t.Name)
if err != nil {
log.Fatal("improperly formatted hostport:", err)
}
pn, _ := strconv.Atoi(p)
s := net.JoinHostPort(vpmap[h], strconv.Itoa(pn+2))
debugServers = append(debugServers, s)
})
return debugServers
}
func main() {
log.SetFlags(log.Lshortfile)
flag.Parse()
log.Println("RUNNING DEPLOY2DETER WITH RATE:", rate)
os.MkdirAll("remote", 0777)
var wg sync.WaitGroup
// start building the necessary packages
packages := []string{"../logserver", "../timeclient", "../exec", "../forkexec", "../deter"}
for _, p := range packages {
wg.Add(1)
if p == "../deter" {
go func(p string) {
defer wg.Done()
// the users node has a 386 FreeBSD architecture
err := cliutils.Build(p, "386", "freebsd")
if err != nil {
log.Fatal(err)
}
}(p)
continue
}
go func(p string) {
defer wg.Done()
// deter has an amd64, linux architecture
err := cliutils.Build(p, "amd64", "linux")
if err != nil {
log.Fatal(err)
}
}(p)
}
// killssh processes on users
cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net", "killall ssh scp deter 2>/dev/null 1>/dev/null")
// parse the hosts.txt file to create a separate list (and file)
// of physical nodes and virtual nodes. Such that each host on line i, in phys.txt
// corresponds to each host on line i, in virt.txt.
physVirt, err := cliutils.ReadLines("hosts.txt")
phys := make([]string, 0, len(physVirt)/2)
virt := make([]string, 0, len(physVirt)/2)
for i := 0; i < len(physVirt); i += 2 {
phys = append(phys, physVirt[i])
virt = append(virt, physVirt[i+1])
}
nloggers := 3
// only use the number of machines that we need
phys = phys[:nmachs+nloggers]
virt = virt[:nmachs+nloggers]
physOut := strings.Join(phys, "\n")
virtOut := strings.Join(virt, "\n")
// phys.txt and virt.txt only contain the number of machines that we need
err = ioutil.WriteFile("remote/phys.txt", []byte(physOut), 0666)
if err != nil {
log.Fatal("failed to write physical nodes file", err)
}
err = ioutil.WriteFile("remote/virt.txt", []byte(virtOut), 0666)
if err != nil {
log.Fatal("failed to write virtual nodes file", err)
}
masterLogger := phys[0]
// slaveLogger1 := phys[1]
// slaveLogger2 := phys[2]
virt = virt[3:]
phys = phys[3:]
t, hostnames, depth, err := graphs.TreeFromList(virt, hpn, bf)
log.Println("DEPTH:", depth)
log.Println("TOTAL HOSTS:", len(hostnames))
// wait for the build to finish
wg.Wait()
// copy the logserver directory to the current directory
err = exec.Command("rsync", "-au", "../logserver", "remote/").Run()
if err != nil {
log.Fatal("error rsyncing logserver directory into remote directory:", err)
}
err = exec.Command("rsync", "-au", "remote/phys.txt", "remote/virt.txt", "remote/logserver/").Run()
if err != nil {
log.Fatal("error rsyncing phys, virt, and remote/logserver:", err)
}
err = os.Rename("logserver", "remote/logserver/logserver")
if err != nil {
log.Fatal("error renaming logserver:", err)
}
b, err := json.Marshal(t)
if err != nil {
log.Fatal("unable to generate tree from list")
}
err = ioutil.WriteFile("remote/logserver/cfg.json", b, 0660)
if err != nil {
log.Fatal("unable to write configuration file")
}
// NOTE: now remote/logserver is ready for transfer
// it has logserver/ folder, binary, and cfg.json, and phys.txt, virt.txt
// generate the configuration file from the tree
cf := config.ConfigFromTree(t, hostnames)
cfb, err := json.Marshal(cf)
err = ioutil.WriteFile("remote/cfg.json", cfb, 0666)
if err != nil {
log.Fatal(err)
}
// scp the files that we need over to the boss node
files := []string{"timeclient", "exec", "forkexec", "deter"}
for _, f := range files {
cmd := exec.Command("rsync", "-au", f, "remote/")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
log.Fatal("error unable to rsync file into remote directory:", err)
}
}
err = cliutils.Rsync("dvisher", "users.isi.deterlab.net", "remote", "")
if err != nil {
log.Fatal(err)
}
killssh := exec.Command("pkill", "-f", "ssh -t -t")
killssh.Stdout = os.Stdout
killssh.Stderr = os.Stderr
err = killssh.Run()
if err != nil {
log.Print(err)
}
// setup port forwarding for viewing log server
// ssh -L 8080:pcXXX:80 [email protected]
// ssh [email protected] -L 8118:somenode.experiment.YourClass.isi.deterlab.net:80
fmt.Println("setup port forwarding for master logger: ", masterLogger)
cmd := exec.Command(
"ssh",
"-t",
"-t",
"*****@*****.**",
"-L",
"8080:"+masterLogger+":10000")
err = cmd.Start()
if err != nil {
log.Fatal("failed to setup portforwarding for logging server")
}
log.Println("runnning deter with nmsgs:", nmsgs)
// run the deter lab boss nodes process
// it will be responsible for forwarding the files and running the individual
// timestamping servers
log.Fatal(cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net",
"GOMAXPROCS=8 remote/deter -nmsgs="+strconv.Itoa(nmsgs)+
" -hpn="+strconv.Itoa(hpn)+
" -bf="+strconv.Itoa(bf)+
" -rate="+strconv.Itoa(rate)+
" -rounds="+strconv.Itoa(rounds)+
" -debug="+strconv.FormatBool(debug)+
" -failures="+strconv.Itoa(failures)+
" -rfail="+strconv.Itoa(rFail)+
" -ffail="+strconv.Itoa(fFail)+
" -test_connect="+strconv.FormatBool(testConnect)+
" -app="+app+
" -suite="+suite+
" -kill="+strconv.FormatBool(kill)))
}
func main() {
flag.Parse()
log.SetFlags(log.Lshortfile)
fmt.Println("running deter with nmsgs:", nmsgs, rate, rounds)
virt, err := cliutils.ReadLines("remote/virt.txt")
if err != nil {
log.Fatal(err)
}
phys, err := cliutils.ReadLines("remote/phys.txt")
if err != nil {
log.Fatal(err)
}
vpmap := make(map[string]string)
for i := range virt {
vpmap[virt[i]] = phys[i]
}
// kill old processes
var wg sync.WaitGroup
for _, h := range phys {
wg.Add(1)
go func(h string) {
defer wg.Done()
cliutils.SshRun("", h, "sudo killall exec logserver timeclient scp ssh 2>/dev/null >/dev/null")
time.Sleep(1 * time.Second)
cliutils.SshRun("", h, "sudo killall forkexec 2>/dev/null >/dev/null")
}(h)
}
wg.Wait()
if kill {
return
}
for _, h := range phys {
wg.Add(1)
go func(h string) {
defer wg.Done()
cliutils.Rsync("", h, "remote", "")
}(h)
}
wg.Wait()
nloggers := 3
masterLogger := phys[0]
slaveLogger1 := phys[1]
slaveLogger2 := phys[2]
loggers := []string{masterLogger, slaveLogger1, slaveLogger2}
phys = phys[nloggers:]
virt = virt[nloggers:]
// Read in and parse the configuration file
file, err := ioutil.ReadFile("remote/cfg.json")
if err != nil {
log.Fatal("deter.go: error reading configuration file: %v\n", err)
}
log.Println("cfg file:", string(file))
var cf config.ConfigFile
err = json.Unmarshal(file, &cf)
if err != nil {
log.Fatal("unable to unmarshal config.ConfigFile:", err)
}
hostnames := cf.Hosts
depth := graphs.Depth(cf.Tree)
var random_leaf string
cf.Tree.TraverseTree(func(t *graphs.Tree) {
if random_leaf != "" {
return
}
if len(t.Children) == 0 {
random_leaf = t.Name
}
})
rootname = hostnames[0]
log.Println("depth of tree:", depth)
// mapping from physical node name to the timestamp servers that are running there
// essentially a reverse mapping of vpmap except ports are also used
physToServer := make(map[string][]string)
for _, virt := range hostnames {
v, _, _ := net.SplitHostPort(virt)
p := vpmap[v]
ss := physToServer[p]
ss = append(ss, virt)
physToServer[p] = ss
}
// start up the logging server on the final host at port 10000
fmt.Println("starting up logserver")
// start up the master logger
loggerports := make([]string, len(loggers))
for i, logger := range loggers {
loggerport := logger + ":10000"
loggerports[i] = loggerport
// redirect to the master logger
master := masterLogger + ":10000"
// if this is the master logger than don't set the master to anything
if loggerport == masterLogger+":10000" {
master = ""
}
go cliutils.SshRunStdout("", logger, "cd remote/logserver; sudo ./logserver -addr="+loggerport+
" -hosts="+strconv.Itoa(len(hostnames))+
" -depth="+strconv.Itoa(depth)+
" -bf="+bf+
" -hpn="+hpn+
" -nmsgs="+nmsgs+
" -rate="+strconv.Itoa(rate)+
" -master="+master)
}
// wait a little bit for the logserver to start up
time.Sleep(5 * time.Second)
fmt.Println("starting time clients")
// start up one timeclient per physical machine
// it requests timestamps from all the servers on that machine
i := 0
for p, ss := range physToServer {
if len(ss) == 0 {
continue
}
servers := strings.Join(ss, ",")
go func(i int, p string) {
_, err := cliutils.SshRun("", p, "cd remote; sudo ./timeclient -nmsgs="+nmsgs+
" -name=client@"+p+
" -server="+servers+
" -logger="+loggerports[i]+
" -debug="+debug+
" -rate="+strconv.Itoa(rate))
if err != nil {
log.Println(err)
}
}(i, p)
i = (i + 1) % len(loggerports)
}
rootwait := strconv.Itoa(10)
for phys, virts := range physToServer {
if len(virts) == 0 {
continue
}
log.Println("starting timestamper")
cmd := GenExecCmd(rFail, fFail, failures, phys, virts, loggerports[i], rootwait, random_leaf)
i = (i + 1) % len(loggerports)
wg.Add(1)
//time.Sleep(500 * time.Millisecond)
go func(phys, cmd string) {
//log.Println("running on ", phys, cmd)
defer wg.Done()
err := cliutils.SshRunStdout("", phys, cmd)
if err != nil {
log.Fatal("ERROR STARTING TIMESTAMPER:", err)
}
}(phys, cmd)
}
// wait for the servers to finish before stopping
wg.Wait()
time.Sleep(10 * time.Minute)
}