func setupLogger() {
h, _, err := net.SplitHostPort(logger)
if err != nil {
log.Info(err)
h = logger
}
// build the logserver for the logger's environment
err = cliutils.Build("../logserver", "386", "linux")
if err != nil {
log.Fatal("failed to build logserver:", err)
}
// move the build to the logserver directory
err = os.Rename("logserver", "../logserver/logserver")
if err != nil {
log.Fatal("failed to rename logserver:", err)
}
// scp the logserver to the environment it will run on
err = cliutils.Scp(uname, h, "../logserver", "~/")
if err != nil {
log.Fatal("failed to scp logserver:", err)
}
// startup the logserver
go func() {
cliutils.SshRunStdout(uname, h, "cd logserver; ./logserver -addr="+logger)
if err != nil {
log.Fatal("failed to run logserver:", err)
}
}()
}
func deployClient(client string, host string) {
time.Sleep(1 * time.Second)
// log.Println("running client")
// timestamp server runs a port one higher than the Node port
h, p, err := net.SplitHostPort(host)
if err != nil {
log.Fatal(err)
}
pn, _ := strconv.Atoi(p)
pn += 1
hp := net.JoinHostPort(h, strconv.Itoa(pn))
if err := cliutils.SshRunStdout(uname, client, "./timeclient -name="+client+" -server="+hp+" -nmsgs="+strconv.Itoa(nrounds)+" -logger="+logger); err != nil {
log.Fatal(host, err)
}
}
func deployServers(hostnames []string) {
var wg sync.WaitGroup
for _, hostport := range hostnames {
wg.Add(1)
go func(hostport string) {
defer wg.Done()
host, _, err := net.SplitHostPort(hostport)
if err != nil {
log.Fatal(err)
}
// log.Println("running signing node")
if err := cliutils.SshRunStdout(uname, host, "./cocoexec -hostname="+hostport+" -app="+app+" -nrounds="+strconv.Itoa(nrounds)+" -config=cfg.json -logger="+logger); err != nil {
log.Fatal(host, err)
}
}(hostport)
}
wg.Wait()
}
func main() {
log.SetFlags(log.Lshortfile)
flag.Parse()
log.Println("RUNNING DEPLOY2DETER WITH RATE:", rate)
os.MkdirAll("remote", 0777)
var wg sync.WaitGroup
// start building the necessary packages
packages := []string{"../logserver", "../timeclient", "../exec", "../forkexec", "../deter"}
for _, p := range packages {
wg.Add(1)
if p == "../deter" {
go func(p string) {
defer wg.Done()
// the users node has a 386 FreeBSD architecture
err := cliutils.Build(p, "386", "freebsd")
if err != nil {
log.Fatal(err)
}
}(p)
continue
}
go func(p string) {
defer wg.Done()
// deter has an amd64, linux architecture
err := cliutils.Build(p, "amd64", "linux")
if err != nil {
log.Fatal(err)
}
}(p)
}
// killssh processes on users
cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net", "killall ssh scp deter 2>/dev/null 1>/dev/null")
// parse the hosts.txt file to create a separate list (and file)
// of physical nodes and virtual nodes. Such that each host on line i, in phys.txt
// corresponds to each host on line i, in virt.txt.
physVirt, err := cliutils.ReadLines("hosts.txt")
phys := make([]string, 0, len(physVirt)/2)
virt := make([]string, 0, len(physVirt)/2)
for i := 0; i < len(physVirt); i += 2 {
phys = append(phys, physVirt[i])
virt = append(virt, physVirt[i+1])
}
nloggers := 3
// only use the number of machines that we need
phys = phys[:nmachs+nloggers]
virt = virt[:nmachs+nloggers]
physOut := strings.Join(phys, "\n")
virtOut := strings.Join(virt, "\n")
// phys.txt and virt.txt only contain the number of machines that we need
err = ioutil.WriteFile("remote/phys.txt", []byte(physOut), 0666)
if err != nil {
log.Fatal("failed to write physical nodes file", err)
}
err = ioutil.WriteFile("remote/virt.txt", []byte(virtOut), 0666)
if err != nil {
log.Fatal("failed to write virtual nodes file", err)
}
masterLogger := phys[0]
// slaveLogger1 := phys[1]
// slaveLogger2 := phys[2]
virt = virt[3:]
phys = phys[3:]
t, hostnames, depth, err := graphs.TreeFromList(virt, hpn, bf)
log.Println("DEPTH:", depth)
log.Println("TOTAL HOSTS:", len(hostnames))
// wait for the build to finish
wg.Wait()
// copy the logserver directory to the current directory
err = exec.Command("rsync", "-au", "../logserver", "remote/").Run()
if err != nil {
log.Fatal("error rsyncing logserver directory into remote directory:", err)
}
err = exec.Command("rsync", "-au", "remote/phys.txt", "remote/virt.txt", "remote/logserver/").Run()
if err != nil {
log.Fatal("error rsyncing phys, virt, and remote/logserver:", err)
}
err = os.Rename("logserver", "remote/logserver/logserver")
if err != nil {
log.Fatal("error renaming logserver:", err)
}
b, err := json.Marshal(t)
if err != nil {
log.Fatal("unable to generate tree from list")
}
err = ioutil.WriteFile("remote/logserver/cfg.json", b, 0660)
if err != nil {
log.Fatal("unable to write configuration file")
}
// NOTE: now remote/logserver is ready for transfer
// it has logserver/ folder, binary, and cfg.json, and phys.txt, virt.txt
// generate the configuration file from the tree
cf := config.ConfigFromTree(t, hostnames)
cfb, err := json.Marshal(cf)
err = ioutil.WriteFile("remote/cfg.json", cfb, 0666)
if err != nil {
log.Fatal(err)
}
// scp the files that we need over to the boss node
files := []string{"timeclient", "exec", "forkexec", "deter"}
for _, f := range files {
cmd := exec.Command("rsync", "-au", f, "remote/")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
log.Fatal("error unable to rsync file into remote directory:", err)
}
}
err = cliutils.Rsync("dvisher", "users.isi.deterlab.net", "remote", "")
if err != nil {
log.Fatal(err)
}
killssh := exec.Command("pkill", "-f", "ssh -t -t")
killssh.Stdout = os.Stdout
killssh.Stderr = os.Stderr
err = killssh.Run()
if err != nil {
log.Print(err)
}
// setup port forwarding for viewing log server
// ssh -L 8080:pcXXX:80 [email protected]
// ssh [email protected] -L 8118:somenode.experiment.YourClass.isi.deterlab.net:80
fmt.Println("setup port forwarding for master logger: ", masterLogger)
cmd := exec.Command(
"ssh",
"-t",
"-t",
"*****@*****.**",
"-L",
"8080:"+masterLogger+":10000")
err = cmd.Start()
if err != nil {
log.Fatal("failed to setup portforwarding for logging server")
}
log.Println("runnning deter with nmsgs:", nmsgs)
// run the deter lab boss nodes process
// it will be responsible for forwarding the files and running the individual
// timestamping servers
log.Fatal(cliutils.SshRunStdout("dvisher", "users.isi.deterlab.net",
"GOMAXPROCS=8 remote/deter -nmsgs="+strconv.Itoa(nmsgs)+
" -hpn="+strconv.Itoa(hpn)+
" -bf="+strconv.Itoa(bf)+
" -rate="+strconv.Itoa(rate)+
" -rounds="+strconv.Itoa(rounds)+
" -debug="+strconv.FormatBool(debug)+
" -failures="+strconv.Itoa(failures)+
" -rfail="+strconv.Itoa(rFail)+
" -ffail="+strconv.Itoa(fFail)+
" -test_connect="+strconv.FormatBool(testConnect)+
" -app="+app+
" -suite="+suite+
" -kill="+strconv.FormatBool(kill)))
}
func testNodes(hostnames []string, failed map[string]bool) ([]string, []byte) {
var mu sync.Mutex
var edgelist []byte
var wg sync.WaitGroup
for _, host := range hostnames {
log.Println("latency test:", host)
if _, ok := failed[host]; ok {
continue
}
wg.Add(1)
go func(host string) {
defer wg.Done()
starttime := time.Now()
// kill latent processes
err := cliutils.TimeoutRun(10*time.Second,
func() error {
return cliutils.SshRunStdout(uname, host, "killall logserver; killall timeclient; killall latency_test; killall cocoexec; rm -rf cocoexec")
})
if err != nil {
log.Println("Failed:", host, err)
mu.Lock()
failed[host] = true
mu.Unlock()
return
}
// run the latency test
log.Println("running latency_test:", host)
output, err := cliutils.SshRun(uname, host, "./latency_test -hostfile=hosts.txt -hostname="+host)
if err != nil {
log.Println("Failed:", host, err)
mu.Lock()
failed[host] = true
mu.Unlock()
return
}
// if this took to long say that this has failed
if time.Since(starttime) > (20 * time.Minute) {
log.Println("Failed:", host, err)
mu.Lock()
failed[host] = true
mu.Unlock()
return
}
fmt.Println("output:", string(output))
mu.Lock()
edgelist = append(edgelist, output...)
mu.Unlock()
return
}(host)
}
wg.Wait()
log.Println("latency test done")
goodhosts := make([]string, 0, len(hostnames)-len(failed))
for _, h := range hostnames {
if !failed[h] {
goodhosts = append(goodhosts, h)
}
}
return goodhosts, edgelist
}
func main() {
flag.Parse()
log.SetFlags(log.Lshortfile)
fmt.Println("running deter with nmsgs:", nmsgs, rate, rounds)
virt, err := cliutils.ReadLines("remote/virt.txt")
if err != nil {
log.Fatal(err)
}
phys, err := cliutils.ReadLines("remote/phys.txt")
if err != nil {
log.Fatal(err)
}
vpmap := make(map[string]string)
for i := range virt {
vpmap[virt[i]] = phys[i]
}
// kill old processes
var wg sync.WaitGroup
for _, h := range phys {
wg.Add(1)
go func(h string) {
defer wg.Done()
cliutils.SshRun("", h, "sudo killall exec logserver timeclient scp ssh 2>/dev/null >/dev/null")
time.Sleep(1 * time.Second)
cliutils.SshRun("", h, "sudo killall forkexec 2>/dev/null >/dev/null")
}(h)
}
wg.Wait()
if kill {
return
}
for _, h := range phys {
wg.Add(1)
go func(h string) {
defer wg.Done()
cliutils.Rsync("", h, "remote", "")
}(h)
}
wg.Wait()
nloggers := 3
masterLogger := phys[0]
slaveLogger1 := phys[1]
slaveLogger2 := phys[2]
loggers := []string{masterLogger, slaveLogger1, slaveLogger2}
phys = phys[nloggers:]
virt = virt[nloggers:]
// Read in and parse the configuration file
file, err := ioutil.ReadFile("remote/cfg.json")
if err != nil {
log.Fatal("deter.go: error reading configuration file: %v\n", err)
}
log.Println("cfg file:", string(file))
var cf config.ConfigFile
err = json.Unmarshal(file, &cf)
if err != nil {
log.Fatal("unable to unmarshal config.ConfigFile:", err)
}
hostnames := cf.Hosts
depth := graphs.Depth(cf.Tree)
var random_leaf string
cf.Tree.TraverseTree(func(t *graphs.Tree) {
if random_leaf != "" {
return
}
if len(t.Children) == 0 {
random_leaf = t.Name
}
})
rootname = hostnames[0]
log.Println("depth of tree:", depth)
// mapping from physical node name to the timestamp servers that are running there
// essentially a reverse mapping of vpmap except ports are also used
physToServer := make(map[string][]string)
for _, virt := range hostnames {
v, _, _ := net.SplitHostPort(virt)
p := vpmap[v]
ss := physToServer[p]
ss = append(ss, virt)
physToServer[p] = ss
}
// start up the logging server on the final host at port 10000
fmt.Println("starting up logserver")
// start up the master logger
loggerports := make([]string, len(loggers))
for i, logger := range loggers {
loggerport := logger + ":10000"
loggerports[i] = loggerport
// redirect to the master logger
master := masterLogger + ":10000"
// if this is the master logger than don't set the master to anything
if loggerport == masterLogger+":10000" {
master = ""
}
go cliutils.SshRunStdout("", logger, "cd remote/logserver; sudo ./logserver -addr="+loggerport+
" -hosts="+strconv.Itoa(len(hostnames))+
" -depth="+strconv.Itoa(depth)+
" -bf="+bf+
" -hpn="+hpn+
" -nmsgs="+nmsgs+
" -rate="+strconv.Itoa(rate)+
" -master="+master)
}
// wait a little bit for the logserver to start up
time.Sleep(5 * time.Second)
fmt.Println("starting time clients")
// start up one timeclient per physical machine
// it requests timestamps from all the servers on that machine
i := 0
for p, ss := range physToServer {
if len(ss) == 0 {
continue
}
servers := strings.Join(ss, ",")
go func(i int, p string) {
_, err := cliutils.SshRun("", p, "cd remote; sudo ./timeclient -nmsgs="+nmsgs+
" -name=client@"+p+
" -server="+servers+
" -logger="+loggerports[i]+
" -debug="+debug+
" -rate="+strconv.Itoa(rate))
if err != nil {
log.Println(err)
}
}(i, p)
i = (i + 1) % len(loggerports)
}
rootwait := strconv.Itoa(10)
for phys, virts := range physToServer {
if len(virts) == 0 {
continue
}
log.Println("starting timestamper")
cmd := GenExecCmd(rFail, fFail, failures, phys, virts, loggerports[i], rootwait, random_leaf)
i = (i + 1) % len(loggerports)
wg.Add(1)
//time.Sleep(500 * time.Millisecond)
go func(phys, cmd string) {
//log.Println("running on ", phys, cmd)
defer wg.Done()
err := cliutils.SshRunStdout("", phys, cmd)
if err != nil {
log.Fatal("ERROR STARTING TIMESTAMPER:", err)
}
}(phys, cmd)
}
// wait for the servers to finish before stopping
wg.Wait()
time.Sleep(10 * time.Minute)
}