// Start will execute one cothority-binary for each server
// configured
func (d *Localhost) Start(args ...string) error {
if err := os.Chdir(d.runDir); err != nil {
return err
}
log.Lvl4("Localhost: chdir into", d.runDir)
ex := d.runDir + "/" + d.Simulation
d.running = true
log.Lvl1("Starting", d.servers, "applications of", ex)
for index := 0; index < d.servers; index++ {
d.wgRun.Add(1)
log.Lvl3("Starting", index)
host := "localhost" + strconv.Itoa(index)
cmdArgs := []string{"-address", host, "-monitor",
"localhost:" + strconv.Itoa(d.monitorPort),
"-simul", d.Simulation,
"-debug", strconv.Itoa(log.DebugVisible()),
}
cmdArgs = append(args, cmdArgs...)
log.Lvl3("CmdArgs are", cmdArgs)
cmd := exec.Command(ex, cmdArgs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
go func(i int, h string) {
log.Lvl3("Localhost: will start host", h)
err := cmd.Run()
if err != nil {
log.Error("Error running localhost", h, ":", err)
d.errChan <- err
}
d.wgRun.Done()
log.Lvl3("host (index", i, ")", h, "done")
}(index, host)
}
return nil
}
// Rsync copies files or directories to the remote host. If the DebugVisible
// is > 1, the rsync-operation is displayed on screen.
func Rsync(username, host, file, dest string) error {
addr := host + ":" + dest
if username != "" {
addr = username + "@" + addr
}
cmd := exec.Command("rsync", "-Pauz", "-e", "ssh -T -c arcfour -o Compression=no -x", file, addr)
cmd.Stderr = os.Stderr
if log.DebugVisible() > 1 {
cmd.Stdout = os.Stdout
}
return cmd.Run()
}
// Reads in the platform that we want to use and prepares for the tests
func main() {
flag.Parse()
deployP = platform.NewPlatform(platformDst)
if deployP == nil {
log.Fatal("Platform not recognized.", platformDst)
}
log.Lvl1("Deploying to", platformDst)
simulations := flag.Args()
if len(simulations) == 0 {
log.Fatal("Please give a simulation to run")
}
for _, simulation := range simulations {
runconfigs := platform.ReadRunFile(deployP, simulation)
if len(runconfigs) == 0 {
log.Fatal("No tests found in", simulation)
}
deployP.Configure(&platform.Config{
MonitorPort: monitorPort,
Debug: log.DebugVisible(),
})
if clean {
err := deployP.Deploy(runconfigs[0])
if err != nil {
log.Fatal("Couldn't deploy:", err)
}
if err := deployP.Cleanup(); err != nil {
log.Error("Couldn't cleanup correctly:", err)
}
} else {
logname := strings.Replace(filepath.Base(simulation), ".toml", "", 1)
testsDone := make(chan bool)
go func() {
RunTests(logname, runconfigs)
testsDone <- true
}()
timeout := getExperimentWait(runconfigs)
select {
case <-testsDone:
log.Lvl3("Done with test", simulation)
case <-time.After(time.Second * time.Duration(timeout)):
log.Fatal("Test failed to finish in", timeout, "seconds")
}
}
}
}
// MarshalRegisteredType will marshal a struct with its respective type into a
// slice of bytes. That slice of bytes can be then decoded in
// UnmarshalRegisteredType.
func MarshalRegisteredType(data Body) ([]byte, error) {
marshalLock.Lock()
defer marshalLock.Unlock()
var msgType MessageTypeID
if msgType = TypeFromData(data); msgType == ErrorType {
return nil, fmt.Errorf("Type of message %s not registered to the network library.", reflect.TypeOf(data))
}
b := new(bytes.Buffer)
if err := binary.Write(b, globalOrder, msgType); err != nil {
return nil, err
}
var buf []byte
var err error
if buf, err = protobuf.Encode(data); err != nil {
log.Error("Error for protobuf encoding:", err)
if log.DebugVisible() >= 3 {
log.Error(log.Stack())
}
return nil, err
}
_, err = b.Write(buf)
return b.Bytes(), err
}
func main() {
// init with deter.toml
deter := deterFromConfig()
flag.Parse()
// kill old processes
var wg sync.WaitGroup
re := regexp.MustCompile(" +")
hosts, err := exec.Command("/usr/testbed/bin/node_list", "-e", deter.Project+","+deter.Experiment).Output()
if err != nil {
log.Fatal("Deterlab experiment", deter.Project+"/"+deter.Experiment, "seems not to be swapped in. Aborting.")
os.Exit(-1)
}
hostsTrimmed := strings.TrimSpace(re.ReplaceAllString(string(hosts), " "))
hostlist := strings.Split(hostsTrimmed, " ")
doneHosts := make([]bool, len(hostlist))
log.Lvl2("Found the following hosts:", hostlist)
if kill {
log.Lvl1("Cleaning up", len(hostlist), "hosts.")
}
for i, h := range hostlist {
wg.Add(1)
go func(i int, h string) {
defer wg.Done()
if kill {
log.Lvl3("Cleaning up host", h, ".")
runSSH(h, "sudo killall -9 cothority scp 2>/dev/null >/dev/null")
time.Sleep(1 * time.Second)
runSSH(h, "sudo killall -9 cothority 2>/dev/null >/dev/null")
time.Sleep(1 * time.Second)
// Also kill all other process that start with "./" and are probably
// locally started processes
runSSH(h, "sudo pkill -9 -f '\\./'")
time.Sleep(1 * time.Second)
if log.DebugVisible() > 3 {
log.Lvl4("Cleaning report:")
_ = platform.SSHRunStdout("", h, "ps aux")
}
} else {
log.Lvl3("Setting the file-limit higher on", h)
// Copy configuration file to make higher file-limits
err := platform.SSHRunStdout("", h, "sudo cp remote/cothority.conf /etc/security/limits.d")
if err != nil {
log.Fatal("Couldn't copy limit-file:", err)
}
}
doneHosts[i] = true
log.Lvl3("Host", h, "cleaned up")
}(i, h)
}
cleanupChannel := make(chan string)
go func() {
wg.Wait()
log.Lvl3("Done waiting")
cleanupChannel <- "done"
}()
select {
case msg := <-cleanupChannel:
log.Lvl3("Received msg from cleanupChannel", msg)
case <-time.After(time.Second * 20):
for i, m := range doneHosts {
if !m {
log.Lvl1("Missing host:", hostlist[i], "- You should run")
log.Lvl1("/usr/testbed/bin/node_reboot", hostlist[i])
}
}
log.Fatal("Didn't receive all replies while cleaning up - aborting.")
}
if kill {
log.Lvl2("Only cleaning up - returning")
return
}
// ADDITIONS : the monitoring part
// Proxy will listen on Sink:SinkPort and redirect every packet to
// RedirectionAddress:SinkPort-1. With remote tunnel forwarding it will
// be forwarded to the real sink
proxyAddress := deter.ProxyAddress + ":" + strconv.Itoa(deter.MonitorPort+1)
log.Lvl2("Launching proxy redirecting to", proxyAddress)
err = monitor.Proxy(proxyAddress)
if err != nil {
log.Fatal("Couldn't start proxy:", err)
}
log.Lvl1("starting", deter.Servers, "cothorities for a total of", deter.Hosts, "processes.")
killing := false
for i, phys := range deter.Phys {
log.Lvl2("Launching cothority on", phys)
wg.Add(1)
go func(phys, internal string) {
//log.Lvl4("running on", phys, cmd)
defer wg.Done()
monitorAddr := deter.MonitorAddress + ":" + strconv.Itoa(deter.MonitorPort)
log.Lvl4("Starting servers on physical machine ", internal, "with monitor = ",
monitorAddr)
args := " -address=" + internal +
" -simul=" + deter.Simulation +
" -monitor=" + monitorAddr +
" -debug=" + strconv.Itoa(log.DebugVisible())
log.Lvl3("Args is", args)
err := platform.SSHRunStdout("", phys, "cd remote; sudo ./cothority "+
args)
if err != nil && !killing {
log.Lvl1("Error starting cothority - will kill all others:", err, internal)
killing = true
err := exec.Command("killall", "ssh").Run()
if err != nil {
log.Fatal("Couldn't killall ssh:", err)
}
}
log.Lvl4("Finished with cothority on", internal)
}(phys, deter.Virt[i])
}
// wait for the servers to finish before stopping
wg.Wait()
}
