/*
* Invoke the tegu_add_mirror or tegu_del_mirror command on a remote host in order to add/remove a mirror.
*/
func do_mirrorwiz(req json_action, broker *ssh_broker.Broker, path *string) {
startt := time.Now().UnixNano()
cstr := ""
switch req.Qdata[0] {
case "add":
cstr = fmt.Sprintf(`PATH=%s:$PATH tegu_add_mirror %s %s %s`, *path, req.Qdata[1], req.Qdata[2], req.Qdata[3])
if len(req.Qdata) > 4 {
// If VLAN list is in the arguments, tack that on the end
cstr += " " + req.Qdata[4]
}
case "del":
cstr = fmt.Sprintf(`PATH=%s:$PATH tegu_del_mirror %s`, *path, req.Qdata[1])
}
if cstr != "" {
sheep.Baa(1, "via broker on %s: %s", req.Hosts[0], cstr)
_, stderr, err := broker.Run_cmd(req.Hosts[0], cstr)
if err != nil {
sheep.Baa(0, "ERR: send mirror cmd failed host=%s: %s [TGUAGN005]", req.Hosts[0], err)
} else {
sheep.Baa(2, "mirror cmd succesfully sent: %s", cstr)
}
if sheep.Would_baa(2) || err != nil {
dump_stderr(*stderr, "addmirror"+req.Hosts[0]) // always dump on error, or if chatty
}
} else {
sheep.Baa(0, "Unrecognized mirror command: "+req.Qdata[0])
}
endt := time.Now().UnixNano()
sheep.Baa(1, "do_mirrorwiz: %d ms elapsed", (endt-startt)/1000)
}
/*
Extracts the information from the action passed in and causes the fmod command
to be executed.
*/
func do_fmod(req json_action, broker *ssh_broker.Broker, path *string, timeout time.Duration) (err error) {
startt := time.Now().Unix()
errcount := 0
for f := range req.Fdata {
cstr := fmt.Sprintf(`PATH=%s:$PATH send_ovs_fmod %s`, *path, req.Fdata[f])
ssh_rch := make(chan *ssh_broker.Broker_msg, 256) // channel for ssh results
// do NOT close the channel here; only senders should close
wait4 := 0 // number of responses to wait for
for i := range req.Hosts {
sheep.Baa(1, "via broker on %s send fmod: %s", req.Hosts[i], cstr)
err := broker.NBRun_cmd(req.Hosts[i], cstr, wait4, ssh_rch) // sends the file as input to be executed on the host
if err != nil {
msg_007(req.Hosts[i], cstr, err)
} else {
wait4++
}
}
timer_pop := false
errcount := 0
for wait4 > 0 && !timer_pop { // collect responses logging any errors
select {
case <-time.After(timeout * time.Second): // timeout
msg_008(wait4)
timer_pop = true
case resp := <-ssh_rch: // response back from the broker
wait4--
_, stderr, elapsed, err := resp.Get_results()
host, _, _ := resp.Get_info()
sheep.Baa(1, "send-fmod: received response from %s elap=%d err=%v, waiting for %d more", host, elapsed, err != nil, wait4)
if err != nil {
sheep.Baa(0, "ERR: unable to execute send-fmod command on %s: data=%s %s [TGUAGN004]", host, cstr, err)
errcount++
} else {
sheep.Baa(1, "flow mod set on: %s", host)
}
if err != nil || sheep.Would_baa(2) {
dump_stderr(stderr, "send-fmod"+host) // always dump on error, or if chatty
}
}
}
}
endt := time.Now().Unix()
sheep.Baa(1, "fmod: %ds elapsed %d fmods %d errors", endt-startt, len(req.Fdata), errcount)
return
}
/*
Executes the setup_ovs_intermed script on each host listed. This command can take
a significant amount of time on each host (10s of seconds) and so we submit the
command to the broker for each host in non-blocking mode to allow them to
run concurrently. Once submitted, we collect the results (reporting errors)
as the broker writes the response back on the channel.
*/
func do_intermedq(req json_action, broker *ssh_broker.Broker, path *string, timeout time.Duration) {
startt := time.Now().Unix()
running_sim = true // prevent queuing another of these
sheep.Baa(1, "running intermediate switch queue/fmod setup on all hosts (broker)")
ssh_rch := make(chan *ssh_broker.Broker_msg, 256) // channel for ssh results
// do NOT close the channel here; only senders should close
wait4 := 0 // number of responses to wait for
for i := range req.Hosts {
cmd_str := fmt.Sprintf(`PATH=%s:$PATH setup_ovs_intermed -d "%s"`, *path, req.Dscps)
sheep.Baa(1, "via broker on %s: %s", req.Hosts[i], cmd_str)
err := broker.NBRun_cmd(req.Hosts[i], cmd_str, wait4, ssh_rch)
if err != nil {
msg_007(req.Hosts[i], cmd_str, err)
} else {
wait4++
}
}
timer_pop := false
errcount := 0
for wait4 > 0 && !timer_pop { // collect responses logging any errors
select {
case <-time.After(timeout * time.Second): // timeout
msg_008(wait4)
timer_pop = true
case resp := <-ssh_rch: // response back from the broker
wait4--
_, stderr, elapsed, err := resp.Get_results()
host, _, _ := resp.Get_info()
sheep.Baa(2, "setup-intermed: received response from %s elap=%d err=%v, waiting for %d more", host, elapsed, err != nil, wait4)
if err != nil {
msg_009("setup_intermed", host)
errcount++
}
if err != nil || sheep.Would_baa(2) {
dump_stderr(stderr, "setup-intermed"+host) // always dump on error, or if chatty
}
}
}
endt := time.Now().Unix()
sheep.Baa(1, "setup-intermed: timeout=%v %ds elapsed for %d hosts %d errors", timer_pop, endt-startt, len(req.Hosts), errcount)
running_sim = false
}
/*
run a command -- run as a go routine to run multiple in parallel
*/
func test_cmd(broker *ssh_broker.Broker, ch chan int, host *string, cmd *string) {
fmt.Fprintf(os.Stderr, "test_cmd: running command %s\n", *cmd)
stdout, stderr, err := broker.Run_cmd(*host, *cmd)
if err != nil {
fmt.Fprintf(os.Stderr, "command failed: %s: %s \n", *host, err)
fmt.Fprintf(os.Stderr, "%s", stderr.String())
} else {
fmt.Fprintf(os.Stderr, "command was successful:\n")
fmt.Printf("%s\n", stdout.String())
}
fmt.Fprintf(os.Stderr, "go routine done: %s \n", *cmd)
ch <- 1
return
}
/*
run a local script -- run as a go routine to run multiple in parallel
*/
func test_script(broker *ssh_broker.Broker, ch chan int, host *string, script *string, parms *string, env_file *string) {
fmt.Fprintf(os.Stderr, "running commnand=%s parms=%s\n", *script, *parms)
stdout, stderr, err := broker.Run_on_host(*host, *script, *parms, *env_file)
if err != nil {
fmt.Fprintf(os.Stderr, "command failed: %s: %s \n", *host, err)
fmt.Fprintf(os.Stderr, "%s", stderr.String())
} else {
fmt.Fprintf(os.Stderr, "command was successful:\n")
fmt.Printf("%s\n", stdout.String())
}
fmt.Fprintf(os.Stderr, "go routine done:%s \n", *parms)
ch <- 1
return
}
/*
Execute a create_ovs_queues for each host in the list. The create queues script is unique inasmuch
as it expects an input file that is supplied either as a filename as $1, or on stdin if $1 is omitted.
To send the data file for the command to execute, we'll create a tmp file on the local machine which
is a script that echos the data into the script:
cat <<endKat | create_ovs_queues
data passed to us
endKat
We'll use the brokers 'send script for execution' feature rather to execute our script.
*/
func do_setqueues(req json_action, broker *ssh_broker.Broker, path *string, timeout time.Duration) {
var (
err error
)
startt := time.Now().Unix()
fname := fmt.Sprintf("/tmp/tegu_setq_%d_%x_%02d.data", os.Getpid(), time.Now().Unix(), rand.Intn(10))
sheep.Baa(3, "adjusting queues: creating %s will contain %d items", fname, len(req.Qdata))
f, err := os.Create(fname)
if err != nil {
sheep.Baa(0, "ERR: unable to create data file: %s: %s [TGUAGN002]", fname, err)
return
}
fmt.Fprintf(f, "#!/usr/bin/env ksh\ncat <<endKat | PATH=%s:$PATH create_ovs_queues\n", *path)
for i := range req.Qdata {
sheep.Baa(3, "writing queue info: %s", req.Qdata[i])
fmt.Fprintf(f, "%s\n", req.Qdata[i])
}
fmt.Fprintf(f, "endKat\n")
err = f.Close()
if err != nil {
sheep.Baa(0, "ERR: unable to create data file (close): %s: %s [TGUAGN003]", fname, err)
return
}
ssh_rch := make(chan *ssh_broker.Broker_msg, 256) // channel for ssh results
// do NOT close the channel here; only senders should close
wait4 := 0 // number of responses to wait for
for i := range req.Hosts {
sheep.Baa(1, "via broker on %s: create_ovs_queues embedded in %s", req.Hosts[i], fname)
err := broker.NBRun_on_host(req.Hosts[i], fname, "", wait4, ssh_rch) // sends the file as input to be executed on the host
if err != nil {
msg_007(req.Hosts[i], "create_ovs_queues", err)
} else {
wait4++
}
}
timer_pop := false
errcount := 0
for wait4 > 0 && !timer_pop { // collect responses logging any errors
select {
case <-time.After(timeout * time.Second): // timeout
msg_008(wait4)
timer_pop = true
case resp := <-ssh_rch: // response back from the broker
wait4--
_, stderr, elapsed, err := resp.Get_results()
host, _, _ := resp.Get_info()
sheep.Baa(2, "create-q: received response from %s elap=%d err=%v, waiting for %d more", host, elapsed, err != nil, wait4)
if err != nil {
sheep.Baa(0, "ERR: unable to execute set queue command on %s: data=%s: %s [TGUAGN004]", host, fname, err)
errcount++
} else {
sheep.Baa(1, "queues adjusted succesfully on: %s", host)
}
if err != nil || sheep.Would_baa(2) {
dump_stderr(stderr, "create-q"+host) // always dump on error, or if chatty
}
}
}
endt := time.Now().Unix()
sheep.Baa(1, "create-q: timeout=%v %ds elapsed %d hosts %d errors", timer_pop, endt-startt, len(req.Hosts), errcount)
if errcount == 0 { // ditch the script we built earlier if all successful
os.Remove(fname)
} else {
sheep.Baa(1, "create-q: %d errors, generated script file kept: %s", fname)
}
}
/*
Generate a map that lists physical host and mac addresses. Timeout is the max number of
seconds that we will wait for all responses. If timeout seconds passes before all
responses are received we will return what we have. The map command is executed on all
hosts, so we send a non-blocking command to the broker for each host and wait for the
responses to come back on the channel. This allows them to run in parallel across
the cluster.
*/
func do_map_mac2phost(req json_action, broker *ssh_broker.Broker, path *string, timeout time.Duration) (jout []byte, err error) {
var (
cmd_str string
)
startt := time.Now().Unix()
ssh_rch := make(chan *ssh_broker.Broker_msg, 256) // channel for ssh results
// do NOT close this channel, only senders should close
wait4 := 0 // number of responses to wait for
for k, v := range req.Hosts { // submit them all out non-blocking
cmd_str = fmt.Sprintf("PATH=%s:$PATH map_mac2phost -p %s localhost", *path, v)
err := broker.NBRun_cmd(req.Hosts[k], cmd_str, wait4, ssh_rch)
if err != nil {
msg_007(req.Hosts[k], cmd_str, err)
} else {
wait4++
}
}
msg := agent_msg{} // message to return
msg.Ctype = "response"
msg.Rtype = "map_mac2phost"
msg.Vinfo = version
msg.State = 0
rdata := make([]string, 8192) // might need to revisit this limit
ridx := 0
sheep.Baa(2, "map_mac2phost: waiting for %d responses", wait4)
timer_pop := false // indicates a timeout for loop exit
errcount := 0
for wait4 > 0 && !timer_pop { // wait for responses back on the channel or the timer to pop
select {
case <-time.After(timeout * time.Second): // timeout after 15 seconds
msg_008(wait4)
timer_pop = true
case resp := <-ssh_rch: // response from broker
wait4--
stdout, stderr, elapsed, err := resp.Get_results()
host, _, _ := resp.Get_info()
sheep.Baa(2, "map_mac2phost: received response from %s elap=%d err=%v, waiting for %d more", host, elapsed, err != nil, wait4)
if err != nil {
msg_009("map_mac2phost", host)
errcount++
} else {
ridx = buf_into_array(stdout, rdata, ridx) // capture what came back for return
}
if err != nil || sheep.Would_baa(2) {
dump_stderr(stderr, "map_mac2phost"+host) // always dump stderr on error, or in chatty mode
}
}
}
msg.Rdata = rdata[0:ridx] // return just what was filled in
endt := time.Now().Unix()
sheep.Baa(1, "map-mac2phost: timeout=%v %ds elapsed for %d hosts %d errors %d elements", timer_pop, endt-startt, len(req.Hosts), errcount, len(msg.Rdata))
jout, err = json.Marshal(msg)
return
}
/*
Oneway bandwidth flow-mod generation rolls the creation of a set of flow-mods into a single script which
eliminates the need for Tegu to understand/know things like command line parms, bridge names and
such. Parms in the map are converted to script command line options.
*/
func (act *json_action) do_bwow_fmod(cmd_type string, broker *ssh_broker.Broker, path *string, timeout time.Duration) (jout []byte, err error) {
var (
cmd_str string
)
pstr := ""
if path != nil {
pstr = fmt.Sprintf("PATH=%s:$PATH ", *path) // path to add if needed
}
parms := act.Data
cmd_str = fmt.Sprintf(`%sql_bwow_fmods `, pstr) +
build_opt(parms["smac"], "-s") +
build_opt(parms["dmac"], "-d") +
build_opt(parms["extip"], "-E") +
build_opt(parms["sproto"], "-p") +
build_opt(parms["dproto"], "-P") +
build_opt(parms["queue"], "-q") +
build_opt(parms["timeout"], "-t") +
build_opt(parms["dscp"], "-T") +
build_opt(parms["vlan_match"], "-V") +
build_opt(parms["ipv6"], "-6")
sheep.Baa(1, "via broker on %s: %s", act.Hosts[0], cmd_str)
msg := agent_msg{} // build response to send back
msg.Ctype = "response"
msg.Rtype = cmd_type
msg.Rid = act.Aid // response id so tegu can map back to requestor
msg.Vinfo = version
msg.State = 0 // assume success
ssh_rch := make(chan *ssh_broker.Broker_msg, 256) // channel for ssh results
// do NOT close the channel here; only senders should close
err = broker.NBRun_cmd(act.Hosts[0], cmd_str, 0, ssh_rch) // oneway fmods are only ever applied to one host so [0] is ok
if err != nil {
sheep.Baa(1, "WRN: error submitting bwow command to %s: %s", act.Hosts[0], err)
jout, _ = json.Marshal(msg)
return
}
// TODO: this can be moved to a function
rdata := make([]string, 8192) // response output converted to strings
edata := make([]string, 8192)
ridx := 0 // index of next insert point into rdata
wait4 := 1
timer_pop := false
for wait4 > 0 && !timer_pop { // wait for response back on the channel or the timer to pop
select {
case <-time.After(timeout * time.Second): // timeout if we don't get something back soonish
sheep.Baa(1, "WRN: timeout waiting for response from %s; cmd: %s", act.Hosts[0], cmd_str)
timer_pop = true
case resp := <-ssh_rch: // response from broker
wait4--
stdout, stderr, _, err := resp.Get_results()
host, _, _ := resp.Get_info()
eidx := buf_into_array(stderr, edata, 0) // capture error messages, if any
msg.Edata = edata[0:eidx]
if err != nil {
msg.State = 1
sheep.Baa(1, "WRN: error running command: host=%s: %s", host, err)
} else {
ridx = buf_into_array(stdout, rdata, ridx) // capture what came back for return
}
if err != nil || sheep.Would_baa(2) {
dump_stderr(stderr, "bw_fmod "+host) // always dump stderr on error, or in chatty mode
}
}
}
msg.Rdata = rdata[0:ridx] // return just what was filled in
if msg.State > 0 {
sheep.Baa(1, "bwow_fmod (%s) failed: stdout: %d lines; stderr: %d lines", cmd_type, len(msg.Rdata), len(msg.Edata))
sheep.Baa(0, "ERR: %s unable to execute: %s [TGUAGN000]", cmd_type, cmd_str)
} else {
sheep.Baa(1, "bwow_fmod cmd (%s) completed: stdout: %d lines; stderr: %d lines", cmd_type, len(msg.Rdata), len(msg.Edata))
}
jout, err = json.Marshal(msg)
return
}