/* Get the default gateway for a project. Returns the string directly to the channel
that send the osif the message. Expects to be executed as a go routine.
go get_os_defgw( msg, os_refs, os_projects, id2pname, pname2id ) // do it asynch and return the result on the message channel
*/
func get_os_defgw(msg *ipc.Chmsg, os_refs map[string]*ostack.Ostack, os_projs map[string]*osif_project, id2pname map[string]*string, pname2id map[string]*string) {
if msg == nil || msg.Response_ch == nil {
return // prevent accidents
}
msg.Response_data = nil
if msg.Req_data != nil {
tokens := strings.Split(*(msg.Req_data.(*string)), "/") // split off /junk if it's there
if tokens[0] == "!" || tokens[0] == "" { // nothing to work with; bail now
osif_sheep.Baa(1, "get_defgw: unable to map to a project -- bad token[0] --: %s", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
if tokens[0][0:1] == "!" { // first character is a bang, but there is a name/id that follows
tokens[0] = tokens[0][1:] // ditch the bang and go on
}
pid := &tokens[0]
pname := id2pname[*pid]
if pname == nil { // it should be an id, but allow for a name/host to be sent in
osif_sheep.Baa(1, "get_defgw: unable to map to a project -- no pname --: %s", *(msg.Req_data.(*string)))
pname = &tokens[0]
pid = pname2id[*pname]
}
if pid == nil {
osif_sheep.Baa(1, "get_defgw: unable to map to a project: %s", *(msg.Req_data.(*string)))
msg.State = fmt.Errorf("%s could not be mapped to a osif_project", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
p := os_projs[*pid] // finally we can find the data associated with the project; maybe
if p == nil {
osif_sheep.Baa(1, "get_defgw: unable to map project to data: %s", *pid)
msg.State = fmt.Errorf("%s could not be mapped to a osif_project", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
creds := os_refs[*pname]
if creds == nil {
msg.State = fmt.Errorf("defgw: %s could not be mapped to openstack creds ", *pname)
msg.Response_ch <- msg
return
}
msg.Response_data, _, msg.State = p.Get_default_gw(pid, creds, true)
msg.Response_ch <- msg
return
}
osif_sheep.Baa(1, "get_defgw: missing data (nil) in request")
msg.State = fmt.Errorf("defgw: missing data in request")
msg.Response_ch <- msg // and send it on its merry way
return
}
/*
For a single passthrough pledge, this function sets things up and sends needed requests to the fq-manger to
create any necessary flow-mods.
We send the following information to fq_mgr:
source mac or endpoint (VM-- the host in the pledge)
source IP and optionally port and protocol more specific reservations
expiry
switch (physical host -- compute node)
Errors are returned to res_mgr via channel, but asycnh; we do not wait for responses to each message
generated here.
To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume)
use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If
to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit,
and we assume that this function is called periodically to update long running reservations.
*/
func pass_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_pass) // generic pledge better be a passthrough pledge!
if !ok {
rm_sheep.Baa(1, "internal error in pass_push_reservation: pledge isn't a passthrough pledge")
(*gp).Set_pushed() // prevent looping
return
}
host, _, _, expiry, proto := p.Get_values() // reservation info that we need
ip := name2ip(host)
if ip != nil { // good ip addresses so we're good to go
freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things)
freq.Match.Smac = ip // fq_mgr has conversion map to convert to mac
freq.Swid = p.Get_phost() // the phyiscal host where the VM lives and where fmods need to be deposited
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
freq.Extip = &empty_str
// this will change when ported to endpoint branch as the endpoint allows address and port 'in line'
freq.Match.Ip1 = proto // the proto on the reservation should be [{udp|tcp:}]address[:port]
freq.Match.Ip2 = nil
freq.Espq = nil
dup_str := ""
freq.Exttyp = &dup_str
rm_sheep.Baa(1, "pushing passthru reservation: %s", p)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, ch, REQ_PT_RESERVE, freq, nil) // queue work with fq-manger to read the struct and send cmd(s) to agent to get it done
p.Set_pushed() // safe to mark the pledge as having been pushed.
}
}
/*
Gathers the VM information for all VMs in one or more projects. If "_all_proj" is given as the project name then
all projects known to Tegu are fetched.
Expected to execute as a go routine and writes the resulting array to the channel specified in the message.
*/
func get_all_osvm_info(msg *ipc.Chmsg, os_refs map[string]*ostack.Ostack, os_projs map[string]*osif_project, id2pname map[string]*string, pname2id map[string]*string) {
if msg == nil || msg.Response_ch == nil {
return // prevent accidents
}
msg.Response_data = nil
msg.State = nil
if msg.Req_data == nil {
osif_sheep.Baa(1, "osvm_info: request data didn't contain a project name or ID")
msg.State = fmt.Errorf("osvm_info: request data didn't contain a project name or ID")
msg.Response_ch <- msg
return
}
pid := msg.Req_data.(*string)
if *pid == "_all_proj" {
ilist := make([]*Net_vm, 0)
for k := range os_refs {
if k != "_ref_" {
nlist, err := get_projvm_info(&k, os_refs, os_projs, id2pname, pname2id) // dig out next project's stuff
if err == nil {
llist := make([]*Net_vm, len(ilist)+len(nlist)) // create array large enough
copy(llist[:], ilist[:]) // copy contents into new array
copy(llist[len(ilist):], nlist[:])
ilist = llist
} else {
osif_sheep.Baa(1, "osvm_info: could not dig out VM information for project: %s: %s", k, err)
}
}
}
msg.Response_data = ilist
if len(ilist) <= 0 {
msg.State = fmt.Errorf("osvm_info: unable to dig any information for all projects")
} else {
msg.State = fmt.Errorf("osvm_info: fetched info for all projects: %d elements", len(ilist))
msg.State = nil
}
} else {
msg.Response_data, msg.State = get_projvm_info(pid, os_refs, os_projs, id2pname, pname2id) // just dig out for the one project
}
msg.Response_ch <- msg
}
/*
Get openstack host information.
Given a project-id/host as input, dig out all of the host's information and build a struct
that can be passed into the network manager as an add host to graph request. This
expects to run as a go routine and to write the response directly back on the channel
givn in the message block.
*/
func get_os_hostinfo(msg *ipc.Chmsg, os_refs map[string]*ostack.Ostack, os_projs map[string]*osif_project, id2pname map[string]*string, pname2id map[string]*string) {
if msg == nil || msg.Response_ch == nil {
return // prevent accidents
}
msg.Response_data = nil
tokens := strings.Split(*(msg.Req_data.(*string)), "/") // break project/host into bits
if len(tokens) != 2 || tokens[0] == "" || tokens[1] == "" {
osif_sheep.Baa(1, "get hostinfo: unable to map to a project: %s bad tokens", *(msg.Req_data.(*string)))
msg.State = fmt.Errorf("invalid project/hostname string: %s", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
if tokens[0] == "!" { // !//ipaddress was given; we've got nothing, so bail now
osif_sheep.Baa(1, "get hostinfo: unable to map to a project: %s lone bang", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
if tokens[0][0:1] == "!" { // first character is a bang, but there is a name/id that follows
tokens[0] = tokens[0][1:] // ditch it for this
}
pid := &tokens[0]
pname := id2pname[*pid]
if pname == nil { // it should be an id, but allow for a name/host to be sent in
pname = &tokens[0]
pid = pname2id[*pname]
}
if pid == nil {
osif_sheep.Baa(1, "get hostinfo: unable to map to an project (nil pid): %s", *(msg.Req_data.(*string))) // might be !project/vm, and so this is ok
msg.State = fmt.Errorf("%s could not be mapped to an osif_project", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
p := os_projs[*pid]
if p == nil {
osif_sheep.Baa(1, "get hostinfo: %s mapped to a pid; pid did not map to project: %s", *(msg.Req_data.(*string)), *pid)
msg.State = fmt.Errorf("%s could not be mapped to an osif_project", *(msg.Req_data.(*string)))
msg.Response_ch <- msg
return
}
creds := os_refs[*pname]
if creds == nil {
osif_sheep.Baa(1, "get hostinfo: %s mapped to a project; did not map to creds: %s", *(msg.Req_data.(*string)), *pname)
msg.State = fmt.Errorf("%s could not be mapped to openstack creds ", *pname)
msg.Response_ch <- msg
return
}
osif_sheep.Baa(2, "lazy update: get host info setup complete for (%s) %s", *pname, *(msg.Req_data.(*string)))
search := *pid + "/" + tokens[1] // search string must be id/hostname
name, id, ip4, fip4, mac, gw, phost, gwmap, _, err := p.Get_info(&search, creds, true)
if err != nil {
msg.State = fmt.Errorf("unable to retrieve host info: %s", err)
msg.Response_ch <- msg
return
}
osif_sheep.Baa(2, "lazyupdate: Response_data = %s %s %s %s %s %s", safe(name), safe(id), safe(ip4), safe(phost), safe(mac), safe(gw))
msg.Response_data = Mk_netreq_vm(name, id, ip4, nil, phost, mac, gw, fip4, gwmap) // build the vm data block for network manager
msg.Response_ch <- msg // and send it on its merry way
return
}
/*
Executes as a goroutine to drive the reservation manager portion of tegu.
*/
func Res_manager( my_chan chan *ipc.Chmsg, cookie *string ) {
var (
inv *Inventory
msg *ipc.Chmsg
ckptd string
last_qcheck int64 = 0 // time that the last queue check was made to set window
last_chkpt int64 = 0 // time that the last checkpoint was written
retry_chkpt bool = false // checkpoint needs to be retried because of a timing issue
queue_gen_type = REQ_GEN_EPQMAP
alt_table = DEF_ALT_TABLE // table number where meta marking happens
all_sys_up bool = false; // set when we receive the all_up message; some functions (chkpt) must wait for this
hto_limit int = 3600 * 18 // OVS has a size limit to the hard timeout value, this caps it just under the OVS limit
res_refresh int64 = 0 // next time when we must force all reservations to refresh flow-mods (hto_limit nonzero)
rr_rate int = 3600 // refresh rate (1 hour)
favour_v6 bool = true // favour ipv6 addresses if a host has both defined.
)
super_cookie = cookie // global for all methods
rm_sheep = bleater.Mk_bleater( 0, os.Stderr ) // allocate our bleater and attach it to the master
rm_sheep.Set_prefix( "res_mgr" )
tegu_sheep.Add_child( rm_sheep ) // we become a child so that if the master vol is adjusted we'll react too
p := cfg_data["default"]["queue_type"] // lives in default b/c used by fq-mgr too
if p != nil {
if *p == "endpoint" {
queue_gen_type = REQ_GEN_EPQMAP
} else {
queue_gen_type = REQ_GEN_QMAP
}
}
p = cfg_data["default"]["alttable"] // alt table for meta marking
if p != nil {
alt_table = clike.Atoi( *p )
}
p = cfg_data["default"]["favour_ipv6"]
if p != nil {
favour_v6 = *p == "true"
}
if cfg_data["resmgr"] != nil {
cdp := cfg_data["resmgr"]["chkpt_dir"]
if cdp == nil {
ckptd = "/var/lib/tegu/resmgr" // default directory and prefix
} else {
ckptd = *cdp + "/resmgr" // add prefix to directory in config
}
p = cfg_data["resmgr"]["verbose"]
if p != nil {
rm_sheep.Set_level( uint( clike.Atoi( *p ) ) )
}
/*
p = cfg_data["resmgr"]["set_vlan"]
if p != nil {
set_vlan = *p == "true"
}
*/
p = cfg_data["resmgr"]["super_cookie"]
if p != nil {
super_cookie = p
rm_sheep.Baa( 1, "super-cookie was set from config file" )
}
p = cfg_data["resmgr"]["hto_limit"] // if OVS or whatever has a max timeout we can ensure it's not surpassed
if p != nil {
hto_limit = clike.Atoi( *p )
}
p = cfg_data["resmgr"]["res_refresh"] // rate that reservations are refreshed if hto_limit is non-zero
if p != nil {
rr_rate = clike.Atoi( *p )
if rr_rate < 900 {
if rr_rate < 120 {
rm_sheep.Baa( 0, "NOTICE: reservation refresh rate in config is insanely low (%ds) and was changed to 1800s", rr_rate )
rr_rate = 1800
} else {
rm_sheep.Baa( 0, "NOTICE: reservation refresh rate in config is too low: %ds", rr_rate )
}
}
}
}
send_meta_counter := 200; // send meta f-mods only now and again
rm_sheep.Baa( 1, "ovs table number %d used for metadata marking", alt_table )
res_refresh = time.Now().Unix() + int64( rr_rate ) // set first refresh in an hour (ignored if hto_limit not set
inv = Mk_inventory( )
inv.chkpt = chkpt.Mk_chkpt( ckptd, 10, 90 )
last_qcheck = time.Now().Unix()
tkl_ch := make( chan *ipc.Chmsg, 5 ) // special, short buffer, channel for tickles allows 5 to queue before blocking sender
tklr.Add_spot( 2, tkl_ch, REQ_PUSH, nil, ipc.FOREVER ) // push reservations to agent just before they go live
tklr.Add_spot( 1, tkl_ch, REQ_SETQUEUES, nil, ipc.FOREVER ) // drives us to see if queues need to be adjusted
tklr.Add_spot( 5, tkl_ch, REQ_RTRY_CHKPT, nil, ipc.FOREVER ) // ensures that we retried any missed checkpoints
tklr.Add_spot( 60, tkl_ch, REQ_VET_RETRY, nil, ipc.FOREVER ) // run the retry queue if it has size
go rm_lookup( rmgrlu_ch, inv )
rm_sheep.Baa( 3, "res_mgr is running %x", my_chan )
for {
select { // select next ready message on either channel
case msg = <- tkl_ch: // msg available on tickle channel
msg.State = nil // nil state is OK, no error
my_chan <- msg; // just pass it through; tkl_ch has a small buffer (blocks quickly) and this prevents filling the main queue w/ tickles if we get busy
case msg = <- my_chan: // process message from the main channel
rm_sheep.Baa( 3, "processing message: %d", msg.Msg_type )
switch msg.Msg_type {
case REQ_NOOP: // just ignore
case REQ_ADD:
msg.State = inv.Add_res( msg.Req_data ) // add will determine the pledge type and do the right thing
msg.Response_data = nil
case REQ_ALLUP: // signals that all initialisation is complete (chkpting etc. can go)
all_sys_up = true
// periodic checkpointing turned off with the introduction of tegu_ha
//tklr.Add_spot( 180, my_chan, REQ_CHKPT, nil, ipc.FOREVER ) // tickle spot to drive us every 180 seconds to checkpoint
case REQ_RTRY_CHKPT: // called to attempt to send a queued checkpoint request
if all_sys_up {
if retry_chkpt {
rm_sheep.Baa( 3, "invoking checkpoint (retry)" )
retry_chkpt, last_chkpt = inv.write_chkpt( last_chkpt )
}
}
case REQ_CHKPT: // external thread has requested checkpoint
if all_sys_up {
rm_sheep.Baa( 3, "invoking checkpoint" )
retry_chkpt, last_chkpt = inv.write_chkpt( last_chkpt )
}
case REQ_DEL: // user initiated delete -- requires cookie
data := msg.Req_data.( []*string ) // assume pointers to name and cookie
if data[0] != nil && *data[0] == "all" {
inv.Del_all_res( data[1] )
msg.State = nil
} else {
msg.State = inv.Del_res( data[0], data[1] )
}
inv.push_reservations( my_chan, alt_table, int64( hto_limit ), favour_v6 ) // must force a push to push augmented (shortened) reservations
msg.Response_data = nil
case REQ_DUPCHECK:
if msg.Req_data != nil {
msg.Response_data, msg.State = inv.dup_check( msg.Req_data.( *gizmos.Pledge ) )
}
case REQ_GET: // user initiated get -- requires cookie
data := msg.Req_data.( []*string ) // assume pointers to name and cookie
msg.Response_data, msg.State = inv.Get_res( data[0], data[1] )
case REQ_LIST: // list reservations (for a client)
msg.Response_data, msg.State = inv.res2json( )
case REQ_LOAD: // load from a checkpoint file
data := msg.Req_data.( *string ) // assume pointers to name and cookie
msg.State = inv.load_chkpt( data )
msg.Response_data = nil
rm_sheep.Baa( 1, "checkpoint file loaded" )
case REQ_PAUSE:
msg.State = nil // right now this cannot fail in ways we know about
msg.Response_data = ""
inv.pause_on()
res_refresh = 0; // must force a push of everything on next push tickle
rm_sheep.Baa( 1, "pausing..." )
case REQ_RESUME:
msg.State = nil // right now this cannot fail in ways we know about
msg.Response_data = ""
res_refresh = 0; // must force a push of everything on next push tickle
inv.pause_off()
case REQ_SETQUEUES: // driven about every second to reset the queues if a reservation state has changed
now := time.Now().Unix()
if now > last_qcheck && inv.any_concluded( now - last_qcheck ) || inv.any_commencing( now - last_qcheck, 0 ) {
rm_sheep.Baa( 1, "channel states: rm=%d rmlu=%d fq=%d net=%d agent=%d", len( rmgr_ch ), len( rmgrlu_ch ), len( fq_ch ), len( nw_ch ), len( am_ch ) )
rm_sheep.Baa( 1, "reservation state change detected, requesting queue map from net-mgr" )
tmsg := ipc.Mk_chmsg( )
tmsg.Send_req( nw_ch, my_chan, queue_gen_type, time.Now().Unix(), nil ) // get a queue map; when it arrives we'll push to fqmgr and trigger flow-mod push
}
last_qcheck = now
case REQ_PUSH: // driven every few seconds to check for need to refresh because of switch max timeout setting
if hto_limit > 0 { // if reservation flow-mods are capped with a hard timeout limit
now := time.Now().Unix()
if now > res_refresh {
rm_sheep.Baa( 2, "refreshing all reservations" )
inv.reset_push() // reset pushed flag on all reservations to cause active ones to be pushed again
res_refresh = now + int64( rr_rate ) // push everything again in an hour
inv.push_reservations( my_chan, alt_table, int64( hto_limit ), favour_v6 ) // force a push of all
}
}
case REQ_PLEDGE_LIST: // generate a list of pledges that are related to the given VM
msg.Response_data, msg.State = inv.pledge_list( msg.Req_data.( *string ) )
case REQ_SETULCAP: // user link capacity; expect array of two string pointers (name and value)
data := msg.Req_data.( []*string )
inv.add_ulcap( data[0], data[1] )
retry_chkpt, last_chkpt = inv.write_chkpt( last_chkpt )
// CAUTION: the requests below come back as asynch responses rather than as initial message
case REQ_IE_RESERVE: // an IE reservation failed
msg.Response_ch = nil // immediately disable to prevent loop
inv.failed_push( msg ) // suss out the pledge and mark it unpushed
case REQ_GEN_QMAP: // response caries the queue map that now should be sent to fq-mgr to drive a queue update
fallthrough
case REQ_GEN_EPQMAP:
rm_sheep.Baa( 1, "received queue map from network manager" )
qlist := msg.Response_data.( []string ) // get the qulist map for our use first
if send_meta_counter >= 200 {
send_meta_fmods( qlist, alt_table ) // push meta rules
send_meta_counter = 0
} else {
send_meta_counter++
}
msg.Response_ch = nil // immediately disable to prevent loop
fq_data := make( []interface{}, 1 )
fq_data[FQ_QLIST] = msg.Response_data
tmsg := ipc.Mk_chmsg( )
tmsg.Send_req( fq_ch, nil, REQ_SETQUEUES, fq_data, nil ) // send the queue list to fq manager to deal with
inv.push_reservations( my_chan, alt_table, int64( hto_limit ), favour_v6 ) // now safe to push reservations if any activated
case REQ_VET_RETRY:
if inv != nil && len( inv.retry ) > 0 {
inv.vet_retries( )
}
case REQ_YANK_RES: // yank a reservation from the inventory returning the pledge and allowing flow-mods to purge
if msg.Response_ch != nil {
msg.Response_data, msg.State = inv.yank_res( msg.Req_data.( *string ) )
}
/* deprecated -- moved to rm_lookup
case REQ_GET_MIRRORS: // user initiated get list of mirrors
t := inv.Get_mirrorlist()
msg.Response_data = &t;
*/
default:
rm_sheep.Baa( 0, "WRN: res_mgr: unknown message: %d [TGURMG001]", msg.Msg_type )
msg.Response_data = nil
msg.State = fmt.Errorf( "res_mgr: unknown message (%d)", msg.Msg_type )
msg.Response_ch = nil // we don't respond to these.
} // end main channel case
} // end select
rm_sheep.Baa( 3, "processing message complete: %d", msg.Msg_type )
if msg.Response_ch != nil { // if a response channel was provided
msg.Response_ch <- msg // send our result back to the requester
}
}
}
/*
For a single bandwidth pledge, this function sets things up and sends needed requests to the fq-manger to
create any necessary flow-mods. This has changed drastically now that we expect one agent
onvocation to set up all bandwidth flow-mods for an endpoint switch.
With the new method of managing queues per reservation on ingress/egress hosts, we now send
the following information to fq_mgr:
h1, h2 -- hosts
expiry
switch/port/queue
Path list will have path(s) in both directions (to support different bandwidth rates).
The above info is sent for each 'link' in the forward direction path, and then we reverse
the path and send requests to fq_mgr for each 'link' in the backwards direction. Errors are
returned to res_mgr via channel, but asycnh; we do not wait for responses to each message
generated here.
To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume)
use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If
to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit,
and we assume that this function is called periodically to update long running reservations.
Alt_table is the base alternate table set that we use for meta marking
If pref_ip6 is true, then if a host has both v4 and v6 addresses we will use the v6 address.
*/
func bw_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, alt_table int, pref_v6 bool) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_bw) // generic pledge better be a bw pledge!
if !ok {
rm_sheep.Baa(1, "internal error in push_bw_reservation: pledge isn't a bandwidth pledge")
(*gp).Set_pushed() // prevent looping
return
}
h1, h2, p1, p2, _, expiry, _, _ := p.Get_values() // hosts, transport (tcp/udp) ports and expiry are all we need
v1, v2 := p.Get_vlan() // vlan match criteria for one/both endpoints
ip1 := name2ip(h1)
ip2 := name2ip(h2)
if ip1 != nil && ip2 != nil { // good ip addresses so we're good to go
plist := p.Get_path_list() // each path that is a part of the reservation
timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice
for i := range plist { // for each path, send fmgr requests for each endpoint
freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things)
freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4?
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
freq.Single_switch = false // path involves multiple switches by default
freq.Dscp, freq.Dscp_koe = p.Get_dscp() // reservation supplied dscp value that we're to match and maybe preserve on exit
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
extip := plist[i].Get_extip() // if an external IP address is necessary on the freq get it
if extip != nil {
freq.Extip = extip
} else {
freq.Extip = &empty_str
}
espq1, _ := plist[i].Get_endpoint_spq(rname, timestamp) // end point switch, port, queue information; ep1 nil if single switch
if espq1 == nil { // if single switch ep1 will be nil
freq.Single_switch = true
}
freq.Match.Ip1 = plist[i].Get_h1().Get_address(pref_v6) // must use path h1/h2 as this could be the reverse with respect to the overall pledge and thus reverse of pledge
freq.Match.Ip2 = plist[i].Get_h2().Get_address(pref_v6)
freq.Espq = plist[i].Get_ilink_spq(rname, timestamp) // spq info comes from the first link off of the switch, not the endpoint link back to the VM
if freq.Single_switch {
freq.Espq.Queuenum = 1 // same switch always over br-rl queue 1
}
freq.Exttyp = plist[i].Get_extflag() // indicates whether the external IP is the source or dest along this path
tptype_list := p.Get_proto() // pick up protocol supplied on the reservation
if (*p1 != "0" || *p2 != "0") && *tptype_list == "" { // if either port is specified, and no specific proto on reservation
tpl := "udp tcp" // if port supplied, generate f-mods for both udp and tcp matches on the port
tptype_list = &tpl
}
tptype_toks := strings.Split(*tptype_list, " ")
for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr
cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy
cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none)
if *cfreq.Exttyp == "-S" { // indicates that this is a 'reverse' path (h2 sending) and we must invert the Tp port numbers and vland ids
cfreq.Match.Tpsport = p2
cfreq.Match.Tpdport = p1
cfreq.Match.Vlan_id = v2
} else {
cfreq.Match.Tpsport = p1
cfreq.Match.Tpdport = p2
cfreq.Match.Vlan_id = v1
}
rm_sheep.Baa(1, "res_mgr/push_rea: forward endpoint flow-mods for path %d: %s flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d ext=%s exp/fm_exp=%d/%d",
i, *rname, *cfreq.Exttyp, tptype_toks[tidx], *h1, *h2, *cfreq.Match.Ip1, *cfreq.Match.Ip2, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport,
cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, *cfreq.Extip, expiry, cfreq.Expiry)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, nil, REQ_BW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup
// WARNING: this is q-lite only -- there is no attempt to set up intermediate switches!
}
}
p.Set_pushed() // safe to mark the pledge as having been pushed.
}
}
/*
This builds a fq-mgr request and passes it to the fq-mgr to 'refine' and send along
to the agent-manager for ultimate execution. It might be possible to pass it
directly to the agent manager, but because res-mgr thinks in IP addresses, and
fq-manager (that might be the only source of IP->Mac translation which flow-mods
need) is still left in the middle.
CAUTION: this function is called for both new reservations, _and_ to refresh
reservations with expiry value further in the future than the switch can
handle. It is also invoked when pausing reservations and will cause new
flow-mods to be sent with a short duration timeout to flush existing flow-mods
from the switch.
*/
func bwow_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, pref_v6 bool) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_bwow) // generic pledge better be a bw oneway pledge!
if !ok {
rm_sheep.Baa(1, "internal mishap in push_bwow_res: pledge isn't a oneway pledge")
(*gp).Set_pushed() // prevent looping
return
}
src, dest, src_tpport, dest_tpport, _, expiry := p.Get_values() // hosts, transport ports, and expiry time
vlan := p.Get_vlan() // vlan match criteria for source
ip_src := name2ip(src)
ip_dest := name2ip(dest)
if ip_src != nil && ip_dest != nil { // good ip addresses so we're good to go
gate := p.Get_gate() // get the gate information that is applied for the oneway
if gate != nil { // be parinoid
//timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice
freq := Mk_fqreq(rname) // default flow mod request no match/actions
freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4?
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
freq.Single_switch = true // implied with a oneway, but set it anyway
freq.Dscp = p.Get_dscp() // reservation supplied dscp value that we're to match (koe is meaningless in one way)
freq.Dscp_koe = false // meaningless for oneway, but ensure it's false so flag isn't accidently set later
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force existing flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
freq.Match.Ip1 = gate.Get_src().Get_address(pref_v6) // should match pledge, but gate is the ultimate authority
freq.Match.Ip2 = gate.Get_dest().Get_address(pref_v6)
freq.Espq = gate.Get_spq(rname, now+16) // switch port queue
freq.Extip = gate.Get_extip() // returns nil if not an external and that's what we need
tptype_list := p.Get_proto() // pick up protocol supplied on the reservation
if (*src_tpport != "0" || *dest_tpport != "0") && *tptype_list == "" { // if port supplied we must set proto; default to both if
tpl := "udp tcp" // user didn't supply one
tptype_list = &tpl
}
tptype_toks := strings.Split(*tptype_list, " ")
for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr
cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy
cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none)
cfreq.Match.Tpsport = src_tpport
cfreq.Match.Tpdport = dest_tpport
cfreq.Match.Vlan_id = vlan
ip2_str := ""
if cfreq.Match.Ip2 != nil {
ip2_str = *cfreq.Match.Ip2
}
rm_sheep.Baa(1, "res_mgr/push_bwow: flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d exp/fm_exp=%d/%d",
*rname, tptype_toks[tidx], *src, *dest, *cfreq.Match.Ip1, ip2_str, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport,
cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, expiry, cfreq.Expiry)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, nil, REQ_BWOW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup
}
}
p.Set_pushed() // safe to mark the pledge as having been pushed.
} else {
rm_sheep.Baa(1, "oneway not pushed: could not map one/both hosts to an IP address")
}
}
/*
executed as a goroutine this loops waiting for messages from the tickler and takes
action based on what is needed.
*/
func Osif_mgr(my_chan chan *ipc.Chmsg) {
var (
msg *ipc.Chmsg
os_list string = ""
os_sects []string // sections in the config file
os_refs map[string]*ostack.Ostack // creds for each project we need to request info from
os_projects map[string]*osif_project // list of project info (maps)
os_admin *ostack.Ostack // admin creds
refresh_delay int = 15 // config file can override
id2pname map[string]*string // project id/name translation maps
pname2id map[string]*string
req_token bool = false // if set to true in config file the token _must_ be present when called to validate
def_passwd *string // defaults and what we assume are the admin creds
def_usr *string
def_url *string
def_project *string
def_region *string
)
osif_sheep = bleater.Mk_bleater(0, os.Stderr) // allocate our bleater and attach it to the master
osif_sheep.Set_prefix("osif_mgr")
tegu_sheep.Add_child(osif_sheep) // we become a child so that if the master vol is adjusted we'll react too
//ostack.Set_debugging( 0 );
// ---- pick up configuration file things of interest --------------------------
if cfg_data["osif"] != nil { // cannot imagine that this section is missing, but don't fail if it is
def_passwd = cfg_data["osif"]["passwd"] // defaults applied if non-section given in list, or info omitted from the section
def_usr = cfg_data["osif"]["usr"]
def_url = cfg_data["osif"]["url"]
def_project = cfg_data["osif"]["project"]
p := cfg_data["osif"]["refresh"]
if p != nil {
refresh_delay = clike.Atoi(*p)
if refresh_delay < 15 {
osif_sheep.Baa(1, "resresh was too small (%ds), setting to 15", refresh_delay)
refresh_delay = 15
}
}
p = cfg_data["osif"]["debug"]
if p != nil {
v := clike.Atoi(*p)
if v > -5 {
ostack.Set_debugging(v)
}
}
p = cfg_data["osif"]["region"]
if p != nil {
def_region = p
}
p = cfg_data["osif"]["ostack_list"] // preferred placement in osif section
if p == nil {
p = cfg_data["default"]["ostack_list"] // originally in default, so backwards compatable
}
if p != nil {
os_list = *p
}
p = cfg_data["osif"]["require_token"]
if p != nil && *p == "true" {
req_token = true
}
p = cfg_data["osif"]["verbose"]
if p != nil {
osif_sheep.Set_level(uint(clike.Atoi(*p)))
}
}
if os_list == " " || os_list == "" || os_list == "off" {
osif_sheep.Baa(0, "osif disabled: no openstack list (ostack_list) defined in configuration file or setting is 'off'")
} else {
// TODO -- investigate getting id2pname maps from each specific set of creds defined if an overarching admin name is not given
os_admin = get_admin_creds(def_url, def_usr, def_passwd, def_project, def_region) // this will block until we authenticate
if os_admin != nil {
osif_sheep.Baa(1, "admin creds generated, mapping tenants")
pname2id, id2pname, _ = os_admin.Map_tenants() // list only projects we belong to
for k, v := range pname2id {
osif_sheep.Baa(1, "project known: %s %s", k, *v) // useful to see in log what projects we can see
}
} else {
id2pname = make(map[string]*string) // empty maps and we'll never generate a translation from project name to tenant ID since there are no default admin creds
pname2id = make(map[string]*string)
if def_project != nil {
osif_sheep.Baa(0, "WRN: unable to use admin information (%s, proj=%s, reg=%s) to authorise with openstack [TGUOSI009]", def_usr, def_project, def_region)
} else {
osif_sheep.Baa(0, "WRN: unable to use admin information (%s, proj=no-project, reg=%s) to authorise with openstack [TGUOSI009]", def_usr, def_region) // YES msg ids are duplicated here
}
}
if os_list == "all" {
os_refs, _ = refresh_creds(os_admin, os_refs, id2pname) // for each project in id2pname get current ostack struct (auth)
for k := range os_refs {
osif_sheep.Baa(1, "inital os_list member: %s", k)
}
} else {
if strings.Index(os_list, ",") > 0 {
os_sects = strings.Split(os_list, ",")
} else {
os_sects = strings.Split(os_list, " ")
}
os_refs = make(map[string]*ostack.Ostack, len(os_sects)*2) // length is a guideline, not a hard value
for i := 0; i < len(os_sects); i++ {
osif_sheep.Baa(1, "creating openstack interface for %s", os_sects[i])
url := def_url
usr := def_usr
passwd := def_passwd
project := &os_sects[i]
if cfg_data[os_sects[i]] != nil { // section name supplied, override defaults with information from the section
if cfg_data[os_sects[i]]["url"] != nil {
url = cfg_data[os_sects[i]]["url"]
}
if cfg_data[os_sects[i]]["usr"] != nil {
usr = cfg_data[os_sects[i]]["usr"]
}
if cfg_data[os_sects[i]]["passwd"] != nil {
passwd = cfg_data[os_sects[i]]["passwd"]
}
if cfg_data[os_sects[i]]["project"] != nil {
project = cfg_data[os_sects[i]]["project"]
}
}
os_refs[*project] = ostack.Mk_ostack(url, usr, passwd, project)
os_refs["_ref_"] = os_refs[*project] // a quick access reference when any one will do
}
}
os_projects = make(map[string]*osif_project)
add2projects(os_projects, os_refs, pname2id, 0) // add refernces to the projects list
}
// ---------------- end config parsing ----------------------------------------
if os_admin != nil { // only if we are using openstack as a database
//tklr.Add_spot( 3, my_chan, REQ_GENCREDS, nil, 1 ) // add tickle spot to drive us once in 3s and then another to drive us based on config refresh rate
tklr.Add_spot(int64(180), my_chan, REQ_GENCREDS, nil, ipc.FOREVER)
}
osif_sheep.Baa(2, "osif manager is running %x", my_chan)
for {
msg = <-my_chan // wait for next message from tickler
msg.State = nil // default to all OK
osif_sheep.Baa(3, "processing request: %d", msg.Msg_type)
switch msg.Msg_type {
case REQ_GENMAPS: // driven by tickler
// deprecated with switch to lazy update
case REQ_GENCREDS: // driven by tickler now and then
if os_admin != nil {
os_refs, pname2id, id2pname = update_project(os_admin, os_refs, os_projects, pname2id, id2pname, os_list == "all")
}
/* ---- before lite ----
case REQ_VM2IP: // driven by tickler; gen a new vm translation map and push to net mgr
m := mapvm2ip( os_refs )
if m != nil {
count := 0;
msg := ipc.Mk_chmsg( )
msg.Send_req( nw_ch, nil, REQ_VM2IP, m, nil ) // send new map to network as it is managed there
osif_sheep.Baa( 2, "VM2IP mapping updated from openstack" )
for k, v := range m {
osif_sheep.Baa( 3, "VM mapped: %s ==> %s", k, *v )
count++;
}
osif_sheep.Baa( 2, "mapped %d VM names/IDs from openstack (verbose 3 for debug list)", count )
}
*/
case REQ_IP2MACMAP: // generate an ip to mac map and send to those who need it (fq_mgr at this point)
freq := ipc.Mk_chmsg() // need a new request to pass to fq_mgr
data, err := get_ip2mac(os_projects)
if err == nil {
osif_sheep.Baa(2, "sending ip2mac map to fq_mgr")
freq.Send_req(fq_ch, nil, REQ_IP2MACMAP, data, nil) // request data forward
msg.State = nil // response ok back to requestor
} else {
msg.State = err // error goes back to requesting process
}
case REQ_CHOSTLIST:
if msg.Response_ch != nil { // no sense going off to ostack if no place to send the list
osif_sheep.Baa(2, "starting list host")
msg.Response_data, msg.State = get_hosts(os_refs)
osif_sheep.Baa(2, "finishing list host")
} else {
osif_sheep.Baa(0, "WRN: no response channel for host list request [TGUOSI012]")
}
/* ======= don't think these are needed but holding ======
case REQ_PROJNAME2ID: // translate a project name (tenant) to ID
if msg.Response_ch != nil {
pname := msg.Req_data.( *string )
if s, ok := pname2id[*pname]; ok { // translate if there, else assume it's in it's "final" form
msg.Response_data = s
} else {
msg.Response_data = pname
}
}
*/
case REQ_VALIDATE_TOKEN: // given token/tenant validate it and translate tenant name to ID if given; returns just ID
if msg.Response_ch != nil {
s := msg.Req_data.(*string)
*s += "/" // add trailing slant to simulate "data"
if !have_project(s, pname2id, id2pname) { // ensure that we have creds for this project, if not attempt to get
os_refs, pname2id, id2pname = update_project(os_admin, os_refs, os_projects, pname2id, id2pname, os_list == "all")
}
msg.Response_data, msg.State = validate_token(s, os_refs, pname2id, req_token)
}
case REQ_GET_HOSTINFO: // dig out all of the bits of host info for a single host from openstack and return in a network update struct
if msg.Response_ch != nil {
go get_os_hostinfo(msg, os_refs, os_projects, id2pname, pname2id) // do it asynch and return the result on the message channel
msg = nil // prevent early response
}
case REQ_GET_PROJ_HOSTS:
if msg.Response_ch != nil {
go get_all_osvm_info(msg, os_refs, os_projects, id2pname, pname2id) // do it asynch and return the result on the message channel
msg = nil // prevent response from this function
}
case REQ_GET_DEFGW: // dig out the default gateway for a project
if msg.Response_ch != nil {
go get_os_defgw(msg, os_refs, os_projects, id2pname, pname2id) // do it asynch and return the result on the message channel
msg = nil // prevent early response
}
case REQ_VALIDATE_HOST: // validate and translate a [token/]project-name/host string
if msg.Response_ch != nil {
if !have_project(msg.Req_data.(*string), pname2id, id2pname) { // ensure that we have creds for this project, if not attempt to get
os_refs, pname2id, id2pname = update_project(os_admin, os_refs, os_projects, pname2id, id2pname, os_list == "all")
}
msg.Response_data, msg.State = validate_token(msg.Req_data.(*string), os_refs, pname2id, req_token)
}
case REQ_XLATE_HOST: // accepts a [token/][project/]host name and translate project to an ID
if msg.Response_ch != nil {
if !have_project(msg.Req_data.(*string), pname2id, id2pname) { // ensure that we have creds for this project, if not attempt to get
os_refs, pname2id, id2pname = update_project(os_admin, os_refs, os_projects, pname2id, id2pname, os_list == "all")
}
msg.Response_data, msg.State = validate_token(msg.Req_data.(*string), os_refs, pname2id, false) // same process as validation but token not required
}
case REQ_VALIDATE_TEGU_ADMIN: // validate that the token is for the tegu user
if msg.Response_ch != nil {
if !have_project(msg.Req_data.(*string), pname2id, id2pname) { // ensure that we have creds for this project, if not attempt to get
os_refs, pname2id, id2pname = update_project(os_admin, os_refs, os_projects, pname2id, id2pname, os_list == "all")
}
msg.State = validate_admin_token(os_admin, msg.Req_data.(*string), def_usr)
msg.Response_data = ""
}
case REQ_HAS_ANY_ROLE: // given a token and list of roles, returns true if any role listed is listed by openstack for the token
if msg.Response_ch != nil {
d := msg.Req_data.(*string)
dtoks := strings.Split(*d, " ") // data assumed to be token <space> role[,role...]
if len(dtoks) > 1 {
msg.Response_data, msg.State = has_any_role(os_refs, os_admin, &dtoks[0], &dtoks[1])
} else {
msg.State = fmt.Errorf("has_any_role: bad input data")
msg.Response_data = false
}
}
case REQ_PNAME2ID: // user, project, tenant (what ever) name to ID
if msg.Response_ch != nil {
msg.Response_data = pname2id[*(msg.Req_data.(*string))]
if msg.Response_data.(*string) == nil { // maybe it was an ID that came in
if id2pname[*(msg.Req_data.(*string))] != nil { // if in id map, then return the stirng (the id) they passed (#202)
msg.Response_data = msg.Req_data.(*string)
} else {
msg.Response_data = nil // couldn't translate
}
}
}
default:
osif_sheep.Baa(1, "unknown request: %d", msg.Msg_type)
msg.Response_data = nil
if msg.Response_ch != nil {
msg.State = fmt.Errorf("osif: unknown request (%d)", msg.Msg_type)
}
}
if msg != nil { // if msg wasn't passed off to a go routine
osif_sheep.Baa(3, "processing request complete: %d", msg.Msg_type)
if msg.Response_ch != nil { // if a reqponse channel was provided
msg.Response_ch <- msg // send our result back to the requestor
}
}
}
}
/*
Opens the filename passed in and reads the reservation data from it. The assumption is
that records in the file were saved via the write_chkpt() function and are JSON pledges
or other serializable objects. We will drop any pledges that expired while 'sitting'
in the file.
*/
func (i *Inventory) load_chkpt(fname *string) (err error) {
var (
rec string
nrecs int = 0
p *gizmos.Pledge
my_ch chan *ipc.Chmsg
req *ipc.Chmsg
)
err = nil
my_ch = make(chan *ipc.Chmsg)
defer close(my_ch) // close it on return
f, err := os.Open(*fname)
if err != nil {
return
}
defer f.Close()
br := bufio.NewReader(f)
for err == nil {
rec, err = br.ReadString('\n')
if err == nil {
nrecs++
switch rec[0:5] {
case "ucap:":
toks := strings.Split(rec, " ")
if len(toks) == 3 {
i.add_ulcap(&toks[1], &toks[2])
}
default:
p, err = gizmos.Json2pledge(&rec) // convert any type of json pledge to Pledge
if err == nil {
if (*p).Is_expired() {
rm_sheep.Baa(1, "resmgr: ckpt_load: ignored expired pledge: %s", (*p).String())
} else {
switch sp := (*p).(type) { // work on specific pledge type, but pass the Pledge interface to add()
case *gizmos.Pledge_mirror:
err = i.Add_res(p) // assume we can just add it back in as is
case *gizmos.Pledge_steer:
rm_sheep.Baa(0, "did not restore steering reservation from checkpoint; not implemented")
case *gizmos.Pledge_bwow:
h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph
push_block := h2 == nil
update_graph(h1, push_block, push_block) // dig h1 info; push to netmgr if h2 isn't known and block on response
if h2 != nil {
update_graph(h2, true, true) // dig h2 data and push to netmgr blocking for a netmgr response
}
req = ipc.Mk_chmsg() // now safe to ask netmgr to validate the oneway pledge
req.Send_req(nw_ch, my_ch, REQ_BWOW_RESERVE, sp, nil)
req = <-my_ch // should be OK, but the underlying network could have changed
if req.Response_data != nil {
gate := req.Response_data.(*gizmos.Gate) // expect that network sent us a gate
sp.Set_gate(gate)
rm_sheep.Baa(1, "gate allocated for oneway reservation: %s %s %s %s", *(sp.Get_id()), *h1, *h2, *(gate.Get_extip()))
err = i.Add_res(p)
} else {
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for oneway pledge: %s [TGURMG000]", (*p).To_str())
}
case *gizmos.Pledge_bw:
h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph
update_graph(h1, false, false) // don't need to block on this one, nor update fqmgr
update_graph(h2, true, true) // wait for netmgr to update graph and then push related data to fqmgr
req = ipc.Mk_chmsg() // now safe to ask netmgr to find a path for the pledge
rm_sheep.Baa(2, "reserving path starts")
req.Send_req(nw_ch, my_ch, REQ_BW_RESERVE, sp, nil)
req = <-my_ch // should be OK, but the underlying network could have changed
if req.Response_data != nil {
rm_sheep.Baa(2, "reserving path finished")
path_list := req.Response_data.([]*gizmos.Path) // path(s) that were found to be suitable for the reservation
sp.Set_path_list(path_list)
rm_sheep.Baa(1, "path allocated for chkptd reservation: %s %s %s; path length= %d", *(sp.Get_id()), *h1, *h2, len(path_list))
err = i.Add_res(p)
} else {
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for pledge: %s [TGURMG000]", (*p).To_str())
}
case *gizmos.Pledge_pass:
host, _ := sp.Get_hosts()
update_graph(host, true, true)
req.Send_req(nw_ch, my_ch, REQ_GETPHOST, host, nil) // need to find the current phost for the vm
req = <-my_ch
if req.Response_data != nil {
phost := req.Response_data.(*string)
sp.Set_phost(phost)
rm_sheep.Baa(1, "passthrou phost found for chkptd reservation: %s %s %s", *(sp.Get_id()), *host, *phost)
err = i.Add_res(p)
} else {
s := fmt.Errorf("unknown reason")
if req.State != nil {
s = req.State
}
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to find phost for passthru pledge: %s [TGURMG000]", s)
rm_sheep.Baa(0, "erroring passthru pledge: %s", (*p).To_str())
}
default:
rm_sheep.Baa(0, "rmgr/load_ckpt: unrecognised pledge type")
} // end switch on specific pledge type
}
} else {
rm_sheep.Baa(0, "CRI: %s", err)
return // quickk escape
}
} // outer switch
}
}
if err == io.EOF {
err = nil
}
rm_sheep.Baa(1, "read %d records from checkpoint file: %s", nrecs, *fname)
return
}
/*
the main go routine to act on messages sent to our channel. We expect messages from the
reservation manager, and from a tickler that causes us to evaluate the need to resize
ovs queues.
DSCP values: Dscp values range from 0-64 decimal, but when described on or by
flow-mods are shifted two bits to the left. The send flow mod function will
do the needed shifting so all values outside of that one funciton should assume/use
decimal values in the range of 0-64.
*/
func Fq_mgr(my_chan chan *ipc.Chmsg, sdn_host *string) {
var (
uri_prefix string = ""
msg *ipc.Chmsg
data []interface{} // generic list of data on some requests
fdata *Fq_req // flow-mod request data
qcheck_freq int64 = 5
hcheck_freq int64 = 180
host_list *string // current set of openstack real hosts
ip2mac map[string]*string // translation from ip address to mac
switch_hosts *string // from config file and overrides openstack list if given (mostly testing)
ssq_cmd *string // command string used to set switch queues (from config file)
send_all bool = false // send all flow-mods; false means send just ingress/egress and not intermediate switch f-mods
alt_table int = DEF_ALT_TABLE // meta data marking table
phost_suffix *string = nil // physical host suffix added to each host name in the list from openstack (config)
//max_link_used int64 = 0 // the current maximum link utilisation
)
fq_sheep = bleater.Mk_bleater(0, os.Stderr) // allocate our bleater and attach it to the master
fq_sheep.Set_prefix("fq_mgr")
tegu_sheep.Add_child(fq_sheep) // we become a child so that if the master vol is adjusted we'll react too
// -------------- pick up config file data if there --------------------------------
if *sdn_host == "" { // not supplied on command line, pull from config
if sdn_host = cfg_data["default"]["sdn_host"]; sdn_host == nil { // no default; when not in config, then it's turned off and we send to agent
sdn_host = &empty_str
}
}
if cfg_data["default"]["queue_type"] != nil {
if *cfg_data["default"]["queue_type"] == "endpoint" {
send_all = false
} else {
send_all = true
}
}
if p := cfg_data["default"]["alttable"]; p != nil { // this is the base; we use alt_table to alt_table + (n-1) when we need more than 1 table
alt_table = clike.Atoi(*p)
}
if cfg_data["fqmgr"] != nil { // pick up things in our specific setion
if dp := cfg_data["fqmgr"]["ssq_cmd"]; dp != nil { // set switch queue command
ssq_cmd = dp
}
/*
if p := cfg_data["fqmgr"]["default_dscp"]; p != nil { // this is a single value and should not be confused with the dscp list in the default section of the config
dscp = clike.Atoi( *p )
}
*/
if p := cfg_data["fqmgr"]["queue_check"]; p != nil { // queue check frequency from the control file
qcheck_freq = clike.Atoi64(*p)
if qcheck_freq < 5 {
qcheck_freq = 5
}
}
if p := cfg_data["fqmgr"]["host_check"]; p != nil { // frequency of checking for new _real_ hosts from openstack
hcheck_freq = clike.Atoi64(*p)
if hcheck_freq < 30 {
hcheck_freq = 30
}
}
if p := cfg_data["fqmgr"]["switch_hosts"]; p != nil {
switch_hosts = p
}
if p := cfg_data["fqmgr"]["verbose"]; p != nil {
fq_sheep.Set_level(uint(clike.Atoi(*p)))
}
if p := cfg_data["fqmgr"]["phost_suffix"]; p != nil { // suffix added to physical host strings for agent commands
if *p != "" {
phost_suffix = p
fq_sheep.Baa(1, "physical host names will be suffixed with: %s", *phost_suffix)
}
}
}
// ----- end config file munging ---------------------------------------------------
//tklr.Add_spot( qcheck_freq, my_chan, REQ_SETQUEUES, nil, ipc.FOREVER ); // tickle us every few seconds to adjust the ovs queues if needed
if switch_hosts == nil {
tklr.Add_spot(2, my_chan, REQ_CHOSTLIST, nil, 1) // tickle once, very soon after starting, to get a host list
tklr.Add_spot(hcheck_freq, my_chan, REQ_CHOSTLIST, nil, ipc.FOREVER) // tickles us every once in a while to update host list
fq_sheep.Baa(2, "host list will be requested from openstack every %ds", hcheck_freq)
} else {
host_list = switch_hosts
fq_sheep.Baa(0, "static host list from config used for setting OVS queues: %s", *host_list)
}
if sdn_host != nil && *sdn_host != "" {
uri_prefix = fmt.Sprintf("http://%s", *sdn_host)
}
fq_sheep.Baa(1, "flowmod-queue manager is running, sdn host: %s", *sdn_host)
for {
msg = <-my_chan // wait for next message
msg.State = nil // default to all OK
fq_sheep.Baa(3, "processing message: %d", msg.Msg_type)
switch msg.Msg_type {
case REQ_GEN_FMOD: // generic fmod; just pass it along w/o any special handling
if msg.Req_data != nil {
fdata = msg.Req_data.(*Fq_req) // pointer at struct with all of our expected goodies
send_gfmod_agent(fdata, ip2mac, host_list, phost_suffix)
}
case REQ_BWOW_RESERVE: // oneway bandwidth flow-mod generation
msg.Response_ch = nil // nothing goes back from this
fdata = msg.Req_data.(*Fq_req) // pointer at struct with all of the expected goodies
send_bwow_fmods(fdata, ip2mac, phost_suffix)
case REQ_BW_RESERVE: // bandwidth endpoint flow-mod creation; single agent script creates all needed fmods
fdata = msg.Req_data.(*Fq_req) // pointer at struct with all of the expected goodies
send_bw_fmods(fdata, ip2mac, phost_suffix)
msg.Response_ch = nil // nothing goes back from this
case REQ_IE_RESERVE: // proactive ingress/egress reservation flowmod (this is likely deprecated as of 3/21/2015 -- resmgr invokes the bw_fmods script via agent)
fdata = msg.Req_data.(*Fq_req) // user view of what the flow-mod should be
if uri_prefix != "" { // an sdn controller -- skoogi -- is enabled
msg.State = gizmos.SK_ie_flowmod(&uri_prefix, *fdata.Match.Ip1, *fdata.Match.Ip2, fdata.Expiry, fdata.Espq.Queuenum, fdata.Espq.Switch, fdata.Espq.Port)
if msg.State == nil { // no error, no response to requestor
fq_sheep.Baa(2, "proactive reserve successfully sent: uri=%s h1=%s h2=%s exp=%d qnum=%d swid=%s port=%d dscp=%d",
uri_prefix, fdata.Match.Ip1, fdata.Match.Ip2, fdata.Expiry, fdata.Espq.Queuenum, fdata.Espq.Switch, fdata.Espq.Port)
msg.Response_ch = nil
} else {
// do we need to suss out the id and mark it failed, or set a timer on it, so as not to flood reqmgr with errors?
fq_sheep.Baa(1, "ERR: proactive reserve failed: uri=%s h1=%s h2=%s exp=%d qnum=%d swid=%s port=%d [TGUFQM008]",
uri_prefix, fdata.Match.Ip1, fdata.Match.Ip2, fdata.Expiry, fdata.Espq.Queuenum, fdata.Espq.Switch, fdata.Espq.Port)
}
} else {
// q-lite now generates one flowmod in each direction because of the ITONS requirements
if send_all || fdata.Espq.Queuenum > 1 { // if sending all fmods, or this has a non-intermediate queue
cdata := fdata.Clone() // copy so we can alter w/o affecting sender's copy
if cdata.Espq.Port == -128 { // we'll assume in this case that the switch given is the host name and we need to set the switch to br-int
swid := "br-int"
cdata.Swid = &swid
}
if cdata.Resub == nil {
resub_list := "" // resub to alternate table to set a meta mark, then to table 0 to hit openstack junk
if cdata.Single_switch || fdata.Dir_in { // must use the base table for inbound traffic OR same switch traffic (bug 2015/1/26)
resub_list = fmt.Sprintf("%d 0", alt_table) // base alt_table is for 'local' traffic (trafic that doesn't go through br-rl
} else {
resub_list = fmt.Sprintf("%d 0", alt_table+1) // base+1 is for OUTBOUND only traffic that must go through the rate limiting bridge
}
cdata.Resub = &resub_list
}
meta := "0x00/0x07" // match-value/mask; match only when meta neither of our two bits, nor the agent bit (0x04) are set
cdata.Match.Meta = &meta
if fdata.Dir_in { // inbound to this switch we need to revert dscp from our settings to the 'origianal' settings
if cdata.Single_switch {
cdata.Match.Dscp = -1 // there is no match if both on same switch
send_gfmod_agent(cdata, ip2mac, host_list, phost_suffix)
} else {
cdata.Match.Dscp = cdata.Dscp // match the dscp that was added on ingress
if !cdata.Dscp_koe { // dropping the value on exit
cdata.Action.Dscp = 0 // set action to turn it off, otherwise we let it ride (no overt action)
}
send_gfmod_agent(cdata, ip2mac, host_list, phost_suffix)
}
} else { // outbound from this switch set the dscp value specified on the reservation
cdata.Match.Dscp = -1 // on outbound there is no dscp match, ensure this is off
if cdata.Single_switch {
send_gfmod_agent(cdata, ip2mac, host_list, phost_suffix) // in single switch mode there is no dscp value needed
} else {
cdata.Action.Dscp = cdata.Dscp // otherwise set the value and send
send_gfmod_agent(cdata, ip2mac, host_list, phost_suffix)
}
}
}
msg.Response_ch = nil
}
case REQ_ST_RESERVE: // reservation fmods for traffic steering
msg.Response_ch = nil // for now, nothing goes back
if msg.Req_data != nil {
fq_data := msg.Req_data.(*Fq_req) // request data
if uri_prefix != "" { // an sdn controller -- skoogi -- is enabled (not supported)
fq_sheep.Baa(0, "ERR: steering reservations are not supported with skoogi (SDNC); no flow-mods pushed")
} else {
send_stfmod_agent(fq_data, ip2mac, host_list)
}
} else {
fq_sheep.Baa(0, "CRI: missing data on st-reserve request to fq-mgr")
}
case REQ_SK_RESERVE: // send a reservation to skoogi
data = msg.Req_data.([]interface{}) // msg data expected to be array of interface: h1, h2, expiry, queue h1/2 must be IP addresses
if uri_prefix != "" {
fq_sheep.Baa(2, "msg to reserve: %s %s %s %d %d", uri_prefix, data[0].(string), data[1].(string), data[2].(int64), data[3].(int))
msg.State = gizmos.SK_reserve(&uri_prefix, data[0].(string), data[1].(string), data[2].(int64), data[3].(int))
} else {
fq_sheep.Baa(1, "reservation not sent, no sdn-host defined: %s %s %s %d %d", uri_prefix, data[0].(string), data[1].(string), data[2].(int64), data[3].(int))
}
case REQ_SETQUEUES: // request from reservation manager which indicates something changed and queues need to be reset
qlist := msg.Req_data.([]interface{})[0].([]string)
if ssq_cmd != nil {
adjust_queues(qlist, ssq_cmd, host_list) // if writing to a file and driving a local script
} else {
adjust_queues_agent(qlist, host_list, phost_suffix) // if sending json to an agent
}
case REQ_CHOSTLIST: // this is tricky as it comes from tickler as a request, and from osifmgr as a response, be careful!
msg.Response_ch = nil // regardless of source, we should not reply to this request
if msg.State != nil || msg.Response_data != nil { // response from ostack if with list or error
if msg.Response_data.(*string) != nil {
hls := strings.TrimLeft(*(msg.Response_data.(*string)), " \t") // ditch leading whitespace
hl := &hls
if *hl != "" {
host_list = hl // ok to use it
if phost_suffix != nil {
fq_sheep.Baa(2, "host list from osif before suffix added: %s", *host_list)
host_list = add_phost_suffix(host_list, phost_suffix) // in some cases ostack sends foo, but we really need to use foo-suffix (sigh)
}
send_hlist_agent(host_list) // send to agent_manager
fq_sheep.Baa(2, "host list received from osif: %s", *host_list)
} else {
fq_sheep.Baa(1, "host list received from osif was discarded: ()")
}
} else {
fq_sheep.Baa(0, "WRN: no data from openstack; expected host list string [TGUFQM009]")
}
} else {
req_hosts(my_chan, fq_sheep) // send requests to osif for data
}
case REQ_IP2MACMAP: // a new map from osif
if msg.Req_data != nil {
newmap := msg.Req_data.(map[string]*string)
if len(newmap) > 0 {
ip2mac = newmap // safe to replace
fq_sheep.Baa(2, "ip2mac translation received from osif: %d elements", len(ip2mac))
} else {
if ip2mac != nil {
fq_sheep.Baa(2, "ip2mac translation received from osif: 0 elements -- kept old table with %d elements", len(ip2mac))
} else {
fq_sheep.Baa(2, "ip2mac translation received from osif: 0 elements -- no existing table to keep")
}
}
} else {
fq_sheep.Baa(0, "WRN: no data from osif (nil map); expected ip2mac translation map [TGUFQM010]")
}
msg.State = nil // state is always good
default:
fq_sheep.Baa(1, "unknown request: %d", msg.Msg_type)
msg.Response_data = nil
if msg.Response_ch != nil {
msg.State = fmt.Errorf("unknown request (%d)", msg.Msg_type)
}
}
fq_sheep.Baa(3, "processing message complete: %d", msg.Msg_type)
if msg.Response_ch != nil { // if a reqponse channel was provided
fq_sheep.Baa(3, "sending response: %d", msg.Msg_type)
msg.Response_ch <- msg // send our result back to the requestor
}
}
}