/*
For a single passthrough pledge, this function sets things up and sends needed requests to the fq-manger to
create any necessary flow-mods.
We send the following information to fq_mgr:
source mac or endpoint (VM-- the host in the pledge)
source IP and optionally port and protocol more specific reservations
expiry
switch (physical host -- compute node)
Errors are returned to res_mgr via channel, but asycnh; we do not wait for responses to each message
generated here.
To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume)
use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If
to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit,
and we assume that this function is called periodically to update long running reservations.
*/
func pass_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_pass) // generic pledge better be a passthrough pledge!
if !ok {
rm_sheep.Baa(1, "internal error in pass_push_reservation: pledge isn't a passthrough pledge")
(*gp).Set_pushed() // prevent looping
return
}
host, _, _, expiry, proto := p.Get_values() // reservation info that we need
ip := name2ip(host)
if ip != nil { // good ip addresses so we're good to go
freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things)
freq.Match.Smac = ip // fq_mgr has conversion map to convert to mac
freq.Swid = p.Get_phost() // the phyiscal host where the VM lives and where fmods need to be deposited
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
freq.Extip = &empty_str
// this will change when ported to endpoint branch as the endpoint allows address and port 'in line'
freq.Match.Ip1 = proto // the proto on the reservation should be [{udp|tcp:}]address[:port]
freq.Match.Ip2 = nil
freq.Espq = nil
dup_str := ""
freq.Exttyp = &dup_str
rm_sheep.Baa(1, "pushing passthru reservation: %s", p)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, ch, REQ_PT_RESERVE, freq, nil) // queue work with fq-manger to read the struct and send cmd(s) to agent to get it done
p.Set_pushed() // safe to mark the pledge as having been pushed.
}
}
/*
This builds a fq-mgr request and passes it to the fq-mgr to 'refine' and send along
to the agent-manager for ultimate execution. It might be possible to pass it
directly to the agent manager, but because res-mgr thinks in IP addresses, and
fq-manager (that might be the only source of IP->Mac translation which flow-mods
need) is still left in the middle.
CAUTION: this function is called for both new reservations, _and_ to refresh
reservations with expiry value further in the future than the switch can
handle. It is also invoked when pausing reservations and will cause new
flow-mods to be sent with a short duration timeout to flush existing flow-mods
from the switch.
*/
func bwow_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, pref_v6 bool) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_bwow) // generic pledge better be a bw oneway pledge!
if !ok {
rm_sheep.Baa(1, "internal mishap in push_bwow_res: pledge isn't a oneway pledge")
(*gp).Set_pushed() // prevent looping
return
}
src, dest, src_tpport, dest_tpport, _, expiry := p.Get_values() // hosts, transport ports, and expiry time
vlan := p.Get_vlan() // vlan match criteria for source
ip_src := name2ip(src)
ip_dest := name2ip(dest)
if ip_src != nil && ip_dest != nil { // good ip addresses so we're good to go
gate := p.Get_gate() // get the gate information that is applied for the oneway
if gate != nil { // be parinoid
//timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice
freq := Mk_fqreq(rname) // default flow mod request no match/actions
freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4?
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
freq.Single_switch = true // implied with a oneway, but set it anyway
freq.Dscp = p.Get_dscp() // reservation supplied dscp value that we're to match (koe is meaningless in one way)
freq.Dscp_koe = false // meaningless for oneway, but ensure it's false so flag isn't accidently set later
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force existing flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
freq.Match.Ip1 = gate.Get_src().Get_address(pref_v6) // should match pledge, but gate is the ultimate authority
freq.Match.Ip2 = gate.Get_dest().Get_address(pref_v6)
freq.Espq = gate.Get_spq(rname, now+16) // switch port queue
freq.Extip = gate.Get_extip() // returns nil if not an external and that's what we need
tptype_list := p.Get_proto() // pick up protocol supplied on the reservation
if (*src_tpport != "0" || *dest_tpport != "0") && *tptype_list == "" { // if port supplied we must set proto; default to both if
tpl := "udp tcp" // user didn't supply one
tptype_list = &tpl
}
tptype_toks := strings.Split(*tptype_list, " ")
for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr
cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy
cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none)
cfreq.Match.Tpsport = src_tpport
cfreq.Match.Tpdport = dest_tpport
cfreq.Match.Vlan_id = vlan
ip2_str := ""
if cfreq.Match.Ip2 != nil {
ip2_str = *cfreq.Match.Ip2
}
rm_sheep.Baa(1, "res_mgr/push_bwow: flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d exp/fm_exp=%d/%d",
*rname, tptype_toks[tidx], *src, *dest, *cfreq.Match.Ip1, ip2_str, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport,
cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, expiry, cfreq.Expiry)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, nil, REQ_BWOW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup
}
}
p.Set_pushed() // safe to mark the pledge as having been pushed.
} else {
rm_sheep.Baa(1, "oneway not pushed: could not map one/both hosts to an IP address")
}
}
/*
For a single bandwidth pledge, this function sets things up and sends needed requests to the fq-manger to
create any necessary flow-mods. This has changed drastically now that we expect one agent
onvocation to set up all bandwidth flow-mods for an endpoint switch.
With the new method of managing queues per reservation on ingress/egress hosts, we now send
the following information to fq_mgr:
h1, h2 -- hosts
expiry
switch/port/queue
Path list will have path(s) in both directions (to support different bandwidth rates).
The above info is sent for each 'link' in the forward direction path, and then we reverse
the path and send requests to fq_mgr for each 'link' in the backwards direction. Errors are
returned to res_mgr via channel, but asycnh; we do not wait for responses to each message
generated here.
To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume)
use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If
to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit,
and we assume that this function is called periodically to update long running reservations.
Alt_table is the base alternate table set that we use for meta marking
If pref_ip6 is true, then if a host has both v4 and v6 addresses we will use the v6 address.
*/
func bw_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, alt_table int, pref_v6 bool) {
var (
msg *ipc.Chmsg
)
now := time.Now().Unix()
p, ok := (*gp).(*gizmos.Pledge_bw) // generic pledge better be a bw pledge!
if !ok {
rm_sheep.Baa(1, "internal error in push_bw_reservation: pledge isn't a bandwidth pledge")
(*gp).Set_pushed() // prevent looping
return
}
h1, h2, p1, p2, _, expiry, _, _ := p.Get_values() // hosts, transport (tcp/udp) ports and expiry are all we need
v1, v2 := p.Get_vlan() // vlan match criteria for one/both endpoints
ip1 := name2ip(h1)
ip2 := name2ip(h2)
if ip1 != nil && ip2 != nil { // good ip addresses so we're good to go
plist := p.Get_path_list() // each path that is a part of the reservation
timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice
for i := range plist { // for each path, send fmgr requests for each endpoint
freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things)
freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4?
freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems
freq.Single_switch = false // path involves multiple switches by default
freq.Dscp, freq.Dscp_koe = p.Get_dscp() // reservation supplied dscp value that we're to match and maybe preserve on exit
if (*p).Is_paused() {
freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out
} else {
if to_limit > 0 && expiry > now+to_limit {
freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size
} else {
freq.Expiry = expiry
}
}
freq.Id = rname
extip := plist[i].Get_extip() // if an external IP address is necessary on the freq get it
if extip != nil {
freq.Extip = extip
} else {
freq.Extip = &empty_str
}
espq1, _ := plist[i].Get_endpoint_spq(rname, timestamp) // end point switch, port, queue information; ep1 nil if single switch
if espq1 == nil { // if single switch ep1 will be nil
freq.Single_switch = true
}
freq.Match.Ip1 = plist[i].Get_h1().Get_address(pref_v6) // must use path h1/h2 as this could be the reverse with respect to the overall pledge and thus reverse of pledge
freq.Match.Ip2 = plist[i].Get_h2().Get_address(pref_v6)
freq.Espq = plist[i].Get_ilink_spq(rname, timestamp) // spq info comes from the first link off of the switch, not the endpoint link back to the VM
if freq.Single_switch {
freq.Espq.Queuenum = 1 // same switch always over br-rl queue 1
}
freq.Exttyp = plist[i].Get_extflag() // indicates whether the external IP is the source or dest along this path
tptype_list := p.Get_proto() // pick up protocol supplied on the reservation
if (*p1 != "0" || *p2 != "0") && *tptype_list == "" { // if either port is specified, and no specific proto on reservation
tpl := "udp tcp" // if port supplied, generate f-mods for both udp and tcp matches on the port
tptype_list = &tpl
}
tptype_toks := strings.Split(*tptype_list, " ")
for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr
cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy
cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none)
if *cfreq.Exttyp == "-S" { // indicates that this is a 'reverse' path (h2 sending) and we must invert the Tp port numbers and vland ids
cfreq.Match.Tpsport = p2
cfreq.Match.Tpdport = p1
cfreq.Match.Vlan_id = v2
} else {
cfreq.Match.Tpsport = p1
cfreq.Match.Tpdport = p2
cfreq.Match.Vlan_id = v1
}
rm_sheep.Baa(1, "res_mgr/push_rea: forward endpoint flow-mods for path %d: %s flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d ext=%s exp/fm_exp=%d/%d",
i, *rname, *cfreq.Exttyp, tptype_toks[tidx], *h1, *h2, *cfreq.Match.Ip1, *cfreq.Match.Ip2, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport,
cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, *cfreq.Extip, expiry, cfreq.Expiry)
msg = ipc.Mk_chmsg()
msg.Send_req(fq_ch, nil, REQ_BW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup
// WARNING: this is q-lite only -- there is no attempt to set up intermediate switches!
}
}
p.Set_pushed() // safe to mark the pledge as having been pushed.
}
}
/*
Opens the filename passed in and reads the reservation data from it. The assumption is
that records in the file were saved via the write_chkpt() function and are JSON pledges
or other serializable objects. We will drop any pledges that expired while 'sitting'
in the file.
*/
func (i *Inventory) load_chkpt(fname *string) (err error) {
var (
rec string
nrecs int = 0
p *gizmos.Pledge
my_ch chan *ipc.Chmsg
req *ipc.Chmsg
)
err = nil
my_ch = make(chan *ipc.Chmsg)
defer close(my_ch) // close it on return
f, err := os.Open(*fname)
if err != nil {
return
}
defer f.Close()
br := bufio.NewReader(f)
for err == nil {
rec, err = br.ReadString('\n')
if err == nil {
nrecs++
switch rec[0:5] {
case "ucap:":
toks := strings.Split(rec, " ")
if len(toks) == 3 {
i.add_ulcap(&toks[1], &toks[2])
}
default:
p, err = gizmos.Json2pledge(&rec) // convert any type of json pledge to Pledge
if err == nil {
if (*p).Is_expired() {
rm_sheep.Baa(1, "resmgr: ckpt_load: ignored expired pledge: %s", (*p).String())
} else {
switch sp := (*p).(type) { // work on specific pledge type, but pass the Pledge interface to add()
case *gizmos.Pledge_mirror:
err = i.Add_res(p) // assume we can just add it back in as is
case *gizmos.Pledge_steer:
rm_sheep.Baa(0, "did not restore steering reservation from checkpoint; not implemented")
case *gizmos.Pledge_bwow:
h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph
push_block := h2 == nil
update_graph(h1, push_block, push_block) // dig h1 info; push to netmgr if h2 isn't known and block on response
if h2 != nil {
update_graph(h2, true, true) // dig h2 data and push to netmgr blocking for a netmgr response
}
req = ipc.Mk_chmsg() // now safe to ask netmgr to validate the oneway pledge
req.Send_req(nw_ch, my_ch, REQ_BWOW_RESERVE, sp, nil)
req = <-my_ch // should be OK, but the underlying network could have changed
if req.Response_data != nil {
gate := req.Response_data.(*gizmos.Gate) // expect that network sent us a gate
sp.Set_gate(gate)
rm_sheep.Baa(1, "gate allocated for oneway reservation: %s %s %s %s", *(sp.Get_id()), *h1, *h2, *(gate.Get_extip()))
err = i.Add_res(p)
} else {
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for oneway pledge: %s [TGURMG000]", (*p).To_str())
}
case *gizmos.Pledge_bw:
h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph
update_graph(h1, false, false) // don't need to block on this one, nor update fqmgr
update_graph(h2, true, true) // wait for netmgr to update graph and then push related data to fqmgr
req = ipc.Mk_chmsg() // now safe to ask netmgr to find a path for the pledge
rm_sheep.Baa(2, "reserving path starts")
req.Send_req(nw_ch, my_ch, REQ_BW_RESERVE, sp, nil)
req = <-my_ch // should be OK, but the underlying network could have changed
if req.Response_data != nil {
rm_sheep.Baa(2, "reserving path finished")
path_list := req.Response_data.([]*gizmos.Path) // path(s) that were found to be suitable for the reservation
sp.Set_path_list(path_list)
rm_sheep.Baa(1, "path allocated for chkptd reservation: %s %s %s; path length= %d", *(sp.Get_id()), *h1, *h2, len(path_list))
err = i.Add_res(p)
} else {
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for pledge: %s [TGURMG000]", (*p).To_str())
}
case *gizmos.Pledge_pass:
host, _ := sp.Get_hosts()
update_graph(host, true, true)
req.Send_req(nw_ch, my_ch, REQ_GETPHOST, host, nil) // need to find the current phost for the vm
req = <-my_ch
if req.Response_data != nil {
phost := req.Response_data.(*string)
sp.Set_phost(phost)
rm_sheep.Baa(1, "passthrou phost found for chkptd reservation: %s %s %s", *(sp.Get_id()), *host, *phost)
err = i.Add_res(p)
} else {
s := fmt.Errorf("unknown reason")
if req.State != nil {
s = req.State
}
rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to find phost for passthru pledge: %s [TGURMG000]", s)
rm_sheep.Baa(0, "erroring passthru pledge: %s", (*p).To_str())
}
default:
rm_sheep.Baa(0, "rmgr/load_ckpt: unrecognised pledge type")
} // end switch on specific pledge type
}
} else {
rm_sheep.Baa(0, "CRI: %s", err)
return // quickk escape
}
} // outer switch
}
}
if err == io.EOF {
err = nil
}
rm_sheep.Baa(1, "read %d records from checkpoint file: %s", nrecs, *fname)
return
}