/* For a single passthrough pledge, this function sets things up and sends needed requests to the fq-manger to create any necessary flow-mods. We send the following information to fq_mgr: source mac or endpoint (VM-- the host in the pledge) source IP and optionally port and protocol more specific reservations expiry switch (physical host -- compute node) Errors are returned to res_mgr via channel, but asycnh; we do not wait for responses to each message generated here. To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume) use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit, and we assume that this function is called periodically to update long running reservations. */ func pass_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64) { var ( msg *ipc.Chmsg ) now := time.Now().Unix() p, ok := (*gp).(*gizmos.Pledge_pass) // generic pledge better be a passthrough pledge! if !ok { rm_sheep.Baa(1, "internal error in pass_push_reservation: pledge isn't a passthrough pledge") (*gp).Set_pushed() // prevent looping return } host, _, _, expiry, proto := p.Get_values() // reservation info that we need ip := name2ip(host) if ip != nil { // good ip addresses so we're good to go freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things) freq.Match.Smac = ip // fq_mgr has conversion map to convert to mac freq.Swid = p.Get_phost() // the phyiscal host where the VM lives and where fmods need to be deposited freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems if (*p).Is_paused() { freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out } else { if to_limit > 0 && expiry > now+to_limit { freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size } else { freq.Expiry = expiry } } freq.Id = rname freq.Extip = &empty_str // this will change when ported to endpoint branch as the endpoint allows address and port 'in line' freq.Match.Ip1 = proto // the proto on the reservation should be [{udp|tcp:}]address[:port] freq.Match.Ip2 = nil freq.Espq = nil dup_str := "" freq.Exttyp = &dup_str rm_sheep.Baa(1, "pushing passthru reservation: %s", p) msg = ipc.Mk_chmsg() msg.Send_req(fq_ch, ch, REQ_PT_RESERVE, freq, nil) // queue work with fq-manger to read the struct and send cmd(s) to agent to get it done p.Set_pushed() // safe to mark the pledge as having been pushed. } }
/* This builds a fq-mgr request and passes it to the fq-mgr to 'refine' and send along to the agent-manager for ultimate execution. It might be possible to pass it directly to the agent manager, but because res-mgr thinks in IP addresses, and fq-manager (that might be the only source of IP->Mac translation which flow-mods need) is still left in the middle. CAUTION: this function is called for both new reservations, _and_ to refresh reservations with expiry value further in the future than the switch can handle. It is also invoked when pausing reservations and will cause new flow-mods to be sent with a short duration timeout to flush existing flow-mods from the switch. */ func bwow_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, pref_v6 bool) { var ( msg *ipc.Chmsg ) now := time.Now().Unix() p, ok := (*gp).(*gizmos.Pledge_bwow) // generic pledge better be a bw oneway pledge! if !ok { rm_sheep.Baa(1, "internal mishap in push_bwow_res: pledge isn't a oneway pledge") (*gp).Set_pushed() // prevent looping return } src, dest, src_tpport, dest_tpport, _, expiry := p.Get_values() // hosts, transport ports, and expiry time vlan := p.Get_vlan() // vlan match criteria for source ip_src := name2ip(src) ip_dest := name2ip(dest) if ip_src != nil && ip_dest != nil { // good ip addresses so we're good to go gate := p.Get_gate() // get the gate information that is applied for the oneway if gate != nil { // be parinoid //timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice freq := Mk_fqreq(rname) // default flow mod request no match/actions freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4? freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems freq.Single_switch = true // implied with a oneway, but set it anyway freq.Dscp = p.Get_dscp() // reservation supplied dscp value that we're to match (koe is meaningless in one way) freq.Dscp_koe = false // meaningless for oneway, but ensure it's false so flag isn't accidently set later if (*p).Is_paused() { freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force existing flow-mods out } else { if to_limit > 0 && expiry > now+to_limit { freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size } else { freq.Expiry = expiry } } freq.Id = rname freq.Match.Ip1 = gate.Get_src().Get_address(pref_v6) // should match pledge, but gate is the ultimate authority freq.Match.Ip2 = gate.Get_dest().Get_address(pref_v6) freq.Espq = gate.Get_spq(rname, now+16) // switch port queue freq.Extip = gate.Get_extip() // returns nil if not an external and that's what we need tptype_list := p.Get_proto() // pick up protocol supplied on the reservation if (*src_tpport != "0" || *dest_tpport != "0") && *tptype_list == "" { // if port supplied we must set proto; default to both if tpl := "udp tcp" // user didn't supply one tptype_list = &tpl } tptype_toks := strings.Split(*tptype_list, " ") for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none) cfreq.Match.Tpsport = src_tpport cfreq.Match.Tpdport = dest_tpport cfreq.Match.Vlan_id = vlan ip2_str := "" if cfreq.Match.Ip2 != nil { ip2_str = *cfreq.Match.Ip2 } rm_sheep.Baa(1, "res_mgr/push_bwow: flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d exp/fm_exp=%d/%d", *rname, tptype_toks[tidx], *src, *dest, *cfreq.Match.Ip1, ip2_str, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport, cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, expiry, cfreq.Expiry) msg = ipc.Mk_chmsg() msg.Send_req(fq_ch, nil, REQ_BWOW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup } } p.Set_pushed() // safe to mark the pledge as having been pushed. } else { rm_sheep.Baa(1, "oneway not pushed: could not map one/both hosts to an IP address") } }
/* For a single bandwidth pledge, this function sets things up and sends needed requests to the fq-manger to create any necessary flow-mods. This has changed drastically now that we expect one agent onvocation to set up all bandwidth flow-mods for an endpoint switch. With the new method of managing queues per reservation on ingress/egress hosts, we now send the following information to fq_mgr: h1, h2 -- hosts expiry switch/port/queue Path list will have path(s) in both directions (to support different bandwidth rates). The above info is sent for each 'link' in the forward direction path, and then we reverse the path and send requests to fq_mgr for each 'link' in the backwards direction. Errors are returned to res_mgr via channel, but asycnh; we do not wait for responses to each message generated here. To_limit is a cap to the expiration time sent when creating a flow-mod. OVS (and others we assume) use an unsigned int32 as a hard timeout value, and thus have an upper limit of just over 18 hours. If to_limit is > 0, we'll ensure that the timeout passed on the request to fq-mgr won't exceed the limit, and we assume that this function is called periodically to update long running reservations. Alt_table is the base alternate table set that we use for meta marking If pref_ip6 is true, then if a host has both v4 and v6 addresses we will use the v6 address. */ func bw_push_res(gp *gizmos.Pledge, rname *string, ch chan *ipc.Chmsg, to_limit int64, alt_table int, pref_v6 bool) { var ( msg *ipc.Chmsg ) now := time.Now().Unix() p, ok := (*gp).(*gizmos.Pledge_bw) // generic pledge better be a bw pledge! if !ok { rm_sheep.Baa(1, "internal error in push_bw_reservation: pledge isn't a bandwidth pledge") (*gp).Set_pushed() // prevent looping return } h1, h2, p1, p2, _, expiry, _, _ := p.Get_values() // hosts, transport (tcp/udp) ports and expiry are all we need v1, v2 := p.Get_vlan() // vlan match criteria for one/both endpoints ip1 := name2ip(h1) ip2 := name2ip(h2) if ip1 != nil && ip2 != nil { // good ip addresses so we're good to go plist := p.Get_path_list() // each path that is a part of the reservation timestamp := time.Now().Unix() + 16 // assume this will fall within the first few seconds of the reservation as we use it to find queue in timeslice for i := range plist { // for each path, send fmgr requests for each endpoint freq := Mk_fqreq(rname) // default flow mod request with empty match/actions (for bw requests, we don't need priority or such things) freq.Ipv6 = p.Get_matchv6() // should we force a match on IPv6 rather than IPv4? freq.Cookie = 0xffff // should be ignored, if we see this out there we've got problems freq.Single_switch = false // path involves multiple switches by default freq.Dscp, freq.Dscp_koe = p.Get_dscp() // reservation supplied dscp value that we're to match and maybe preserve on exit if (*p).Is_paused() { freq.Expiry = time.Now().Unix() + 15 // if reservation shows paused, then we set the expiration to 15s from now which should force the flow-mods out } else { if to_limit > 0 && expiry > now+to_limit { freq.Expiry = now + to_limit // expiry must be capped so as not to overflow virtual switch variable size } else { freq.Expiry = expiry } } freq.Id = rname extip := plist[i].Get_extip() // if an external IP address is necessary on the freq get it if extip != nil { freq.Extip = extip } else { freq.Extip = &empty_str } espq1, _ := plist[i].Get_endpoint_spq(rname, timestamp) // end point switch, port, queue information; ep1 nil if single switch if espq1 == nil { // if single switch ep1 will be nil freq.Single_switch = true } freq.Match.Ip1 = plist[i].Get_h1().Get_address(pref_v6) // must use path h1/h2 as this could be the reverse with respect to the overall pledge and thus reverse of pledge freq.Match.Ip2 = plist[i].Get_h2().Get_address(pref_v6) freq.Espq = plist[i].Get_ilink_spq(rname, timestamp) // spq info comes from the first link off of the switch, not the endpoint link back to the VM if freq.Single_switch { freq.Espq.Queuenum = 1 // same switch always over br-rl queue 1 } freq.Exttyp = plist[i].Get_extflag() // indicates whether the external IP is the source or dest along this path tptype_list := p.Get_proto() // pick up protocol supplied on the reservation if (*p1 != "0" || *p2 != "0") && *tptype_list == "" { // if either port is specified, and no specific proto on reservation tpl := "udp tcp" // if port supplied, generate f-mods for both udp and tcp matches on the port tptype_list = &tpl } tptype_toks := strings.Split(*tptype_list, " ") for tidx := range tptype_toks { // must have a req for each transport proto type, clone base, add the proto specific changes, & send to fqmgr cfreq := freq.Clone() // since we send this off for asynch processing we must make a copy cfreq.Tptype = &tptype_toks[tidx] // transport type (tcp, udp or none) if *cfreq.Exttyp == "-S" { // indicates that this is a 'reverse' path (h2 sending) and we must invert the Tp port numbers and vland ids cfreq.Match.Tpsport = p2 cfreq.Match.Tpdport = p1 cfreq.Match.Vlan_id = v2 } else { cfreq.Match.Tpsport = p1 cfreq.Match.Tpdport = p2 cfreq.Match.Vlan_id = v1 } rm_sheep.Baa(1, "res_mgr/push_rea: forward endpoint flow-mods for path %d: %s flag=%s tptyp=%s VMs=%s,%s dir=%s->%s tpsport=%s tpdport=%s spq=%s/%d/%d ext=%s exp/fm_exp=%d/%d", i, *rname, *cfreq.Exttyp, tptype_toks[tidx], *h1, *h2, *cfreq.Match.Ip1, *cfreq.Match.Ip2, *cfreq.Match.Tpsport, *cfreq.Match.Tpdport, cfreq.Espq.Switch, cfreq.Espq.Port, cfreq.Espq.Queuenum, *cfreq.Extip, expiry, cfreq.Expiry) msg = ipc.Mk_chmsg() msg.Send_req(fq_ch, nil, REQ_BW_RESERVE, cfreq, nil) // queue work with fq-manger to send cmds for bandwidth f-mod setup // WARNING: this is q-lite only -- there is no attempt to set up intermediate switches! } } p.Set_pushed() // safe to mark the pledge as having been pushed. } }
/* Opens the filename passed in and reads the reservation data from it. The assumption is that records in the file were saved via the write_chkpt() function and are JSON pledges or other serializable objects. We will drop any pledges that expired while 'sitting' in the file. */ func (i *Inventory) load_chkpt(fname *string) (err error) { var ( rec string nrecs int = 0 p *gizmos.Pledge my_ch chan *ipc.Chmsg req *ipc.Chmsg ) err = nil my_ch = make(chan *ipc.Chmsg) defer close(my_ch) // close it on return f, err := os.Open(*fname) if err != nil { return } defer f.Close() br := bufio.NewReader(f) for err == nil { rec, err = br.ReadString('\n') if err == nil { nrecs++ switch rec[0:5] { case "ucap:": toks := strings.Split(rec, " ") if len(toks) == 3 { i.add_ulcap(&toks[1], &toks[2]) } default: p, err = gizmos.Json2pledge(&rec) // convert any type of json pledge to Pledge if err == nil { if (*p).Is_expired() { rm_sheep.Baa(1, "resmgr: ckpt_load: ignored expired pledge: %s", (*p).String()) } else { switch sp := (*p).(type) { // work on specific pledge type, but pass the Pledge interface to add() case *gizmos.Pledge_mirror: err = i.Add_res(p) // assume we can just add it back in as is case *gizmos.Pledge_steer: rm_sheep.Baa(0, "did not restore steering reservation from checkpoint; not implemented") case *gizmos.Pledge_bwow: h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph push_block := h2 == nil update_graph(h1, push_block, push_block) // dig h1 info; push to netmgr if h2 isn't known and block on response if h2 != nil { update_graph(h2, true, true) // dig h2 data and push to netmgr blocking for a netmgr response } req = ipc.Mk_chmsg() // now safe to ask netmgr to validate the oneway pledge req.Send_req(nw_ch, my_ch, REQ_BWOW_RESERVE, sp, nil) req = <-my_ch // should be OK, but the underlying network could have changed if req.Response_data != nil { gate := req.Response_data.(*gizmos.Gate) // expect that network sent us a gate sp.Set_gate(gate) rm_sheep.Baa(1, "gate allocated for oneway reservation: %s %s %s %s", *(sp.Get_id()), *h1, *h2, *(gate.Get_extip())) err = i.Add_res(p) } else { rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for oneway pledge: %s [TGURMG000]", (*p).To_str()) } case *gizmos.Pledge_bw: h1, h2 := sp.Get_hosts() // get the host names, fetch ostack data and update graph update_graph(h1, false, false) // don't need to block on this one, nor update fqmgr update_graph(h2, true, true) // wait for netmgr to update graph and then push related data to fqmgr req = ipc.Mk_chmsg() // now safe to ask netmgr to find a path for the pledge rm_sheep.Baa(2, "reserving path starts") req.Send_req(nw_ch, my_ch, REQ_BW_RESERVE, sp, nil) req = <-my_ch // should be OK, but the underlying network could have changed if req.Response_data != nil { rm_sheep.Baa(2, "reserving path finished") path_list := req.Response_data.([]*gizmos.Path) // path(s) that were found to be suitable for the reservation sp.Set_path_list(path_list) rm_sheep.Baa(1, "path allocated for chkptd reservation: %s %s %s; path length= %d", *(sp.Get_id()), *h1, *h2, len(path_list)) err = i.Add_res(p) } else { rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to reserve for pledge: %s [TGURMG000]", (*p).To_str()) } case *gizmos.Pledge_pass: host, _ := sp.Get_hosts() update_graph(host, true, true) req.Send_req(nw_ch, my_ch, REQ_GETPHOST, host, nil) // need to find the current phost for the vm req = <-my_ch if req.Response_data != nil { phost := req.Response_data.(*string) sp.Set_phost(phost) rm_sheep.Baa(1, "passthrou phost found for chkptd reservation: %s %s %s", *(sp.Get_id()), *host, *phost) err = i.Add_res(p) } else { s := fmt.Errorf("unknown reason") if req.State != nil { s = req.State } rm_sheep.Baa(0, "ERR: resmgr: ckpt_laod: unable to find phost for passthru pledge: %s [TGURMG000]", s) rm_sheep.Baa(0, "erroring passthru pledge: %s", (*p).To_str()) } default: rm_sheep.Baa(0, "rmgr/load_ckpt: unrecognised pledge type") } // end switch on specific pledge type } } else { rm_sheep.Baa(0, "CRI: %s", err) return // quickk escape } } // outer switch } } if err == io.EOF { err = nil } rm_sheep.Baa(1, "read %d records from checkpoint file: %s", nrecs, *fname) return }