/* routerInfo[0] -> Uplink nexthop interface
*/
func NewOfnetAgent(bridgeName string, dpName string, localIp net.IP, rpcPort uint16,
ovsPort uint16, routerInfo ...string) (*OfnetAgent, error) {
agent := new(OfnetAgent)
// Init params
agent.localIp = localIp
agent.MyPort = rpcPort
agent.MyAddr = localIp.String()
agent.dpName = dpName
agent.masterDb = make(map[string]*OfnetNode)
agent.portVlanMap = make(map[uint32]*uint16)
agent.vniVlanMap = make(map[uint32]*uint16)
agent.vlanVniMap = make(map[uint16]*uint32)
// Initialize vtep database
agent.vtepTable = make(map[string]*uint32)
// Initialize endpoint database
agent.endpointDb = make(map[string]*OfnetEndpoint)
agent.localEndpointDb = make(map[uint32]*OfnetEndpoint)
// Create an openflow controller
agent.ctrler = ofctrl.NewController(agent)
// Start listening to controller port
go agent.ctrler.Listen(fmt.Sprintf(":%d", ovsPort))
// FIXME: Figure out how to handle multiple OVS bridges.
rpcServ, listener := rpcHub.NewRpcServer(rpcPort)
agent.rpcServ = rpcServ
agent.rpcListener = listener
// Register for Master add/remove events
rpcServ.Register(agent)
// Create the datapath
switch dpName {
case "vrouter":
agent.datapath = NewVrouter(agent, rpcServ)
case "vxlan":
agent.datapath = NewVxlan(agent, rpcServ)
case "vlan":
agent.datapath = NewVlanBridge(agent, rpcServ)
case "vlrouter":
agent.datapath = NewVlrouter(agent, rpcServ)
agent.ovsDriver = ovsdbDriver.NewOvsDriver(bridgeName)
agent.protopath = NewOfnetBgp(agent, routerInfo)
default:
log.Fatalf("Unknown Datapath %s", dpName)
}
// Return it
return agent, nil
}
// Test if OVS switch connects successfully
func TestMain(m *testing.M) {
// Create a controller
ctrler = NewController(&ofActor)
// start listening
go ctrler.Listen(":6733")
// Connect to ovsdb and add the controller
ovsDriver = ovsdbDriver.NewOvsDriver("ovsbr11")
err := ovsDriver.AddController("127.0.0.1", 6733)
if err != nil {
log.Fatalf("Error adding controller to ovs")
}
//wait for 10sec and see if switch connects
time.Sleep(10 * time.Second)
if !ofActor.isSwitchConnected {
log.Fatalf("ovsbr0 switch did not connect within 20sec")
return
}
log.Infof("Switch connected. Creating tables..")
// Create initial tables
ofActor.inputTable = ofActor.Switch.DefaultTable()
if ofActor.inputTable == nil {
log.Fatalf("Failed to get input table")
return
}
ofActor.nextTable, err = ofActor.Switch.NewTable(1)
if err != nil {
log.Fatalf("Error creating next table. Err: %v", err)
return
}
log.Infof("Openflow tables created successfully")
// run the test
exitCode := m.Run()
// delete the bridge
err = ovsDriver.DeleteBridge("ovsbr11")
if err != nil {
log.Fatalf("Error deleting the bridge. Err: %v", err)
}
os.Exit(exitCode)
}
// NewOvsSwitch Creates a new OVS switch instance
func NewOvsSwitch(bridgeName, netType, localIP string) (*OvsSwitch, error) {
var err error
sw := new(OvsSwitch)
sw.bridgeName = bridgeName
sw.netType = netType
// Create OVS db driver
sw.ovsdbDriver = ovsdbDriver.NewOvsDriver(bridgeName)
// Create an ofnet agent
sw.ofnetAgent, err = ofnet.NewOfnetAgent(netType, net.ParseIP(localIP), ofnet.OFNET_AGENT_PORT, OVS_CTRLER_PORT)
if err != nil {
log.Fatalf("Error initializing ofnet")
return nil, err
}
// Add controller to the OVS
if !sw.ovsdbDriver.IsControllerPresent(OVS_CTRLER_IP, OVS_CTRLER_PORT) {
err = sw.ovsdbDriver.AddController(OVS_CTRLER_IP, OVS_CTRLER_PORT)
if err != nil {
log.Fatalf("Error adding controller to OVS. Err: %v", err)
return nil, err
}
}
// Wait a little for master to be ready before we connect
time.Sleep(300 * time.Millisecond)
log.Infof("Waiting for OVS switch to connect..")
// Wait for a while for OVS switch to connect to ofnet agent
for i := 0; i < 10; i++ {
time.Sleep(1 * time.Second)
if sw.ofnetAgent.IsSwitchConnected() {
break
}
}
log.Infof("Switch connected.")
return sw, nil
}
// Create couple of ofnet masters and few agents
func TestMain(m *testing.M) {
var err error
for i := 0; i < NUM_AGENT; i++ {
localIpList = append(localIpList, fmt.Sprintf("10.10.10.%d", (i+1)))
}
// Create the masters
for i := 0; i < NUM_MASTER; i++ {
vrtrMasters[i] = NewOfnetMaster("", uint16(VRTR_MASTER_PORT+i))
if vrtrMasters[i] == nil {
log.Fatalf("Error creating ofnet master for vrouter: %d", i)
}
log.Infof("Created vrouter Master: %v", vrtrMasters[i])
vxlanMasters[i] = NewOfnetMaster("", uint16(VXLAN_MASTER_PORT+i))
if vxlanMasters[i] == nil {
log.Fatalf("Error creating ofnet master for vxlan: %d", i)
}
log.Infof("Created vxlan Master: %v", vxlanMasters[i])
vlanMasters[i] = NewOfnetMaster("", uint16(VLAN_MASTER_PORT+i))
if vlanMasters[i] == nil {
log.Fatalf("Error creating ofnet master for vlan: %d", i)
}
log.Infof("Created vlan Master: %v", vlanMasters[i])
}
for i := 0; i < NUM_VLRTR_MASTER; i++ {
vlrtrMaster[i] = NewOfnetMaster("", uint16(VLRTR_MASTER_PORT))
if vlrtrMaster[i] == nil {
log.Fatalf("Error creating ofnet master for vlrtr: %d", i)
}
log.Infof("Created vlrtr Master: %v", vlrtrMaster[i])
}
// Wait a second for masters to be up
time.Sleep(1 * time.Second)
// Create agents
for i := 0; i < NUM_AGENT; i++ {
brName := "vrtrBridge" + fmt.Sprintf("%d", i)
rpcPort := uint16(VRTR_RPC_PORT + i)
ovsPort := uint16(VRTR_OVS_PORT + i)
lclIp := net.ParseIP(localIpList[i])
vrtrAgents[i], err = NewOfnetAgent(brName, "vrouter", lclIp, rpcPort, ovsPort)
if err != nil {
log.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vrtrAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vrouter ofnet agent: %v", vrtrAgents[i])
}
for i := 0; i < NUM_AGENT; i++ {
brName := "vxlanBridge" + fmt.Sprintf("%d", i)
rpcPort := uint16(VXLAN_RPC_PORT + i)
ovsPort := uint16(VXLAN_OVS_PORT + i)
lclIp := net.ParseIP(localIpList[i])
vxlanAgents[i], err = NewOfnetAgent(brName, "vxlan", lclIp, rpcPort, ovsPort)
if err != nil {
log.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vxlanAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vxlan ofnet agent: %v", vxlanAgents[i])
}
for i := 0; i < NUM_AGENT; i++ {
brName := "vlanBridge" + fmt.Sprintf("%d", i)
rpcPort := uint16(VLAN_RPC_PORT + i)
ovsPort := uint16(VLAN_OVS_PORT + i)
lclIp := net.ParseIP(localIpList[i])
vlanAgents[i], err = NewOfnetAgent(brName, "vlan", lclIp, rpcPort, ovsPort)
if err != nil {
log.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vlanAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vlan ofnet agent: %v", vlanAgents[i])
}
for i := 0; i < NUM_VLRTR_AGENT; i++ {
brName := "vlrtrBridge" + fmt.Sprintf("%d", i)
rpcPort := uint16(VLRTR_RPC_PORT + i)
ovsPort := uint16(VLRTR_OVS_PORT + i)
lclIp := net.ParseIP(localIpList[i])
portName := "inb0" + fmt.Sprintf("%d", i)
driver := ovsdbDriver.NewOvsDriver(brName)
driver.CreatePort(portName, "internal", uint(1+i))
vlrtrAgents[i], err = NewOfnetAgent(brName, "vlrouter", lclIp, rpcPort, ovsPort, portName)
if err != nil {
log.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vlrtrAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vlrtr ofnet agent: %v", vlrtrAgents[i])
}
for i := 0; i < NUM_HOST_BRIDGE; i++ {
brName := "hostBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(HB_OVS_PORT + i)
driver := ovsdbDriver.NewOvsDriver(brName)
portName := "inb0" + fmt.Sprintf("%d", i)
driver.CreatePort(portName, "internal", uint(1+i))
hostBridges[i], err = NewHostBridge(brName, "hostbridge", ovsPort)
if err != nil {
log.Fatalf("Error creating ofnet agent. Err: %v", err)
}
log.Infof("Created hostBridge agent: %v", hostBridges[i])
}
masterInfo := OfnetNode{
HostAddr: "127.0.0.1",
}
var resp bool
// Add master node to each agent
for i := 0; i < NUM_AGENT; i++ {
// add the two master nodes
for j := 0; j < NUM_MASTER; j++ {
masterInfo.HostPort = uint16(VRTR_MASTER_PORT + j)
// connect vrtr agent to vrtr master
err := vrtrAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
log.Fatalf("Error adding master %+v to vrtr node %d. Err: %v", masterInfo, i, err)
}
// connect vxlan agents to vxlan master
masterInfo.HostPort = uint16(VXLAN_MASTER_PORT + j)
err = vxlanAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
log.Fatalf("Error adding master %+v to vxlan node %d. Err: %v", masterInfo, i, err)
}
// connect vlan agents to vlan master
masterInfo.HostPort = uint16(VLAN_MASTER_PORT + j)
err = vlanAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
log.Fatalf("Error adding master %+v to vlan node %d. Err: %v", masterInfo, i, err)
}
}
}
for i := 0; i < NUM_VLRTR_AGENT; i++ {
for j := 0; j < NUM_VLRTR_MASTER; j++ {
// connect vlrtr agents to vlrtr master
masterInfo.HostPort = uint16(VLRTR_MASTER_PORT + j)
err = vlrtrAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
log.Fatalf("Error adding master %+v to vlrtr node %d. Err: %v", masterInfo, i, err)
}
}
}
log.Infof("Ofnet masters and agents are setup..")
time.Sleep(1 * time.Second)
for i := 0; i < NUM_MASTER; i++ {
err := vrtrMasters[i].MakeDummyRpcCall()
if err != nil {
log.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
err = vxlanMasters[i].MakeDummyRpcCall()
if err != nil {
log.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
err = vlanMasters[i].MakeDummyRpcCall()
if err != nil {
log.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
}
for i := 0; i < NUM_VLRTR_MASTER; i++ {
err = vlrtrMaster[i].MakeDummyRpcCall()
if err != nil {
log.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
}
log.Infof("Made dummy rpc call to all agents")
// Create OVS switches and connect them to vrouter ofnet agents
for i := 0; i < NUM_AGENT; i++ {
brName := "vrtrBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(VRTR_OVS_PORT + i)
ovsDrivers[i] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[i].AddController("127.0.0.1", ovsPort)
if err != nil {
log.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Create OVS switches and connect them to vxlan ofnet agents
for i := 0; i < NUM_AGENT; i++ {
brName := "vxlanBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(VXLAN_OVS_PORT + i)
j := NUM_AGENT + i
ovsDrivers[j] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[j].AddController("127.0.0.1", ovsPort)
if err != nil {
log.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Create OVS switches and connect them to vlan ofnet agents
for i := 0; i < NUM_AGENT; i++ {
brName := "vlanBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(VLAN_OVS_PORT + i)
j := (2 * NUM_AGENT) + i
ovsDrivers[j] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[j].AddController("127.0.0.1", ovsPort)
if err != nil {
log.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Create OVS switches and connect them to vxlan ofnet agents
for i := 0; i < NUM_VLRTR_AGENT; i++ {
brName := "vlrtrBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(VLRTR_OVS_PORT + i)
j := (3 * NUM_AGENT) + i
ovsDrivers[j] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[j].AddController("127.0.0.1", ovsPort)
if err != nil {
log.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Create OVS switches and connect them to hostbridge agents
for i := 0; i < NUM_HOST_BRIDGE; i++ {
brName := "hostBridge" + fmt.Sprintf("%d", i)
ovsPort := uint16(HB_OVS_PORT + i)
j := HB_AGENT_INDEX + i
ovsDrivers[j] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[j].AddController("127.0.0.1", ovsPort)
if err != nil {
log.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Wait for 20sec for switch to connect to controller
time.Sleep(10 * time.Second)
err = setupVlans()
if err != nil {
log.Fatalf("Error setting up Vlans")
}
err = setupVteps()
if err != nil {
log.Fatalf("Error setting up vteps")
}
// run the test
exitCode := m.Run()
// cleanup
waitAndCleanup()
// done
os.Exit(exitCode)
}
func TestSvcProxyInterface(t *testing.T) {
var resp bool
rpcPort := uint16(9600)
ovsPort := uint16(9601)
lclIP := net.ParseIP("10.10.10.10")
ofnetAgent, err := NewOfnetAgent("", "vrouter", lclIP, rpcPort, ovsPort)
if err != nil {
t.Fatalf("Error creating ofnet agent. Err: %v", err)
}
defer func() { ofnetAgent.Delete() }()
// Override MyAddr to local host
ofnetAgent.MyAddr = "127.0.0.1"
// Create a Master
ofnetMaster := NewOfnetMaster("", uint16(9602))
defer func() { ofnetMaster.Delete() }()
masterInfo := OfnetNode{
HostAddr: "127.0.0.1",
HostPort: uint16(9602),
}
// connect vrtr agent to master
err = ofnetAgent.AddMaster(&masterInfo, &resp)
if err != nil {
t.Errorf("Error adding master %+v. Err: %v", masterInfo, err)
}
log.Infof("Created vrouter ofnet agent: %v", ofnetAgent)
brName := "ovsbr60"
ovsDriver := ovsdbDriver.NewOvsDriver(brName)
err = ovsDriver.AddController("127.0.0.1", ovsPort)
if err != nil {
t.Fatalf("Error adding controller to ovs: %s", brName)
}
// Wait for switch to connect to controller
ofnetAgent.WaitForSwitchConnection()
// Create a vlan for the endpoint
ofnetAgent.AddNetwork(1, 1, "", "default")
// add two endpoints
addEP(ofnetAgent, testEP1port, testEP1ip, t)
addEP(ofnetAgent, testEP2port, testEP2ip, t)
// Create a service spec
svcPorts := make([]PortSpec, 3)
svcPorts[0] = PortSpec{
Protocol: "TCP",
SvcPort: 5600,
ProvPort: 9600,
}
svcPorts[1] = PortSpec{
Protocol: "UDP",
SvcPort: 5601,
ProvPort: 9600,
}
svcPorts[2] = PortSpec{
Protocol: "TCP",
SvcPort: 5602,
ProvPort: 9602,
}
svc := ServiceSpec{
IpAddress: "10.254.0.10",
Ports: svcPorts,
}
log.Infof("Adding LipService: %+v", svc)
ofnetAgent.AddSvcSpec("LipService", &svc)
// Add providers
spMacs := make([]string, 0, 2)
sps := make([]string, 1, 2)
sps[0] = "20.1.1.22"
ofnetAgent.SvcProviderUpdate("LipService", sps)
// At this point watch flows should be set up.
wFlow1 := fmt.Sprintf("priority=10,tcp,nw_dst=10.254.0.10 actions=CONTROLLER")
wFlow2 := fmt.Sprintf("priority=10,udp,nw_dst=10.254.0.10 actions=CONTROLLER")
checkFlows(t, brName, "Verify tcp watch flows", wFlow1, SRV_PROXY_DNAT_TBL_ID)
checkFlows(t, brName, "Verify udp watch flows", wFlow2, SRV_PROXY_DNAT_TBL_ID)
// inject a pkt from the EP to the service
pkt1 := getIPPkt(testEP1ip, "10.254.0.10", testEP1port)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt1)
pkt2 := getIPPkt(testEP2ip, "10.254.0.10", testEP2port)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt2)
// At this point, we should have NAT flows set up.
clientIPs := make([]string, 2, 2)
clientIPs[0] = testEP1ip
clientIPs[1] = testEP2ip
// verify DNAT flows
for _, cIP := range clientIPs {
dNFlow1 := fmt.Sprintf("priority=100,tcp,nw_src=%s,nw_dst=10.254.0.10,tp_dst=5600 actions=set_field:9600->tcp_dst,set_field:20.1.1.22->ip_dst,goto_table:3", cIP)
checkFlows(t, brName, "TCP dNAT flow for "+cIP, dNFlow1, SRV_PROXY_DNAT_TBL_ID)
dNFlow2 := fmt.Sprintf("priority=100,tcp,nw_src=%s,nw_dst=10.254.0.10,tp_dst=5602 actions=set_field:9602->tcp_dst,set_field:20.1.1.22->ip_dst,goto_table:3", cIP)
checkFlows(t, brName, "TCP dNAT flow for "+cIP, dNFlow2, SRV_PROXY_DNAT_TBL_ID)
dNFlow3 := fmt.Sprintf("priority=100,udp,nw_src=%s,nw_dst=10.254.0.10,tp_dst=5601 actions=set_field:9600->udp_dst,set_field:20.1.1.22->ip_dst,goto_table:3", cIP)
checkFlows(t, brName, "UDP dNAT flow for "+cIP, dNFlow3, SRV_PROXY_DNAT_TBL_ID)
}
// verify SNAT flows
for _, cIP := range clientIPs {
sNFlow1 := fmt.Sprintf("priority=100,tcp,nw_src=20.1.1.22,nw_dst=%s,tp_src=9600 actions=set_field:5600->tcp_src,set_field:10.254.0.10->ip_src", cIP)
checkFlows(t, brName, "TCP sNAT flow for "+cIP, sNFlow1, SRV_PROXY_SNAT_TBL_ID)
sNFlow2 := fmt.Sprintf("priority=100,tcp,nw_src=20.1.1.22,nw_dst=%s,tp_src=9602 actions=set_field:5602->tcp_src,set_field:10.254.0.10->ip_src", cIP)
checkFlows(t, brName, "TCP sNAT flow for "+cIP, sNFlow2, SRV_PROXY_SNAT_TBL_ID)
sNFlow3 := fmt.Sprintf("priority=100,udp,nw_src=20.1.1.22,nw_dst=%s,tp_src=9600 actions=set_field:5601->udp_src,set_field:10.254.0.10->ip_src", cIP)
checkFlows(t, brName, "UDP sNAT flow for "+cIP, sNFlow3, SRV_PROXY_SNAT_TBL_ID)
}
// Issue a provider update with two different providers
sps[0] = "20.1.1.23"
sps = append(sps, "20.1.1.24")
ofnetAgent.SvcProviderUpdate("LipService", sps)
// now, the NAT flows should have been removed.
verifyNATRemoved(t, brName, "20.1.1.22")
verifyWatchPresent(t, brName)
// inject packets again
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt2)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt1)
// flows should be setup in load balanced manner.
verifyLB(t, brName, sps, spMacs)
// Remove one endpoint
err = ofnetAgent.RemoveLocalEndpoint(testEP1port)
if err != nil {
t.Errorf("Error deleting endpoint: Err: %v", err)
return
}
// flows of that ep should be gone
verifyEPDel(t, brName, testEP1ip)
// add another ep and inject a pkt.
addEP(ofnetAgent, testEP4port, testEP4ip, t)
pkt3 := getIPPkt(testEP4ip, "10.254.0.10", testEP4port)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt3)
// flows should be setup in load balanced manner.
verifyLB(t, brName, sps, spMacs)
newSvcPorts := make([]PortSpec, 3)
newSvcPorts[0] = PortSpec{
Protocol: "TCP",
SvcPort: 5600,
ProvPort: 9600,
}
newSvcPorts[1] = PortSpec{
Protocol: "UDP",
SvcPort: 5601,
ProvPort: 9600,
}
newSvcPorts[2] = PortSpec{
Protocol: "TCP",
SvcPort: 7777,
ProvPort: 9602,
}
updatedSvc := ServiceSpec{
IpAddress: "10.254.0.10",
Ports: newSvcPorts,
}
ofnetAgent.AddSvcSpec("LipService", &updatedSvc)
// Verify NAT flows were removed
verifyNATRemoved(t, brName, "20.1.1.23")
verifyNATRemoved(t, brName, "20.1.1.24")
verifyWatchPresent(t, brName)
// Add two endpoints as providers in the same subnet.
addRemoteEP(ofnetAgent, 23, 1, "10.2.2.23", t)
addRemoteEP(ofnetAgent, 24, 1, "10.2.2.24", t)
// Issue a provider update with the providers in the same subnet
// and verify mac rewrite
sps[0] = "10.2.2.23"
sps[1] = "10.2.2.24"
ofnetAgent.SvcProviderUpdate("LipService", sps)
// verify watch flows
verifyWatchPresent(t, brName)
// inject packets again
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt3)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt2)
spMacs = append(spMacs, "00:01:02:03:04:23")
spMacs = append(spMacs, "00:01:02:03:04:24")
verifyLB(t, brName, sps, spMacs)
// verify no mac-rewrite when the prov is in a different subnet
ofnetAgent.AddNetwork(2, 2, "", "default")
addRemoteEP(ofnetAgent, 25, 2, "10.2.3.25", t)
addRemoteEP(ofnetAgent, 26, 2, "10.2.3.26", t)
sps[0] = "10.2.3.25"
sps[1] = "10.2.3.26"
ofnetAgent.SvcProviderUpdate("LipService", sps)
// previous NAT flows should be removed
verifyNATRemoved(t, brName, "10.2.2.23")
verifyNATRemoved(t, brName, "10.2.2.24")
verifyWatchPresent(t, brName)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt3)
ofnetAgent.PacketRcvd(ofnetAgent.ofSwitch, pkt2)
// new NAT flows should be present
verifyLB(t, brName, sps, []string{})
spMacs[0] = "00:01:02:03:04:25"
spMacs[1] = "00:01:02:03:04:26"
verifyNoMacRewrite(t, brName, spMacs)
// Delete the service
ofnetAgent.DelSvcSpec("LipService", &updatedSvc)
// Make sure flows are gone
verifyWatchRemoved(t, brName)
// Verify NAT flows were removed
verifyNATRemoved(t, brName, "10.2.3.25")
verifyNATRemoved(t, brName, "10.2.3.26")
}
func TestPolicyAddDelete(t *testing.T) {
var resp bool
rpcPort := uint16(9600)
ovsPort := uint16(9601)
lclIP := net.ParseIP("10.10.10.10")
ofnetAgent, err := NewOfnetAgent("vrouter", lclIP, rpcPort, ovsPort)
if err != nil {
t.Fatalf("Error creating ofnet agent. Err: %v", err)
}
defer func() { ofnetAgent.Delete() }()
// Override MyAddr to local host
ofnetAgent.MyAddr = "127.0.0.1"
// Create a Master
ofnetMaster := NewOfnetMaster(uint16(9602))
defer func() { ofnetMaster.Delete() }()
masterInfo := OfnetNode{
HostAddr: "127.0.0.1",
HostPort: uint16(9602),
}
// connect vrtr agent to master
err = ofnetAgent.AddMaster(&masterInfo, &resp)
if err != nil {
t.Errorf("Error adding master %+v. Err: %v", masterInfo, err)
}
log.Infof("Created vrouter ofnet agent: %v", ofnetAgent)
brName := "ovsbr60"
ovsDriver := ovsdbDriver.NewOvsDriver(brName)
err = ovsDriver.AddController("127.0.0.1", ovsPort)
if err != nil {
t.Fatalf("Error adding controller to ovs: %s", brName)
}
// Wait for switch to connect to controller
ofnetAgent.WaitForSwitchConnection()
// Create a vlan for the endpoint
ofnetAgent.AddVlan(1, 1)
macAddr, _ := net.ParseMAC("00:01:02:03:04:05")
endpoint := EndpointInfo{
EndpointGroup: 100,
PortNo: 12,
MacAddr: macAddr,
Vlan: 1,
IpAddr: net.ParseIP("10.2.2.2"),
}
log.Infof("Adding Local endpoint: %+v", endpoint)
// Add an Endpoint
err = ofnetAgent.AddLocalEndpoint(endpoint)
if err != nil {
t.Errorf("Error adding endpoint. Err: %v", err)
return
}
tcpRule := &OfnetPolicyRule{
RuleId: "tcpRule",
Priority: 100,
SrcEndpointGroup: 100,
DstEndpointGroup: 200,
SrcIpAddr: "10.10.10.10/24",
DstIpAddr: "10.1.1.1/24",
IpProtocol: 6,
DstPort: 100,
SrcPort: 200,
Action: "accept",
}
log.Infof("Adding rule: %+v", tcpRule)
// Add a policy
err = ofnetMaster.AddRule(tcpRule)
if err != nil {
t.Errorf("Error installing tcpRule {%+v}. Err: %v", tcpRule, err)
return
}
udpRule := &OfnetPolicyRule{
RuleId: "udpRule",
Priority: 100,
SrcEndpointGroup: 300,
DstEndpointGroup: 400,
SrcIpAddr: "20.20.20.20/24",
DstIpAddr: "20.2.2.2/24",
IpProtocol: 17,
DstPort: 300,
SrcPort: 400,
Action: "deny",
}
log.Infof("Adding rule: %+v", udpRule)
// Add the policy
err = ofnetMaster.AddRule(udpRule)
if err != nil {
t.Errorf("Error installing udpRule {%+v}. Err: %v", udpRule, err)
return
}
// Get all the flows
flowList, err := ofctlFlowDump(brName)
if err != nil {
t.Errorf("Error getting flow entries. Err: %v", err)
return
}
// verify src group flow
srcGrpFlowMatch := fmt.Sprintf("priority=100,in_port=12 actions=write_metadata:0x640000/0x7fff0000")
if !ofctlFlowMatch(flowList, VLAN_TBL_ID, srcGrpFlowMatch) {
t.Errorf("Could not find the flow %s on ovs %s", srcGrpFlowMatch, brName)
return
}
log.Infof("Found src group %s on ovs %s", srcGrpFlowMatch, brName)
// verify dst group flow
dstGrpFlowMatch := fmt.Sprintf("priority=100,ip,nw_dst=10.2.2.2 actions=write_metadata:0xc8/0xfffe")
if !ofctlFlowMatch(flowList, DST_GRP_TBL_ID, dstGrpFlowMatch) {
t.Errorf("Could not find the flow %s on ovs %s", dstGrpFlowMatch, brName)
return
}
log.Infof("Found dst group %s on ovs %s", dstGrpFlowMatch, brName)
// verify tcp rule flow entry exists
tcpFlowMatch := fmt.Sprintf("priority=110,tcp,metadata=0x640190/0x7ffffffe,nw_src=10.10.10.0/24,nw_dst=10.1.1.0/24,tp_src=200,tp_dst=100")
if !ofctlFlowMatch(flowList, POLICY_TBL_ID, tcpFlowMatch) {
t.Errorf("Could not find the flow %s on ovs %s", tcpFlowMatch, brName)
return
}
log.Infof("Found tcp rule %s on ovs %s", tcpFlowMatch, brName)
// verify udp rule flow
udpFlowMatch := fmt.Sprintf("priority=110,udp,metadata=0x12c0320/0x7ffffffe,nw_src=20.20.20.0/24,nw_dst=20.2.2.0/24,tp_src=400,tp_dst=300")
if !ofctlFlowMatch(flowList, POLICY_TBL_ID, udpFlowMatch) {
t.Errorf("Could not find the flow %s on ovs %s", udpFlowMatch, brName)
return
}
log.Infof("Found udp rule %s on ovs %s", udpFlowMatch, brName)
// Delete policies
err = ofnetMaster.DelRule(tcpRule)
if err != nil {
t.Errorf("Error deleting tcpRule {%+v}. Err: %v", tcpRule, err)
return
}
err = ofnetMaster.DelRule(udpRule)
if err != nil {
t.Errorf("Error deleting udpRule {%+v}. Err: %v", udpRule, err)
return
}
err = ofnetAgent.RemoveLocalEndpoint(endpoint.PortNo)
if err != nil {
t.Errorf("Error deleting endpoint: %+v. Err: %v", endpoint, err)
return
}
log.Infof("Deleted all policy entries")
// Get the flows again
flowList, err = ofctlFlowDump(brName)
if err != nil {
t.Errorf("Error getting flow entries. Err: %v", err)
return
}
// Make sure flows are gone
if ofctlFlowMatch(flowList, VLAN_TBL_ID, srcGrpFlowMatch) {
t.Errorf("Still found the flow %s on ovs %s", srcGrpFlowMatch, brName)
return
}
if ofctlFlowMatch(flowList, DST_GRP_TBL_ID, dstGrpFlowMatch) {
t.Errorf("Still found the flow %s on ovs %s", dstGrpFlowMatch, brName)
return
}
if ofctlFlowMatch(flowList, POLICY_TBL_ID, tcpFlowMatch) {
t.Errorf("Still found the flow %s on ovs %s", tcpFlowMatch, brName)
return
}
if ofctlFlowMatch(flowList, POLICY_TBL_ID, udpFlowMatch) {
t.Errorf("Still found the flow %s on ovs %s", udpFlowMatch, brName)
return
}
log.Infof("Verified all flows are deleted")
// Delete the bridge
ovsDriver.DeleteBridge(brName)
}
/* routerInfo[0] -> Uplink nexthop interface
*/
func NewOfnetAgent(bridgeName string, dpName string, localIp net.IP, rpcPort uint16,
ovsPort uint16, routerInfo ...string) (*OfnetAgent, error) {
log.Infof("Creating new ofnet agent for %s,%s,%d,%d,%d,%v \n", bridgeName, dpName, localIp, rpcPort, ovsPort, routerInfo)
agent := new(OfnetAgent)
// Init params
agent.localIp = localIp
agent.MyPort = rpcPort
agent.MyAddr = localIp.String()
agent.dpName = dpName
agent.arpMode = ArpProxy
agent.masterDb = make(map[string]*OfnetNode)
agent.portVlanMap = make(map[uint32]*uint16)
agent.vniVlanMap = make(map[uint32]*uint16)
agent.vlanVniMap = make(map[uint16]*uint32)
// Initialize vtep database
agent.vtepTable = make(map[string]*uint32)
// Initialize endpoint database
agent.endpointDb = cmap.New()
agent.localEndpointDb = cmap.New()
// Initialize vrf database
agent.vrfDb = make(map[string]*OfnetVrfInfo)
agent.vrfIdNameMap = make(map[uint16]*string)
agent.vrfNameIdMap = make(map[string]*uint16)
agent.vrfIdBmp = bitset.New(256)
agent.vlanVrf = make(map[uint16]*string)
// stats db
agent.stats = make(map[string]uint64)
agent.errStats = make(map[string]uint64)
// Create an openflow controller
agent.ctrler = ofctrl.NewController(agent)
// FIXME: Figure out how to handle multiple OVS bridges.
rpcServ, listener := rpcHub.NewRpcServer(rpcPort)
agent.rpcServ = rpcServ
agent.rpcListener = listener
// Register for Master add/remove events
rpcServ.Register(agent)
// Create the datapath
switch dpName {
case "vrouter":
agent.datapath = NewVrouter(agent, rpcServ)
agent.fwdMode = "routing"
case "vxlan":
agent.datapath = NewVxlan(agent, rpcServ)
agent.fwdMode = "bridge"
case "vlan":
agent.datapath = NewVlanBridge(agent, rpcServ)
agent.fwdMode = "bridge"
case "vlrouter":
agent.datapath = NewVlrouter(agent, rpcServ)
agent.fwdMode = "routing"
agent.ovsDriver = ovsdbDriver.NewOvsDriver(bridgeName)
agent.protopath = NewOfnetBgp(agent, routerInfo)
default:
log.Fatalf("Unknown Datapath %s", dpName)
}
// Start listening to controller port
go agent.ctrler.Listen(fmt.Sprintf(":%d", ovsPort))
// Return it
return agent, nil
}
// Create couple of ofnet masters and few agents
func TestOfnetInit(t *testing.T) {
var err error
// Create the masters
for i := 0; i < NUM_MASTER; i++ {
vrtrMasters[i] = NewOfnetMaster(uint16(9001 + i))
if vrtrMasters[i] == nil {
t.Fatalf("Error creating ofnet master")
}
log.Infof("Created Master: %v", vrtrMasters[i])
vxlanMasters[i] = NewOfnetMaster(uint16(9051 + i))
if vxlanMasters[i] == nil {
t.Fatalf("Error creating ofnet master")
}
log.Infof("Created Master: %v", vxlanMasters[i])
}
// Wait a second for masters to be up
time.Sleep(1 * time.Second)
// Create agents
for i := 0; i < NUM_AGENT; i++ {
rpcPort := uint16(9101 + i)
ovsPort := uint16(9151 + i)
lclIp := net.ParseIP(localIpList[i])
vrtrAgents[i], err = NewOfnetAgent("vrouter", lclIp, rpcPort, ovsPort)
if err != nil {
t.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vrtrAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vrouter ofnet agent: %v", vrtrAgents[i])
}
for i := 0; i < NUM_AGENT; i++ {
rpcPort := uint16(9201 + i)
ovsPort := uint16(9251 + i)
lclIp := net.ParseIP(localIpList[i])
vxlanAgents[i], err = NewOfnetAgent("vxlan", lclIp, rpcPort, ovsPort)
if err != nil {
t.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
vxlanAgents[i].MyAddr = "127.0.0.1"
log.Infof("Created vxlan ofnet agent: %v", vxlanAgents[i])
}
// Add master node to each agent
for i := 0; i < NUM_AGENT; i++ {
// add the two master nodes
for j := 0; j < NUM_MASTER; j++ {
var resp bool
masterInfo := OfnetNode{
HostAddr: "127.0.0.1",
HostPort: uint16(9001 + j),
}
// connect vrtr agent to vrtr master
err := vrtrAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
t.Errorf("Error adding master %+v to vrtr node %d. Err: %v", masterInfo, i, err)
}
// connect vxlan agents to vxlan master
masterInfo.HostPort = uint16(9051 + j)
err = vxlanAgents[i].AddMaster(&masterInfo, &resp)
if err != nil {
t.Errorf("Error adding master %+v to vxlan node %d. Err: %v", masterInfo, i, err)
}
}
}
log.Infof("Ofnet masters and agents are setup..")
time.Sleep(1 * time.Second)
for i := 0; i < NUM_MASTER; i++ {
err := vrtrMasters[i].MakeDummyRpcCall()
if err != nil {
t.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
err = vxlanMasters[i].MakeDummyRpcCall()
if err != nil {
t.Fatalf("Error making dummy rpc call. Err: %v", err)
return
}
}
log.Infof("Made dummy rpc call to all agents")
// Create OVS switches and connect them to vrouter ofnet agents
for i := 0; i < NUM_AGENT; i++ {
brName := "ovsbr1" + fmt.Sprintf("%d", i)
ovsPort := uint16(9151 + i)
ovsDrivers[i] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[i].AddController("127.0.0.1", ovsPort)
if err != nil {
t.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Create OVS switches and connect them to vxlan ofnet agents
for i := 0; i < NUM_AGENT; i++ {
brName := "ovsbr2" + fmt.Sprintf("%d", i)
ovsPort := uint16(9251 + i)
j := NUM_AGENT + i
ovsDrivers[j] = ovsdbDriver.NewOvsDriver(brName)
err := ovsDrivers[j].AddController("127.0.0.1", ovsPort)
if err != nil {
t.Fatalf("Error adding controller to ovs: %s", brName)
}
}
// Wait for 10sec for switch to connect to controller
time.Sleep(10 * time.Second)
}
func TestPolicy(t *testing.T) {
rpcPort := uint16(9101)
ovsPort := uint16(9151)
lclIp := net.ParseIP("10.10.10.10")
ofnetAgent, err := NewOfnetAgent("vrouter", lclIp, rpcPort, ovsPort)
if err != nil {
t.Fatalf("Error creating ofnet agent. Err: %v", err)
}
// Override MyAddr to local host
ofnetAgent.MyAddr = "127.0.0.1"
log.Infof("Created vrouter ofnet agent: %v", ofnetAgent)
brName := "ovsbr60"
ovsDriver := ovsdbDriver.NewOvsDriver(brName)
err = ovsDriver.AddController("127.0.0.1", ovsPort)
if err != nil {
t.Fatalf("Error adding controller to ovs: %s", brName)
}
// Wait for 10sec for switch to connect to controller
time.Sleep(10 * time.Second)
// create policy agent
policyAgent := NewPolicyAgent(ofnetAgent, ofnetAgent.ofSwitch)
// Init tables
policyAgent.InitTables(IP_TBL_ID)
// Add an Endpoint
err = policyAgent.AddEndpoint(&OfnetEndpoint{
EndpointID: "1234",
EndpointGroup: 100,
IpAddr: net.ParseIP("10.10.10.10"),
})
if err != nil {
t.Errorf("Error adding endpoint. Err: %v", err)
}
var resp bool
tcpRule := &OfnetPolicyRule{
RuleId: "tcpRule",
SrcEndpointGroup: 100,
DstEndpointGroup: 200,
SrcIpAddr: "10.10.10.10/24",
DstIpAddr: "10.1.1.1/24",
IpProtocol: 6,
DstPort: 100,
SrcPort: 200,
}
// Add a policy
err = policyAgent.AddRule(tcpRule, &resp)
if err != nil {
t.Errorf("Error installing tcpRule {%+v}. Err: %v", tcpRule, err)
}
udpRule := &OfnetPolicyRule{
RuleId: "udpRule",
SrcEndpointGroup: 300,
DstEndpointGroup: 400,
SrcIpAddr: "20.20.20.20/24",
DstIpAddr: "20.2.2.2/24",
IpProtocol: 17,
DstPort: 300,
SrcPort: 400,
}
// Add the policy
err = policyAgent.AddRule(udpRule, &resp)
if err != nil {
t.Errorf("Error installing udpRule {%+v}. Err: %v", udpRule, err)
}
// Cleanup
ofnetAgent.Delete()
ovsDriver.DeleteBridge(brName)
}