func TestOneHostVlan_regress(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() cfgFile := utils.GetCfgFile("one_host_vlan") jsonCfg, err := ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } utils.ConfigSetupCommon(t, string(jsonCfg), testbed.GetNodes()) node1 := testbed.GetNodes()[0] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node1, "orange-myContainer1", u.EtcdNameStr) utils.StartClient(t, node1, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node1, "myContainer2") }() }
func TestTwoHostsMultiVlanPingFailure_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() jsonCfg := `{ "Hosts" : [{ "Name" : "host1", "Intf" : "eth2" }, { "Name" : "host2", "Intf" : "eth2" }], "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "Vlans" : "11-48", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer2", "Host" : "host2" } ] } ] } ] }` utils.ConfigSetupCommon(t, jsonCfg, testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer1", u.EtcdNameStr) utils.StartClientFailure(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() }
func TestMultipleEpsInContainer_regress(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() cfgFile := utils.GetCfgFile("multiple_eps_in_container") jsonCfg, err := ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } utils.ConfigSetupCommon(t, string(jsonCfg), testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] // Container2 is reachable on both orange and purple networks utils.StartServer(t, node1, "myContainer2") defer func() { utils.DockerCleanup(t, node1, "myContainer2") }() ipAddress := utils.GetIPAddress(t, node1, "orange-myContainer2", u.EtcdNameStr) utils.StartClient(t, node1, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node1, "myContainer3") }() ipAddress = utils.GetIPAddress(t, node1, "purple-myContainer2", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() // Container1 is reachable on only on orange network utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress = utils.GetIPAddress(t, node1, "orange-myContainer1", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() utils.DockerCleanup(t, node2, "myContainer4") utils.StartClientFailure(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() }
func TestTwoHostVxlan_regress(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() cfgFile := utils.GetCfgFile("two_host_vxlan") jsonCfg, err := ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } utils.ConfigSetupCommon(t, string(jsonCfg), testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] // all four containers can talk to each other utils.StartServer(t, node1, "myContainer2") defer func() { utils.DockerCleanup(t, node1, "myContainer2") }() utils.StartServer(t, node2, "myContainer4") defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() ipAddress := utils.GetIPAddress(t, node1, "orange-myContainer2", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() utils.StartClient(t, node1, "myContainer1", ipAddress) defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress = utils.GetIPAddress(t, node1, "orange-myContainer4", u.EtcdNameStr) utils.DockerCleanup(t, node2, "myContainer3") utils.StartClient(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() utils.DockerCleanup(t, node1, "myContainer1") utils.StartClient(t, node1, "myContainer1", ipAddress) defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() }
func TestTwoHostsMultipleVxlansNetsLateHostBindings_regress(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() cfgFile := utils.GetCfgFile("late_bindings/multiple_vxlan_nets") jsonCfg, err := ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } utils.ConfigSetupCommon(t, string(jsonCfg), testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] cfgFile = utils.GetCfgFile("late_bindings/multiple_vxlan_nets_host_bindings") jsonCfg, err = ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } utils.ApplyHostBindingsConfig(t, string(jsonCfg), node1) // Container1 and Container3 are on orange network utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node1, "orange-myContainer1", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() // Container2 and Container4 are on purple network utils.StartServer(t, node1, "myContainer2") defer func() { utils.DockerCleanup(t, node1, "myContainer2") }() ipAddress = utils.GetIPAddress(t, node1, "purple-myContainer2", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() }
// Testcase: // - Create a single vlan network with two endpoints // - Verify that the endpoints are able to ping func TestSingleHostSingleVlanPingSuccess_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() //create a single vlan network, with two endpoints jsonCfg := `{ "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "Vlans" : "11-48", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" }, { "Container" : "myContainer2", "Host" : "host1" } ] } ] } ] }` utils.ConfigSetupCommon(t, jsonCfg, testbed.GetNodes()) node := testbed.GetNodes()[0] utils.StartServer(t, node, "myContainer1") defer func() { utils.DockerCleanup(t, node, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node, "orange-myContainer1", u.EtcdNameStr) utils.StartClient(t, node, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node, "myContainer2") }() }
// XXX: don't run this until we upgrade docker to a recent version that supports // labels and build-time env func TestTwoHostVlanPowerstripDocker(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() cfgFile := utils.GetCfgFile("late_bindings/powerstrip_demo_vlan_nets") jsonCfg, err := ioutil.ReadFile(cfgFile) if err != nil { t.Fatalf("failed to read config file %s \n", err) } node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartNetPlugin(t, testbed.GetNodes(), true) utils.StartPowerStripAdapter(t, testbed.GetNodes()) utils.ApplyDesiredConfig(t, string(jsonCfg), node1) env := []string{"DOCKER_HOST=localhost:2375"} utils.StartServerWithEnvAndArgs(t, node1, "server1", env, []string{"--label", "netid=orange", "--label", "tenantid=tenant-one"}) defer func() { utils.DockerCleanupWithEnv(t, node1, "server1", env) }() ipAddress := utils.GetIPAddressFromNetworkAndContainerName(t, node1, "orange", "server1") // test ping success between containers in same network utils.StartClientWithEnvAndArgs(t, node2, "client1", ipAddress, env, []string{"--label", "netid=orange", "--label", "tenantid=tenant-one"}) defer func() { utils.DockerCleanupWithEnv(t, node2, "client1", env) }() // test ping failure between containers in different networks utils.StartClientFailureWithEnvAndArgs(t, node2, "client2", ipAddress, env, []string{"--label", "netid=purple", "--label", "tenantid=tenant-one"}) defer func() { utils.DockerCleanupWithEnv(t, node2, "client2", env) }() }
func TestTwoHostsVxlanAddDelEp_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() jsonCfg := `{ "Hosts" : [{ "Name" : "host1", "VtepIp" : "192.168.2.10" }, { "Name" : "host2", "VtepIp" : "192.168.2.11" }], "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vxlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "VXlans" : "10001-11000", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" }, { "Container" : "myContainer2", "Host" : "host2" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer3", "Host" : "host1" }, { "Container" : "myContainer4", "Host" : "host2" } ] } ] } ] }` utils.ConfigSetupCommon(t, jsonCfg, testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() utils.StartServer(t, node1, "myContainer3") defer func() { utils.DockerCleanup(t, node1, "myContainer3") }() ipAddress := utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer1", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() ipAddress = utils.GetIPAddress(t, node1, "purple.tenant-one-myContainer3", u.EtcdNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() jsonCfg = ` { "Tenants" : [ { "Name" : "tenant-one", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer5", "Host" : "host1" } ] } ] } ] }` utils.AddConfig(t, jsonCfg, testbed.GetNodes()[0]) utils.StartServer(t, node1, "myContainer5") defer func() { utils.DockerCleanup(t, node1, "myContainer5") }() ipAddress = utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer5", u.EtcdNameStr) utils.DockerCleanup(t, node2, "myContainer2") utils.StartClient(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() utils.DelConfig(t, jsonCfg, testbed.GetNodes()[0]) utils.DockerCleanup(t, node2, "myContainer2") utils.StartClientFailure(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() jsonCfg = ` { "Tenants" : [ { "Name" : "tenant-one", "Networks" : [ { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer5", "Host" : "host1" } ] } ] } ] }` utils.AddConfig(t, jsonCfg, testbed.GetNodes()[0]) ipAddress = utils.GetIPAddress(t, node1, "purple.tenant-one-myContainer5", u.EtcdNameStr) utils.DockerCleanup(t, node2, "myContainer4") utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() }
func TestTwoHostsMultiVxlanPingFailureStatefulStart_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() //create a single vlan network, with two endpoints jsonCfg := `{ "Hosts" : [{ "Name" : "host1", "VtepIp" : "192.168.2.10" }, { "Name" : "host2", "VtepIp" : "192.168.2.11" }], "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vxlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "VXlans" : "10001-11000", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" }, { "Container" : "myContainer2", "Host" : "host2" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer3", "Host" : "host1" }, { "Container" : "myContainer4", "Host" : "host2" } ] } ] } ] }` utils.ConfigSetupCommon(t, jsonCfg, testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer1", u.EtcdNameStr) utils.StartClientFailure(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() utils.StartClientFailure(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() // Wait for Netplugin to cleanup time.Sleep(1 * time.Second) //restart the netplugin and retry the pings utils.StopNetPlugin(t, testbed.GetNodes()) utils.StartNetPlugin(t, testbed.GetNodes(), false) utils.DockerCleanup(t, node2, "myContainer3") utils.DockerCleanup(t, node2, "myContainer4") ipAddress = utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer1", u.EtcdNameStr) utils.StartClientFailure(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() utils.StartClientFailure(t, node2, "myContainer3", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer3") }() }
func TestTwoHostsVxlanAddDelNetworkConsul_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() jsonCfg := `{ "Hosts" : [{ "Name" : "host1", "VtepIp" : "192.168.2.10" }, { "Name" : "host2", "VtepIp" : "192.168.2.11" }], "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vxlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "VXlans" : "10001-11000", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" }, { "Container" : "myContainer2", "Host" : "host2" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer3", "Host" : "host1" }, { "Container" : "myContainer4", "Host" : "host2" } ] } ] } ] }` utils.ConfigSetupCommonWithConsul(t, jsonCfg, testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() utils.StartServer(t, node1, "myContainer3") defer func() { utils.DockerCleanup(t, node1, "myContainer3") }() ipAddress := utils.GetIPAddress(t, node1, "orange.tenant-one-myContainer1", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() ipAddress = utils.GetIPAddress(t, node1, "purple.tenant-one-myContainer3", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() jsonCfg = ` { "Tenants" : [ { "Name" : "tenant-one", "Networks" : [ { "Name" : "green", "Endpoints" : [ { "Container" : "myContainer5", "Host" : "host1" }, { "Container" : "myContainer6", "Host" : "host2" } ] } ] } ] }` utils.AddConfigConsul(t, jsonCfg, testbed.GetNodes()[0]) utils.StartServer(t, node1, "myContainer5") defer func() { utils.DockerCleanup(t, node1, "myContainer5") }() ipAddress = utils.GetIPAddress(t, node1, "green.tenant-one-myContainer5", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer6", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer6") }() utils.DelConfigConsul(t, jsonCfg, testbed.GetNodes()[0]) utils.DockerCleanup(t, node2, "myContainer6") utils.StartClientFailure(t, node2, "myContainer6", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer6") }() jsonCfg = ` { "Tenants" : [ { "Name" : "tenant-one", "Networks" : [ { "Name" : "green" } ] } ] }` utils.DelConfigConsul(t, jsonCfg, testbed.GetNodes()[0]) time.Sleep(1 * time.Second) if utils.NetworkStateExists(node2, "green", u.ConsulNameStr) == nil { t.Fatalf("Error - network %s doesn't seem to be deleted \n", "green") } }
func TestTwoHostsMultiVxlanPingSuccessStatefulStartConsul_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() jsonCfg := `{ "Hosts" : [{ "Name" : "host1", "VtepIp" : "192.168.2.10" }, { "Name" : "host2", "VtepIp" : "192.168.2.11" }], "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vxlan", "SubnetPool" : "11.1.0.0/16", "AllocSubnetLen" : 24, "VXlans" : "10001-14000", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" }, { "Container" : "myContainer2", "Host" : "host2" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer3", "Host" : "host1" }, { "Container" : "myContainer4", "Host" : "host2" } ] } ] } ] }` utils.ConfigSetupCommonWithConsul(t, jsonCfg, testbed.GetNodes()) node1 := testbed.GetNodes()[0] node2 := testbed.GetNodes()[1] utils.StartServer(t, node1, "myContainer1") defer func() { utils.DockerCleanup(t, node1, "myContainer1") }() utils.StartServer(t, node1, "myContainer3") defer func() { utils.DockerCleanup(t, node1, "myContainer3") }() ipAddress := utils.GetIPAddress(t, node1, "orange-myContainer1", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() ipAddress = utils.GetIPAddress(t, node1, "purple-myContainer3", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() //restart the netplugin and retry the pings utils.StopNetPlugin(t, testbed.GetNodes()) utils.StartNetPluginWithConfig(t, testbed.GetNodes(), false, utils.GetNetpluginConfigWithConsul()) utils.DockerCleanup(t, node2, "myContainer2") utils.DockerCleanup(t, node2, "myContainer4") ipAddress = utils.GetIPAddress(t, node1, "orange-myContainer1", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer2") }() ipAddress = utils.GetIPAddress(t, node1, "purple-myContainer3", u.ConsulNameStr) utils.StartClient(t, node2, "myContainer4", ipAddress) defer func() { utils.DockerCleanup(t, node2, "myContainer4") }() }
// Default Network Assignment and freeing func TestSingleHostDefaultNetwork_sanity(t *testing.T) { defer func() { utils.ConfigCleanupCommon(t, testbed.GetNodes()) utils.StopOnError(t.Failed()) }() //create a single vlan network, with two endpoints jsonCfg := `{ "Tenants" : [ { "Name" : "tenant-one", "DefaultNetType" : "vlan", "DefaultNetwork" : "orange", "SubnetPool" : "100.1.0.0/16", "AllocSubnetLen" : 24, "Vlans" : "11-48", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" } ] }, { "Name" : "purple", "Endpoints" : [ { "Container" : "myContainer2", "Host" : "host1" } ] } ] } ] }` utils.ConfigSetupCommon(t, jsonCfg, testbed.GetNodes()) node := testbed.GetNodes()[0] utils.StartServer(t, node, "myContainer1") defer func() { utils.DockerCleanup(t, node, "myContainer1") }() ipAddress := utils.GetIPAddress(t, node, "orange.tenant-one-myContainer1", u.EtcdNameStr) utils.StartClientFailure(t, node, "myContainer2", ipAddress) defer func() { utils.DockerCleanup(t, node, "myContainer2") }() // confirm the default gateway in one of the containers output, err := node.RunCommandWithOutput("docker exec myContainer1 /sbin/ip route") if err != nil { t.Fatalf("Error - unable to get default ip route, output = '%s'", output) } if !strings.Contains(output, "default via 100.1.0.254") { t.Fatalf("Error - unable to confirm container's default ip route, output = '%s'", output) } jsonCfg = ` { "Tenants" : [ { "Name" : "tenant-one", "Networks" : [ { "Name" : "orange", "Endpoints" : [ { "Container" : "myContainer1", "Host" : "host1" } ] } ] } ] }` // deletion would result into unassignment utils.DelConfig(t, jsonCfg, testbed.GetNodes()[0]) time.Sleep(1 * time.Second) // confirm the default gateway in one of the containers output, err = node.RunCommandWithOutput("docker exec myContainer1 /sbin/ip route") if err != nil { t.Fatalf("Error - unable to get default ip route, output = '%s'", output) } if strings.Contains(output, "default via 100.1.0.254") { t.Fatalf("Error - able to still find the default rout after network is deleted output = '%s'", output) } }