func (c *networkCmd) doNetworkAttach(client *lxd.Client, name string, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errArgs
}
container := args[0]
devName := name
if len(args) > 1 {
devName = args[1]
}
network, err := client.NetworkGet(name)
if err != nil {
return err
}
nicType := "macvlan"
if network.Type == "bridge" {
nicType = "bridged"
}
props := []string{fmt.Sprintf("nictype=%s", nicType), fmt.Sprintf("parent=%s", name)}
resp, err := client.ContainerDeviceAdd(container, devName, "nic", props)
if err != nil {
return err
}
return client.WaitForSuccess(resp.Operation)
}
func doDelete(d *lxd.Client, name string) error {
resp, err := d.Delete(name)
if err != nil {
return err
}
return d.WaitForSuccess(resp.Operation)
}
func doProfileApply(client *lxd.Client, c string, p string) error {
resp, err := client.ApplyProfile(c, p)
if err == nil {
if p == "" {
p = gettext.Gettext("(none)")
}
fmt.Printf(gettext.Gettext("Profile %s applied to %s")+"\n", p, c)
} else {
return err
}
return client.WaitForSuccess(resp.Operation)
}
func lxdForceDelete(d *lxd.Client, name string) error {
resp, err := d.Action(name, "stop", -1, true, false)
if err == nil {
d.WaitForSuccess(resp.Operation)
}
resp, err = d.Delete(name)
if err != nil {
return err
}
return d.WaitForSuccess(resp.Operation)
}
func (c *networkCmd) doNetworkDetach(client *lxd.Client, name string, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errArgs
}
containerName := args[0]
devName := ""
if len(args) > 1 {
devName = args[1]
}
container, err := client.ContainerInfo(containerName)
if err != nil {
return err
}
if devName == "" {
for n, d := range container.Devices {
if d["type"] == "nic" && d["parent"] == name {
if devName != "" {
return fmt.Errorf(i18n.G("More than one device matches, specify the device name."))
}
devName = n
}
}
}
if devName == "" {
return fmt.Errorf(i18n.G("No device found for this network"))
}
device, ok := container.Devices[devName]
if !ok {
return fmt.Errorf(i18n.G("The specified device doesn't exist"))
}
if device["type"] != "nic" || device["parent"] != name {
return fmt.Errorf(i18n.G("The specified device doesn't match the network"))
}
resp, err := client.ContainerDeviceDelete(containerName, devName)
if err != nil {
return err
}
return client.WaitForSuccess(resp.Operation)
}
func doProfileApply(client *lxd.Client, c string, p string) error {
resp, err := client.ApplyProfile(c, p)
if err != nil {
return err
}
err = client.WaitForSuccess(resp.Operation)
if err == nil {
if p == "" {
p = i18n.G("(none)")
}
fmt.Printf(i18n.G("Profile %s applied to %s")+"\n", p, c)
}
return err
}
func (c *profileCmd) doProfileAssign(client *lxd.Client, d string, p string) error {
resp, err := client.AssignProfile(d, p)
if err != nil {
return err
}
err = client.WaitForSuccess(resp.Operation)
if err == nil {
if p == "" {
p = i18n.G("(none)")
}
fmt.Printf(i18n.G("Profiles %s applied to %s")+"\n", p, d)
}
return err
}
func (c *deleteCmd) doDelete(d *lxd.Client, name string) error {
if c.interactive {
reader := bufio.NewReader(os.Stdin)
fmt.Printf(i18n.G("Remove %s (yes/no): "), name)
input, _ := reader.ReadString('\n')
input = strings.TrimSuffix(input, "\n")
if !shared.StringInSlice(strings.ToLower(input), []string{i18n.G("yes")}) {
return fmt.Errorf(i18n.G("User aborted delete operation."))
}
}
resp, err := d.Delete(name)
if err != nil {
return err
}
return d.WaitForSuccess(resp.Operation)
}
func spawnContainers(c *lxd.Client, count int, image string, privileged bool) error {
batch := *argParallel
if batch < 1 {
// Detect the number of parallel actions
cpus, err := ioutil.ReadDir("/sys/bus/cpu/devices")
if err != nil {
return err
}
batch = len(cpus)
}
batches := count / batch
remainder := count % batch
// Print the test header
st, err := c.ServerStatus()
if err != nil {
return err
}
privilegedStr := "unprivileged"
if privileged {
privilegedStr = "privileged"
}
mode := "normal startup"
if *argFreeze {
mode = "start and freeze"
}
fmt.Printf("Test environment:\n")
fmt.Printf(" Server backend: %s\n", st.Environment.Server)
fmt.Printf(" Server version: %s\n", st.Environment.ServerVersion)
fmt.Printf(" Kernel: %s\n", st.Environment.Kernel)
fmt.Printf(" Kernel architecture: %s\n", st.Environment.KernelArchitecture)
fmt.Printf(" Kernel version: %s\n", st.Environment.KernelVersion)
fmt.Printf(" Storage backend: %s\n", st.Environment.Storage)
fmt.Printf(" Storage version: %s\n", st.Environment.StorageVersion)
fmt.Printf(" Container backend: %s\n", st.Environment.Driver)
fmt.Printf(" Container version: %s\n", st.Environment.DriverVersion)
fmt.Printf("\n")
fmt.Printf("Test variables:\n")
fmt.Printf(" Container count: %d\n", count)
fmt.Printf(" Container mode: %s\n", privilegedStr)
fmt.Printf(" Startup mode: %s\n", mode)
fmt.Printf(" Image: %s\n", image)
fmt.Printf(" Batches: %d\n", batches)
fmt.Printf(" Batch size: %d\n", batch)
fmt.Printf(" Remainder: %d\n", remainder)
fmt.Printf("\n")
// Pre-load the image
var fingerprint string
if strings.Contains(image, ":") {
var remote string
remote, fingerprint = lxd.DefaultConfig.ParseRemoteAndContainer(image)
if fingerprint == "" {
fingerprint = "default"
}
d, err := lxd.NewClient(&lxd.DefaultConfig, remote)
if err != nil {
return err
}
target := d.GetAlias(fingerprint)
if target != "" {
fingerprint = target
}
_, err = c.GetImageInfo(fingerprint)
if err != nil {
logf("Importing image into local store: %s", fingerprint)
err := d.CopyImage(fingerprint, c, false, nil, false, false, nil)
if err != nil {
return err
}
} else {
logf("Found image in local store: %s", fingerprint)
}
} else {
fingerprint = image
logf("Found image in local store: %s", fingerprint)
}
// Start the containers
spawnedCount := 0
nameFormat := "benchmark-%." + fmt.Sprintf("%d", len(fmt.Sprintf("%d", count))) + "d"
wgBatch := sync.WaitGroup{}
nextStat := batch
startContainer := func(name string) {
defer wgBatch.Done()
// Configure
config := map[string]string{}
if privileged {
config["security.privileged"] = "true"
}
config["user.lxd-benchmark"] = "true"
// Create
resp, err := c.Init(name, "local", fingerprint, nil, config, nil, false)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
// Start
resp, err = c.Action(name, "start", -1, false, false)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
// Freeze
if *argFreeze {
resp, err = c.Action(name, "freeze", -1, false, false)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf(fmt.Sprintf("Failed to spawn container '%s': %s", name, err))
return
}
}
}
logf("Starting the test")
timeStart := time.Now()
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
spawnedCount = spawnedCount + 1
name := fmt.Sprintf(nameFormat, spawnedCount)
wgBatch.Add(1)
go startContainer(name)
}
wgBatch.Wait()
if spawnedCount >= nextStat {
interval := time.Since(timeStart).Seconds()
logf("Started %d containers in %.3fs (%.3f/s)", spawnedCount, interval, float64(spawnedCount)/interval)
nextStat = nextStat * 2
}
}
for k := 0; k < remainder; k++ {
spawnedCount = spawnedCount + 1
name := fmt.Sprintf(nameFormat, spawnedCount)
wgBatch.Add(1)
go startContainer(name)
}
wgBatch.Wait()
logf("Test completed in %.3fs", time.Since(timeStart).Seconds())
return nil
}
func deleteContainers(c *lxd.Client) error {
batch := *argParallel
if batch < 1 {
// Detect the number of parallel actions
cpus, err := ioutil.ReadDir("/sys/bus/cpu/devices")
if err != nil {
return err
}
batch = len(cpus)
}
// List all the containers
allContainers, err := c.ListContainers()
if err != nil {
return err
}
containers := []shared.ContainerInfo{}
for _, container := range allContainers {
if container.Config["user.lxd-benchmark"] != "true" {
continue
}
containers = append(containers, container)
}
// Delete them all
count := len(containers)
logf("%d containers to delete", count)
batches := count / batch
deletedCount := 0
wgBatch := sync.WaitGroup{}
nextStat := batch
deleteContainer := func(ct shared.ContainerInfo) {
defer wgBatch.Done()
// Stop
if ct.IsActive() {
resp, err := c.Action(ct.Name, "stop", -1, true, false)
if err != nil {
logf("Failed to delete container: %s", ct.Name)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to delete container: %s", ct.Name)
return
}
}
// Delete
resp, err := c.Delete(ct.Name)
if err != nil {
logf("Failed to delete container: %s", ct.Name)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to delete container: %s", ct.Name)
return
}
}
logf("Starting the cleanup")
timeStart := time.Now()
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
wgBatch.Add(1)
go deleteContainer(containers[deletedCount])
deletedCount = deletedCount + 1
}
wgBatch.Wait()
if deletedCount >= nextStat {
interval := time.Since(timeStart).Seconds()
logf("Deleted %d containers in %.3fs (%.3f/s)", deletedCount, interval, float64(deletedCount)/interval)
nextStat = nextStat * 2
}
}
for k := deletedCount; k < count; k++ {
wgBatch.Add(1)
go deleteContainer(containers[deletedCount])
deletedCount = deletedCount + 1
}
wgBatch.Wait()
logf("Cleanup completed")
return nil
}
func cmdStop(c *lxd.Client, args []string) error {
var wgBatch sync.WaitGroup
if os.Getuid() != 0 {
return fmt.Errorf("Container stop must be run as root.")
}
// Load the simulation
routersMap, err := importFromLXD(c)
if err != nil {
return err
}
routers := []*Router{}
for _, v := range routersMap {
if v.Tier < 1 || v.Tier > 3 {
continue
}
routers = append(routers, v)
}
// Helper function
stopContainer := func(name string) {
defer wgBatch.Done()
resp, err := c.Action(name, "stop", -1, true, false)
if err != nil {
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
return
}
}
// Stop the containers
batch := 8
batches := len(routers) / batch
remainder := len(routers) % batch
current := 0
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
wgBatch.Add(1)
go stopContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
}
for k := 0; k < remainder; k++ {
wgBatch.Add(1)
go stopContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
// Destroy all the interfaces
err = networkDestroy(routersMap)
if err != nil {
return err
}
return nil
}
func cmdStart(c *lxd.Client, args []string) error {
var wgBatch sync.WaitGroup
if os.Getuid() != 0 {
return fmt.Errorf("Container startup must be run as root.")
}
// Load the simulation
routersMap, err := importFromLXD(c)
if err != nil {
return err
}
routers := []*Router{}
for _, v := range routersMap {
if v.Tier < 1 || v.Tier > 3 {
continue
}
routers = append(routers, v)
}
// Create all the needed interfaces
logf("Creating the network interfaces")
err = networkCreate(routersMap)
if err != nil {
return err
}
// Helper function
startContainer := func(name string) {
defer wgBatch.Done()
resp, err := c.Action(name, "start", -1, false, false)
if err != nil {
logf("Failed to start container '%s': %s", name, err)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to start container '%s': %s", name, err)
return
}
}
// Start the containers
batch := 8
batches := len(routers) / batch
remainder := len(routers) % batch
logf("Starting the containers")
current := 0
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
wgBatch.Add(1)
go startContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
}
for k := 0; k < remainder; k++ {
wgBatch.Add(1)
go startContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
logf("%d containers started", len(routers))
return nil
}
func cmdDestroy(c *lxd.Client, args []string) error {
var wgBatch sync.WaitGroup
if os.Getuid() != 0 {
return fmt.Errorf("Container destruction must be run as root.")
}
// Load the simulation
routersMap, err := importFromLXD(c)
if err != nil {
return err
}
routers := []*Router{}
for _, v := range routersMap {
if v.Tier < 1 || v.Tier > 3 {
continue
}
routers = append(routers, v)
}
// Load the LXD container list
containers, err := c.ListContainers()
if err != nil {
return err
}
containersMap := map[string]api.Container{}
for _, ctn := range containers {
containersMap[ctn.Name] = ctn
}
// Helper function
deleteContainer := func(name string) {
defer wgBatch.Done()
ct, ok := containersMap[name]
if !ok {
logf("Failed to delete container: %s: Doesn't exist", ct.Name)
return
}
// Stop
if ct.IsActive() {
resp, err := c.Action(ct.Name, "stop", -1, true, false)
if err != nil {
logf("Failed to delete container: %s: %s", ct.Name, err)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to delete container: %s: %s", ct.Name, err)
return
}
}
// Delete
resp, err := c.Delete(ct.Name)
if err != nil {
logf("Failed to delete container: %s: %s", ct.Name, err)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to delete container: %s: %s", ct.Name, err)
return
}
}
// Delete all the containers
batch := 8
batches := len(routers) / batch
remainder := len(routers) % batch
current := 0
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
wgBatch.Add(1)
go deleteContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
}
for k := 0; k < remainder; k++ {
wgBatch.Add(1)
go deleteContainer(routers[current].Name)
current += 1
}
wgBatch.Wait()
// Destroy all the interfaces
err = networkDestroy(routersMap)
if err != nil {
return err
}
return nil
}
func cmdCreate(c *lxd.Client, args []string) error {
var wgBatch sync.WaitGroup
// A path must be provided
if len(args) < 1 {
return fmt.Errorf("A path must be passed to create.")
}
// Load the simulation
routersMap, err := importFromCSV(args[0])
if err != nil {
return err
}
routers := []*Router{}
for _, v := range routersMap {
if v.Tier < 1 {
continue
}
routers = append(routers, v)
}
// Clear any existing images
fp := c.GetAlias("internet-router")
if fp != "" {
logf("Deleting the existing router image: %s", fp)
err = c.DeleteImage(fp)
if err != nil {
return err
}
}
// Load the image
logf("Importing the router image")
_, err = c.PostImage("image/image-meta.tar.xz", "image/image-rootfs.tar.xz", nil, false, []string{"internet-router"}, nil)
if err != nil {
return err
}
logf("New router image imported: %s", fp)
// Create the profile
_, err = c.ProfileConfig("internet-base")
if err != nil {
logf("Creating the profile")
err := c.ProfileCreate("internet-base")
if err != nil {
return err
}
}
// Helper function
createContainer := func(router *Router) {
defer wgBatch.Done()
var interfaces string
var bgpd string
// Configuration
config := map[string]string{}
devices := map[string]map[string]string{}
config["user.internet.type"] = "router"
config["user.internet.organization"] = router.Organization
config["user.internet.priority"] = fmt.Sprintf("%d", router.Priority)
config["user.internet.tier"] = fmt.Sprintf("%d", router.Tier)
config["user.internet.location"] = router.Location
for i, r := range router.DNS {
config[fmt.Sprintf("user.internet.dns.%d", i)] = r
}
config["user.internet.router.fqdn"] = router.Configuration.FQDN
config["user.internet.router.asn"] = fmt.Sprintf("%d", router.Configuration.ASN)
config["user.internet.router.password.login"] = router.Configuration.PasswordLogin
config["user.internet.router.password.enable"] = router.Configuration.PasswordEnable
if router.Configuration.RouterID != nil {
config["user.internet.router.routerid"] = router.Configuration.RouterID.String()
}
if router.Internal {
config["user.internet.internal"] = "true"
} else {
config["user.internet.internal"] = "false"
}
if router.Tier >= 1 && router.Tier <= 3 {
interfaces = fmt.Sprintf(`auto lo
iface lo inet loopback
pre-up echo 0 > /proc/sys/net/ipv6/conf/all/accept_dad || true
post-up echo 1 > /proc/sys/net/ipv6/conf/all/forwarding || true
auto local
iface local inet6 manual
pre-up ip link add local type dummy || true
pre-up ip link set local up || true
`)
}
for i, r := range router.Configuration.Loopback.Addresses {
config[fmt.Sprintf("user.internet.router.loopback.address.%d", i)] = r.String()
if router.Tier >= 1 && router.Tier <= 3 {
interfaces += fmt.Sprintf(" post-up ip -6 addr add dev local %s/128 || true\n", r.String())
}
}
for i, r := range router.Configuration.Loopback.Routes {
config[fmt.Sprintf("user.internet.router.loopback.route.%d.subnet", i)] = r.Subnet.String()
if router.Tier >= 1 && router.Tier <= 3 {
interfaces += fmt.Sprintf(" post-up sleep 10 ; ip -6 route add dev local %s || true\n", r.Subnet.String())
}
}
if router.Tier >= 1 && router.Tier <= 3 {
bgpd = fmt.Sprintf(fmt.Sprintf(`hostname %s
password %s
enable password %s
router bgp %d
bgp router-id %s
no bgp default ipv4-unicast
`, router.Name, router.Configuration.PasswordLogin, router.Configuration.PasswordEnable, router.Configuration.ASN, router.Configuration.RouterID.String()))
}
if router.Peers != nil {
for _, p := range router.Peers {
if strings.HasPrefix(p.Interface, "v") {
device := map[string]string{
"type": "nic",
"nictype": "physical",
"name": p.Interface,
"parent": p.Interface,
"hwaddr": p.MAC,
}
devices[p.Interface] = device
} else if strings.HasPrefix(p.Interface, "br") {
device := map[string]string{
"type": "nic",
"nictype": "bridged",
"name": p.Interface,
"parent": p.Interface,
"hwaddr": p.MAC,
}
devices[p.Interface] = device
} else {
logf("Failed to configure container '%s': Bad interface name: %s", router.Name, p.Interface)
return
}
if router.Tier >= 1 && router.Tier <= 3 {
interfaces += fmt.Sprintf(`
auto %s
iface %s inet6 manual
post-up tc qdisc add dev %s root netem delay %dms || true
post-up tc qdisc add dev %s root netem rate %dmbit || true
`, p.Interface, p.Interface, p.Interface, p.Delay, p.Interface, p.Speed)
if p.ASN != 0 {
bgpd += fmt.Sprintf(`
neighbor %s remote-as %d
neighbor %s weight %d
neighbor %s interface %s
`, p.Remote, p.ASN, p.Remote, p.Weight, p.Remote, p.Interface)
}
}
config[fmt.Sprintf("user.internet.peer.%s.interface", p.Name)] = p.Interface
config[fmt.Sprintf("user.internet.peer.%s.mac", p.Name)] = p.MAC
config[fmt.Sprintf("user.internet.peer.%s.remote", p.Name)] = p.Remote
config[fmt.Sprintf("user.internet.peer.%s.speed", p.Name)] = fmt.Sprintf("%d", p.Speed)
config[fmt.Sprintf("user.internet.peer.%s.delay", p.Name)] = fmt.Sprintf("%d", p.Delay)
config[fmt.Sprintf("user.internet.peer.%s.asn", p.Name)] = fmt.Sprintf("%d", p.ASN)
config[fmt.Sprintf("user.internet.peer.%s.weight", p.Name)] = fmt.Sprintf("%d", p.Weight)
if p.Routes != nil {
for i, r := range p.Routes {
config[fmt.Sprintf("user.internet.peer.%s.route.%d.subnet", p.Name, i)] = r.Subnet.String()
if r.Gateway != nil {
config[fmt.Sprintf("user.internet.peer.%s.route.%d.gateway", p.Name, i)] = r.Gateway.String()
if router.Tier >= 1 && router.Tier <= 3 {
interfaces += fmt.Sprintf(" post-up sleep 10 ; ip -6 route add dev %s %s via %s || true\n", p.Interface, r.Subnet.String(), r.Gateway.String())
}
} else {
if router.Tier >= 1 && router.Tier <= 3 {
interfaces += fmt.Sprintf(" post-up sleep 10 ; ip -6 route add dev %s %s via %s || true\n", p.Interface, r.Subnet.String(), p.Remote)
}
}
}
}
}
}
if router.Tier >= 1 && router.Tier <= 3 {
bgpd += " address-family ipv6\n"
if router.Peers != nil {
for _, p := range router.Peers {
if p.ASN != 0 {
bgpd += fmt.Sprintf(" neighbor %s activate\n", p.Remote)
}
}
}
bgpd += ` redistribute connected
redistribute kernel
exit-address-family
`
config["user.internet.config.interfaces"] = interfaces
config["user.internet.config.bgpd"] = bgpd
}
// Config-only containers
if router.Tier > 3 {
ct, err := c.ContainerInfo(router.Name)
if err != nil {
logf("Failed to configure container '%s': %s", router.Name, err)
return
}
for k, _ := range ct.Config {
if strings.HasPrefix(k, "user.internet.") {
delete(ct.Config, k)
}
}
for k, v := range config {
ct.Config[k] = v
}
err = c.UpdateContainerConfig(router.Name, ct.Writable())
if err != nil {
logf("Failed to configure container '%s': %s", router.Name, err)
return
}
return
}
// Create the container
resp, err := c.Init(router.Name, "local", "internet-router", &[]string{"internet-base"}, config, nil, false)
if err != nil {
logf("Failed to create container '%s': %s", router.Name, err)
return
}
err = c.WaitForSuccess(resp.Operation)
if err != nil {
logf("Failed to create container '%s': %s", router.Name, err)
return
}
// Setup the devices
ct, err := c.ContainerInfo(router.Name)
if err != nil {
logf("Failed to configure container '%s': %s", router.Name, err)
return
}
for k, v := range devices {
ct.Devices[k] = v
}
err = c.UpdateContainerConfig(router.Name, ct.Writable())
if err != nil {
logf("Failed to configure container '%s': %s", router.Name, err)
return
}
}
// Create the containers
batch := 8
batches := len(routers) / batch
remainder := len(routers) % batch
logf("Creating the containers")
current := 0
for i := 0; i < batches; i++ {
for j := 0; j < batch; j++ {
wgBatch.Add(1)
go createContainer(routers[current])
current += 1
}
wgBatch.Wait()
}
for k := 0; k < remainder; k++ {
wgBatch.Add(1)
go createContainer(routers[current])
current += 1
}
wgBatch.Wait()
logf("%d containers created", len(routers))
return nil
}