virtual_network_resource_group: use vnet peering

docs/run_test/platform.rst

@@ -235,14 +235,35 @@ deployment.
   created and the resulting virtual network name will be
   `<virtual_network_name>`.  If `virtual_network_resource_group` is provided,
   an existing virtual network, with the name equal to `virtual_network_name`,
-  will be used.
+  will be used. `virtual_network_resource_group` does not change the behavior for 
+  test resource group creation. Instead, the subnets in the test RGs vnet will be 
+  peered to a subnet within the RG `virtual_network_resource_group`. Peering requires 
+  the subnets and vnet have no address space collisions; a default schema
+  assuming a remote subnet named 'default' with the address space 10.255.255.0/24 is used
+  to allow a large number of test resource groups to be created without collisions.
+  Virtual networks in the test environments will set the default subnet prefix of:
+    10.$(environment_id/256).$(environment_id%256).0/24; any additional subnets will use the prefix
+    192.168.${nic_index}.0/24.
+   This scheme allows for 0xFFFE test environments with up to 256 nics per VM; however, 
+   Azure will likely restrict these numbers to something smaller than the maximum. 
+   LISA does not remove the subnet peerings from the remote vnet in `virtual_network_resource_group`.
+   The total number of test environments will be limited based on the allowed active subnet peerings per vnet.
+   This complex behavior is intended to enable testing without exposing a public IP address;
+   since a VM in `virtual_network_resource_group` will be able to access the test environments
+   via SSH on the private network. This assumes an automated deployment of this orchestrator resource group,
+   VM and virtual network with the expected default values. '`<virtual_network_name>` and `<subnet_prefix>` are respected.
+   Note that usage of `<subnet_prefix>` with `virtual_network_resource_group` will likely result in address space 
+   collisions and failed deployments. Similarly; use of `<resource_group_name>` with this option will likely
+   result in failed test resource deployments.
 * **subnet_prefix**. Specify the desired subnet prefix.  If
   `virtual_network_resource_group` is not provided, a virtual network and
   subnet will be created and the resulting subnets will look like
-  `<subnet_profile>0`, `<subnet_profile>1`, and so on.  If
-  `virtual_network_resource_group` is provided, an existing virtual network and
-  subnet, with the name equal to `subnet_prefix`, will be used.
-* **use_public_address**. True means to connect to the Azure VMs with their
+  `<subnet_profile>0`, `<subnet_profile>1`, and so on.
+   The '`<subnet_prefix>` option will likely conflict with the use of `<virtual_network_resource_group>`.
+   LISA will warn of this configuration but allow it's use, see notes for `<virtual_network_resource_group>`
+   for more details.
+
+  * **use_public_address**. True means to connect to the Azure VMs with their
   public IP addresses.  False means to connect with the private IP addresses.
   If not provided, the connections will default to using the public IP
   addresses.

lisa/microsoft/runbook/azure.yml

@@ -45,14 +45,14 @@ variable:
     value: ""
   - name: virtual_network_name
     value: ""
-  - name: subnet_prefix
-    value: ""
   - name: use_public_address
     value: true
   - name: create_public_address
     value: true
   - name: resource_group_tags
     value: null
+  - name: subnet_prefix
+    value: ""
     # Example usage:
     # resource_group_tags:
     #   Environment: Testing

lisa/microsoft/testsuites/dpdk/dpdkutil.py

@@ -1,3 +1,4 @@
+import ipaddress
 import itertools
 import re
 import time
@@ -259,23 +260,23 @@ def generate_testpmd_multiple_port_command(
     use_service_cores: int = 1,
     set_mtu: int = 0,
 ) -> Dict[DpdkTestResources, str]:
-    # for N senders, make a list of subnets from
-    # 10.0.1.0/24 to 10.0.N.0/24.
-    # these can be arbitrarily picked, each VM has nics on each
-    # subnets, so it doesn't matter which is picked for each VM
-    # as long as the senders are on distinct subnets.
-    subnets = []
-    for i in range(len(senders)):
-        subnets += [f"10.0.{i + 1}.0/24"]
+    # make a list of nics on each non-primary subnet on the receiver
+    # these can be arbitrarily picked, each VM must have nics on each
+    # subnet for the test to run, so as long as we exclude the ssh
+    # interface, it doesn't matter which is picked as long as the mapping
+    # is consistent between senders and receiver.
+    subnets = [
+        subnet for subnet in receiver.node.nics.get_node_subnets(include_primary=False)
+    ]
     sender_nics: Dict[DpdkTestResources, NicInfo] = dict()
     receiver_nics: Dict[str, NicInfo] = dict()
     for i in range(len(senders)):
         # pick one nic per subnet for the senders
         subnet = subnets[i]  # defined above as "10.0.{i + 1}.0/24"
         sender = senders[i]
-        sender_nics[sender] = sender.node.nics.get_nic_by_subnet(subnet)
+        sender_nics[sender] = sender.node.nics.get_nic_by_subnet(str(subnet))
         # and the corresponding nic on the receiver for that subnet.
-        receiver_nics[subnet] = receiver.node.nics.get_nic_by_subnet(subnet)
+        receiver_nics[str(subnet)] = receiver.node.nics.get_nic_by_subnet(str(subnet))
 
     # for MTU test: check that we can fetch the max MTU size for the NIC
     if set_mtu:
@@ -298,13 +299,13 @@ def generate_testpmd_multiple_port_command(
     for i in range(len(senders)):
         # get the sender
         sender = senders[i]
-        sender_subnet = subnets[i]  # defined above as "10.0.{i + 1}.0/24"
+        sender_subnet = subnets[i]  # will be something like 10.X.Y.0/24
         # get the sender nic we picked
         sender_nic = sender_nics[sender]
         # get the subnet for that nic (follows the pattern from before)
 
         # get the corresponding receiver nic for that subnet
-        receiver_nic = receiver_nics[sender_subnet]
+        receiver_nic = receiver_nics[str(sender_subnet)]
         # generate the command for the sender
         snd_cmd = sender.testpmd.generate_testpmd_command(
             [sender_nic],
@@ -320,7 +321,7 @@ def generate_testpmd_multiple_port_command(
         kit_cmd_pairs[sender] = snd_cmd
         # receiver needs multiple ports, so only generate the include.
         receiver_include = receiver.testpmd.generate_testpmd_include(
-            receiver_nics[sender_subnet], i
+            receiver_nics[str(sender_subnet)], i
         )
         # and save it
         receiver_includes += [receiver_include]
@@ -484,8 +485,13 @@ def initialize_node_resources(
     assert_that(len(node.nics)).described_as(
         "Test needs at least 1 NIC on the test node."
     ).is_greater_than_or_equal_to(1)
-
-    test_nic = node.nics.get_nic_by_subnet("10.0.1.0/24")
+    # get a sorted list of subnets for the nics on the node,
+    # excluding the primary nic subnet,
+    # and pick the one on the lowest subnet as the testing default.
+    subnets = sorted(
+        [str(subnet) for subnet in node.nics.get_node_subnets(include_primary=False)]
+    )
+    test_nic = node.nics.get_nic_by_subnet(str(subnets[0]))
 
     # check an assumption that our nics are bound to hv_netvsc
     # at test start.
@@ -869,13 +875,17 @@ def verify_dpdk_mutliple_ports(
         (f"receiver:{external_ips[0]}\nsenders:{external_ips[1]},{external_ips[2]}\n")
     )
     receiver, sender_a, sender_b = environment.nodes.list()
+    subnets = receiver.nics.get_node_subnets(include_primary=False)
+    subnet_a, subnet_b = subnets[:2]  # will be something like 10.X.Y.0/24
+    # note: will assert if there are no nics on corresponding subnets,
+    #       this is good and proper since we can't run without that setup.
     nic_pairings = {
         receiver: [
-            receiver.nics.get_nic_by_subnet("10.0.1.0/24"),
-            receiver.nics.get_nic_by_subnet("10.0.2.0/24"),
+            receiver.nics.get_nic_by_subnet(str(subnet_a)),
+            receiver.nics.get_nic_by_subnet(str(subnet_b)),
         ],
-        sender_a: [sender_a.nics.get_nic_by_subnet("10.0.1.0/24")],
-        sender_b: [sender_b.nics.get_nic_by_subnet("10.0.2.0/24")],
+        sender_a: [sender_a.nics.get_nic_by_subnet(str(subnet_a))],
+        sender_b: [sender_b.nics.get_nic_by_subnet(str(subnet_b))],
     }
     # get test duration variable if set
     # enables long-running tests to shakeQoS and SLB issue
@@ -986,10 +996,12 @@ def ipv4_to_lpm(addr: str) -> str:
 # enable ip forwarding for secondary and tertiary nics in this test.
 # run in parallel to save a bit of time on this net io step.
 def __enable_ip_forwarding(node: Node) -> None:
-    fwd_subnets = [
-        node.nics.get_nic_by_index(nic_index).ip_addr for nic_index in [1, 2]
-    ]
-    for subnet_ip in fwd_subnets:
+    fwd_subnets = node.nics.get_node_subnets(include_primary=False)
+    subnet_nics = [node.nics.get_nic_by_subnet(str(subnet)) for subnet in fwd_subnets]
+    subnet_ips = [nic.ip_addr for nic in subnet_nics]
+
+    for subnet_ip in subnet_ips:
+        node.log.debug(f"Enabling IP forwarding for nic on subnet {subnet_ip}")
         node.features[NetworkInterface].switch_ip_forwarding(
             enable=True, private_ip_addr=subnet_ip
         )
@@ -1115,22 +1127,8 @@ def verify_dpdk_l3fwd_ntttcp_tcp(
     # 3. enjoy the thrill of victory, ship a cloud net applicance.
 
     l3fwd_app_name = "l3fwd"
-    # pick fwd/send/receive nodes based on well known addresses in our subnets
-    forwarder = [
-        node
-        for node in environment.nodes.list()
-        if node.nics.get_primary_nic().ip_addr.endswith("4")
-    ][0]
-    sender = [
-        node
-        for node in environment.nodes.list()
-        if node.nics.get_primary_nic().ip_addr.endswith("5")
-    ][0]
-    receiver = [
-        node
-        for node in environment.nodes.list()
-        if node.nics.get_primary_nic().ip_addr.endswith("6")
-    ][0]
+
+    forwarder, sender, receiver = environment.nodes.list()
 
     if not (
         forwarder.tools[Lscpu].get_architecture() == CpuArchitecture.X64
@@ -1172,15 +1170,22 @@ def verify_dpdk_l3fwd_ntttcp_tcp(
     forwarder.log.debug(f"fwd: {str(forwarder.nics)}")
     receiver.log.debug(f"rcv: {str(receiver.nics)}")
     sender.log.debug(f"snd: {str(sender.nics)}")
+    subnets = forwarder.nics.get_node_subnets(include_primary=False)
+    if len(subnets) != 2:
+        raise SkippedException(
+            "Expected exactly 2 non-primary subnets for this test. "
+            f"Found subnets: {subnets}"
+        )
+    subnet_a, subnet_b = subnets
     subnet_a_nics = {
-        forwarder: forwarder.nics.get_nic_by_subnet("10.0.1.0/24"),
-        sender: sender.nics.get_nic_by_subnet("10.0.1.0/24"),
-        receiver: receiver.nics.get_nic_by_subnet("10.0.1.0/24"),
+        forwarder: forwarder.nics.get_nic_by_subnet(str(subnet_a)),
+        sender: sender.nics.get_nic_by_subnet(str(subnet_a)),
+        receiver: receiver.nics.get_nic_by_subnet(str(subnet_a)),
     }
     subnet_b_nics = {
-        forwarder: forwarder.nics.get_nic_by_subnet("10.0.2.0/24"),
-        receiver: receiver.nics.get_nic_by_subnet("10.0.2.0/24"),
-        sender: sender.nics.get_nic_by_subnet("10.0.2.0/24"),
+        forwarder: forwarder.nics.get_nic_by_subnet(str(subnet_b)),
+        receiver: receiver.nics.get_nic_by_subnet(str(subnet_b)),
+        sender: sender.nics.get_nic_by_subnet(str(subnet_b)),
     }
 
     # We use ntttcp for snd/rcv which will respect the kernel route table.

lisa/nic.py

@@ -8,7 +8,7 @@
 from collections import OrderedDict
 from dataclasses import dataclass
 from pathlib import PurePosixPath
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from assertpy import assert_that
 from retry import retry
@@ -44,6 +44,10 @@ def __init__(
         self.pci_slot = pci_slot
         self.dev_uuid = ""
         self.module_name = ""
+        self.subnet: Optional[
+            Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
+        ] = None
+
         if driver_sysfs_path is None:
             self.driver_sysfs_path = PurePosixPath("")
         else:
@@ -60,6 +64,15 @@ def __str__(self) -> str:
             f"dev_uuid: {self.dev_uuid}\n"
         )
 
+    def get_subnet(self) -> Union[ipaddress.IPv4Network, ipaddress.IPv6Network]:
+        # get the subnet for this nic, assuming a mask of /24
+        # note: if the bicep template changes to assign different masks for subnet_prefix,
+        # this function will need to be updated to
+        if self.subnet:
+            return self.subnet
+        else:
+            raise LisaException(f"No subnet information available for {self.name} ")
+
     @property
     def is_pci_module_enabled(self) -> bool:
         """
@@ -304,6 +317,19 @@ def get_nic_by_subnet(self, subnet: str) -> NicInfo:
                 return nic
         raise LisaException(f"Could not find a nic for requested subnet: {subnet}")
 
+    # get a list of all subnets associated with the nics on this node,
+    # with the option to include the primary nic subnet or not.
+    def get_node_subnets(
+        self, include_primary: bool = True
+    ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
+        return list(
+            [
+                nic.subnet
+                for nic in self.nics.values()
+                if nic.subnet and (include_primary or nic.name != self.default_nic)
+            ]
+        )
+
     def unbind(self, nic: NicInfo) -> None:
         # unbind nic from current driver and return the old sysfs path
         tee = self._node.tools[Tee]
@@ -341,6 +367,7 @@ def load_nics_info(self, nic_name: Optional[str] = None) -> None:
                 nic_entry = self.nics[nic_name]
                 nic_entry.ip_addr = ip_addr
                 nic_entry.mac_addr = mac
+                nic_entry.subnet = nic_info.subnet
                 found_nics.append(nic_name)
 
         if not nic_name: