DPDK: Add symmetric_mp hotplug test

lisa/microsoft/testsuites/dpdk/dpdksuite.py

@@ -134,6 +134,58 @@ def verify_dpdk_symmetric_mp(
     ) -> None:
         run_dpdk_symmetric_mp(node, log, variables)
 
+    @TestCaseMetadata(
+        description="""
+            netvsc pmd version.
+            This test case checks dpdk symmetic mp app, plus an sriov hotplug.
+            More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501
+        """,
+        priority=2,
+        requirement=simple_requirement(
+            min_core_count=8,
+            min_nic_count=3,
+            network_interface=Sriov(),
+            unsupported_features=[Gpu, Infiniband],
+        ),
+        timeout=600,
+    )
+    def verify_dpdk_symmetric_mp_hotplug(
+        self,
+        node: Node,
+        log: Logger,
+        variables: Dict[str, Any],
+        result: TestResult,
+    ) -> None:
+        run_dpdk_symmetric_mp(
+            node, log, variables, trigger_hotplug=True, hotplug_times=1
+        )
+
+    @TestCaseMetadata(
+        description="""
+            netvsc pmd version.
+            This test case checks dpdk symmetic mp app, plus an sriov hotplug.
+            More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501
+        """,
+        priority=4,
+        requirement=simple_requirement(
+            min_core_count=8,
+            min_nic_count=3,
+            network_interface=Sriov(),
+            unsupported_features=[Gpu, Infiniband],
+        ),
+        timeout=6000,
+    )
+    def stress_dpdk_symmetric_mp_hotplug(
+        self,
+        node: Node,
+        log: Logger,
+        variables: Dict[str, Any],
+        result: TestResult,
+    ) -> None:
+        run_dpdk_symmetric_mp(
+            node, log, variables, trigger_hotplug=True, hotplug_times=40
+        )
+
     @TestCaseMetadata(
         description="""
             netvsc pmd version with 1GiB hugepages

lisa/microsoft/testsuites/dpdk/dpdkutil.py

@@ -74,6 +74,7 @@
 )
 from lisa.tools.hugepages import HugePageSize
 from lisa.tools.lscpu import CpuArchitecture
+from lisa.util import sleep
 from lisa.util.constants import DEVICE_TYPE_SRIOV, SIGINT
 from lisa.util.parallel import TaskManager, run_in_parallel, run_in_parallel_async
 
@@ -477,7 +478,7 @@ def initialize_node_resources(
     hugepages = node.tools[Hugepages]
     numa_nodes = node.tools[Lscpu].get_numa_node_count()
     try:
-        hugepages.init_hugepages(hugepage_size, minimum_gb=4 * numa_nodes)
+        hugepages.init_hugepages(hugepage_size, minimum_gb=8 * numa_nodes)
     except NotEnoughMemoryException as err:
         raise SkippedException(err)
 
@@ -1718,6 +1719,8 @@ def run_dpdk_symmetric_mp(
     - count packets received on tx/rx side of each process and port
 
     """
+
+    test_timeout = 120 + (60 * hotplug_times if trigger_hotplug else 35)
     # setup and unwrap the resources for this test
     # get a list of the upper non-primary nics and select two of them
     test_nics = [
@@ -1790,9 +1793,9 @@ def run_dpdk_symmetric_mp(
             f"{str(symmetric_mp_path)} -l 1 --proc-type auto "
             f"{symmetric_mp_args} --proc-id 0"
         ),
-        timeout=660,
+        timeout=test_timeout,
         signal=SIGINT,
-        kill_timeout=30,
+        kill_timeout=test_timeout + 5,
     )
 
     # wait for it to start
@@ -1804,9 +1807,9 @@ def run_dpdk_symmetric_mp(
             f"{str(symmetric_mp_path)} -l 2 --proc-type secondary "
             f"{symmetric_mp_args} --proc-id 1"
         ),
-        timeout=600,
+        timeout=test_timeout,
         signal=SIGINT,
-        kill_timeout=35,
+        kill_timeout=test_timeout + 5,
     )
     secondary.wait_output("APP: Finished Process Init", timeout=20)
 
@@ -1842,7 +1845,7 @@ def run_dpdk_symmetric_mp(
                 "Device notification type=1",  # RTE_DEV_EVENT_REMOVE
                 delta_only=True,
             )  # relying on compiler defaults here, not great.
-
+            sleep(1)
             # turn SRIOV on
             node.features[NetworkInterface].switch_sriov(
                 enable=True, wait=False, reset_connections=False
@@ -1867,6 +1870,8 @@ def run_dpdk_symmetric_mp(
             )
             # expect additional pings for each post-hotplug instance
             expected_pings += 100
+            # sleep for a moment to avoid api throttling
+            sleep(1)
 
     ping.ping_async(
         target=test_nics[0].ip_addr,

lisa/sut_orchestrator/azure/features.py

@@ -95,6 +95,7 @@
     generate_random_chars,
     get_matched_str,
     set_filtered_fields,
+    sleep,
 )
 
 if TYPE_CHECKING:
@@ -1008,6 +1009,7 @@ def switch_sriov(
                     f"now set its status into [{enable}]."
                 )
                 updated_nic.enable_accelerated_networking = enable
+
                 network_client.network_interfaces.begin_create_or_update(
                     self._resource_group_name, updated_nic.name, updated_nic
                 )
@@ -1019,6 +1021,7 @@ def switch_sriov(
                     f"networking into status [{enable}]"
                 ).is_equal_to(enable)
                 status_changed = True
+                sleep(0.5)
 
         # wait settings effective
         if wait and status_changed: