Harden OpenVMM Azure guest orchestration

lisa/microsoft/runbook/openvmm/openvmm-azure-smoke.yml

@@ -0,0 +1,117 @@
+name: openvmm azure smoke
+extension:
+  - ../../testsuites
+variable:
+  - name: host_admin_username
+    value: lisatest
+  - name: host_admin_password
+    value: ""
+    is_secret: true
+  - name: host_admin_private_key_file
+    value: ""
+    is_secret: true
+  - name: guest_admin_username
+    value: lisatest
+  - name: guest_admin_password
+    value: ""
+    is_secret: true
+  - name: guest_admin_private_key_file
+    value: ""
+    is_secret: true
+  - name: guest_extra_user_data
+    value: ""
+  - name: subscription_id
+    value: ""
+  - name: location
+    value: "westus3"
+  - name: marketplace_image
+    value: ""
+  - name: vm_size
+    value: ""
+  - name: openvmm_binary
+    value: /usr/local/bin/openvmm
+  - name: openvmm_install_path
+    value: /usr/local/bin/openvmm
+  - name: openvmm_installer_repo
+    value: https://github.com/microsoft/openvmm.git
+  - name: openvmm_installer_ref
+    value: ""
+  - name: openvmm_installer_force_install
+    value: false
+  - name: openvmm_host_working_dir
+    value: /var/tmp
+  - name: uefi_firmware_path
+    value: ""
+  - name: uefi_firmware_is_remote_path
+    value: false
+  - name: disk_img_path
+    value: ""
+  - name: disk_img_is_remote_path
+    value: false
+  - name: tap_name
+    value: tap0
+  - name: bridge_name
+    value: ovmbr0
+  - name: tap_host_cidr
+    value: 10.0.0.1/24
+  - name: forwarded_port
+    value: 60022
+notifier:
+  - type: html
+transformer:
+  - type: openvmm_installer
+    phase: environment_connected
+    installer:
+      type: source
+      repo: $(openvmm_installer_repo)
+      ref: $(openvmm_installer_ref)
+      force_install: $(openvmm_installer_force_install)
+      install_path: $(openvmm_install_path)
+platform:
+  - type: azure
+    admin_username: $(host_admin_username)
+    admin_password: $(host_admin_password)
+    admin_private_key_file: $(host_admin_private_key_file)
+    guest_enabled: true
+    guests:
+      - type: openvmm
+        use_parent_capability: false
+        username: $(guest_admin_username)
+        password: $(guest_admin_password)
+        private_key_file: $(guest_admin_private_key_file)
+        cloud_init:
+          extra_user_data: $(guest_extra_user_data)
+        lisa_working_dir: $(openvmm_host_working_dir)
+        openvmm_binary: $(openvmm_binary)
+        boot_mode: uefi
+        capability:
+          core_count: 2
+          memory_mb: 2048
+        uefi:
+          firmware_path: $(uefi_firmware_path)
+          firmware_is_remote_path: $(uefi_firmware_is_remote_path)
+        disk_img: $(disk_img_path)
+        disk_img_is_remote_path: $(disk_img_is_remote_path)
+        serial:
+          mode: file
+        network:
+          mode: tap
+          address_mode: discover
+          tap_name: $(tap_name)
+          bridge_name: $(bridge_name)
+          tap_host_cidr: $(tap_host_cidr)
+          forward_ssh_port: true
+          forwarded_port: $(forwarded_port)
+    azure:
+      subscription_id: $(subscription_id)
+    requirement:
+      azure:
+        marketplace: $(marketplace_image)
+        location: $(location)
+        vm_size: $(vm_size)
+testcase:
+  - criteria:
+      name:
+        - verify_openvmm_guest_boot
+        - verify_openvmm_restart_via_platform
+        - verify_openvmm_stop_start_in_platform

lisa/microsoft/testsuites/openvmm/openvmm.py

@@ -1,60 +1,103 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-from assertpy import assert_that
+from typing import Any
 
-from lisa import Node, SkippedException, TestCaseMetadata, TestSuite, TestSuiteMetadata
+from lisa import (
+    Logger,
+    RemoteNode,
+    SkippedException,
+    TestCaseMetadata,
+    TestSuite,
+    TestSuiteMetadata,
+    simple_requirement,
+)
+from lisa.environment import EnvironmentStatus
 from lisa.features import StartStop
 from lisa.sut_orchestrator.openvmm.node import OpenVmmGuestNode
-from lisa.testsuite import simple_requirement
+from lisa.tools import Uname
 
 
 @TestSuiteMetadata(
     area="openvmm",
     category="functional",
     description="""
-    Smoke coverage for OpenVMM guest provisioning and platform lifecycle.
+    This test suite validates OpenVMM guests running on a prepared L1 host.
     """,
 )
-class OpenVmmSmokeTestSuite(TestSuite):
+class OpenVmmPlatformSuite(TestSuite):
+    def before_case(self, log: Logger, **kwargs: Any) -> None:
+        node = kwargs["node"]
+        if not isinstance(node, OpenVmmGuestNode):
+            raise SkippedException(
+                "This suite only applies to OpenVMM guest nodes. "
+                f"Actual node type: {type(node).__name__}."
+            )
+
     @TestCaseMetadata(
         description="""
-        Validate an OpenVMM guest is provisioned, reachable over SSH, and can
-        execute a simple command after launch.
+        This case validates that an OpenVMM guest is reachable over SSH and that
+        the guest booted successfully.
         """,
-        priority=1,
-        requirement=simple_requirement(supported_features=[StartStop]),
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+        ),
     )
-    def verify_openvmm_provisioning(self, node: Node) -> None:
-        openvmm_node = self._get_openvmm_guest(node)
+    def verify_openvmm_guest_boot(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(f"Connected to OpenVMM guest kernel {kernel_release}")
 
-        result = openvmm_node.execute("echo openvmm-smoke", shell=True)
+    @TestCaseMetadata(
+        description="""
+        This case validates that platform restart keeps the OpenVMM guest
+        reachable after the restart.
+        """,
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+            supported_features=[StartStop],
+        ),
+    )
+    def verify_openvmm_restart_via_platform(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        start_stop = node.features[StartStop]
+        start_stop.restart()
 
-        result.assert_exit_code()
-        assert_that(result.stdout.strip()).is_equal_to("openvmm-smoke")
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(f"OpenVMM guest returned after restart on kernel {kernel_release}")
 
     @TestCaseMetadata(
         description="""
-        Validate the OpenVMM StartStop feature can stop and start a guest while
-        preserving SSH connectivity for subsequent command execution.
+        This case validates that platform stop/start keeps the OpenVMM guest
+        reachable for subsequent command execution.
         """,
-        priority=1,
-        requirement=simple_requirement(supported_features=[StartStop]),
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+            supported_features=[StartStop],
+        ),
     )
-    def verify_openvmm_stop_start_in_platform(self, node: Node) -> None:
-        openvmm_node = self._get_openvmm_guest(node)
-
-        start_stop = openvmm_node.features[StartStop]
+    def verify_openvmm_stop_start_in_platform(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        start_stop = node.features[StartStop]
+        log.info("Stopping OpenVMM guest via platform")
         start_stop.stop(wait=True)
+        log.info("Starting OpenVMM guest via platform")
         start_stop.start(wait=True)
 
-        result = openvmm_node.execute("echo openvmm-recovered", shell=True)
-
-        result.assert_exit_code()
-        assert_that(result.stdout.strip()).is_equal_to("openvmm-recovered")
-
-    def _get_openvmm_guest(self, node: Node) -> OpenVmmGuestNode:
-        if not isinstance(node, OpenVmmGuestNode):
-            raise SkippedException("This suite only applies to OpenVMM guest nodes.")
-
-        return node
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(
+            f"OpenVMM guest returned after platform stop/start on kernel "
+            f"{kernel_release}"
+        )

lisa/node.py

@@ -355,12 +355,30 @@ def execute_async(
         )
 
     def cleanup(self) -> None:
+        for guest in self.guests:
+            try:
+                guest.cleanup()
+            except Exception:
+                self.log.exception(
+                    "failed to clean up guest "
+                    f"'{guest.name or guest.index}' while cleaning node "
+                    f"'{self.name}'. Continuing parent cleanup."
+                )
         self.log.debug("cleaning up...")
         if hasattr(self, "_log_handler") and self._log_handler:
             remove_handler(self._log_handler, self.log)
             self._log_handler.close()
 
     def close(self) -> None:
+        for guest in self.guests:
+            try:
+                guest.close()
+            except Exception:
+                self.log.exception(
+                    "failed to close guest "
+                    f"'{guest.name or guest.index}' while closing node "
+                    f"'{self.name}'. Continuing parent close."
+                )
         self.log.debug("closing node connection...")
         if self._shell:
             self._shell.close()
@@ -553,7 +571,12 @@ def mark_dirty(self) -> None:
         self._is_dirty = True
 
     def test_connection(self) -> bool:
-        assert self._shell
+        if not self._shell:
+            self.log.debug(
+                f"connection test failed for node '{self.name}' because its "
+                "shell is not initialized"
+            )
+            return False
         if not self._shell.is_remote:
             return True
         self.log.debug("testing connection...")

lisa/runners/lisa_runner.py

@@ -200,6 +200,13 @@ def _dispatch_test_result(
         # run on deployed environment
         can_run_results = [x for x in can_run_results if x.can_run]
         if environment.status == EnvironmentStatus.Deployed and can_run_results:
+            if self._guest_enabled:
+                return self._generate_task(
+                    task_method=self._initialize_environment_task,
+                    environment=environment,
+                    test_results=can_run_results[:1],
+                )
+
             selected_test_results = self._get_test_result_to_run(
                 test_results=test_results, environment=environment
             )
@@ -338,6 +345,9 @@ def _initialize_environment_task(
                 phase=constants.TRANSFORMER_PHASE_ENVIRONMENT_CONNECTED,
                 environment=environment,
             )
+            if self._guest_enabled:
+                guest_environment = environment.get_guest_environment()
+                guest_environment.nodes.initialize()
         except Exception as e:
             self._attach_failed_environment_to_result(
                 environment=environment,
@@ -636,8 +646,10 @@ def _get_runnable_test_results(
             )
             and (
                 environment_status is None
-                or x.runtime_data.metadata.requirement.environment_status
-                == environment_status
+                or self._matches_environment_status(
+                    x.runtime_data.metadata.requirement.environment_status,
+                    environment_status,
+                )
             )
         ]
         if environment:
@@ -685,6 +697,23 @@ def _get_runnable_test_results(
         results = self._sort_test_results(results)
         return results
 
+    def _matches_environment_status(
+        self,
+        requirement_status: EnvironmentStatus,
+        actual_status: EnvironmentStatus,
+    ) -> bool:
+        if requirement_status == actual_status:
+            return True
+
+        if (
+            self._guest_enabled
+            and actual_status == EnvironmentStatus.Connected
+            and requirement_status == EnvironmentStatus.Deployed
+        ):
+            return True
+
+        return False
+
     def _get_test_result_to_run(
         self, test_results: List[TestResult], environment: Environment
     ) -> List[TestResult]:

lisa/sut_orchestrator/azure/platform_.py

@@ -802,7 +802,9 @@ def _get_node_information(self, node: Node) -> Dict[str, str]:  # noqa: C901
 
             # Guest nodes (like WslContainerNode) don't have features attribute
             # Skip security profile collection for guest nodes
-            if hasattr(node, "features"):
+            if hasattr(node, "features") and node.features.is_supported(
+                SecurityProfile
+            ):
                 security_profile = node.features[SecurityProfile].get_settings()
             else:
                 security_profile = None
@@ -968,7 +970,11 @@ def _get_kernel_version(self, node: Node) -> str:
             linux_information = node.tools[Uname].get_linux_information()
             result = linux_information.kernel_version_raw
         elif not node.is_connected or node.is_posix:
-            if not result and hasattr(node, ATTRIBUTE_FEATURES):
+            if (
+                not result
+                and hasattr(node, ATTRIBUTE_FEATURES)
+                and node.features.is_supported(features.SerialConsole)
+            ):
                 # try to get kernel version in Azure. use it, when uname doesn't work
                 node.log.debug("detecting kernel version from serial log...")
                 serial_console = node.features[features.SerialConsole]
@@ -1004,7 +1010,11 @@ def _get_wala_version(self, node: Node) -> str:
             node.log.debug(f"error on run waagent: {e}")
 
         if not node.is_connected or node.is_posix:
-            if not result and hasattr(node, ATTRIBUTE_FEATURES):
+            if (
+                not result
+                and hasattr(node, ATTRIBUTE_FEATURES)
+                and node.features.is_supported(features.SerialConsole)
+            ):
                 node.log.debug("detecting wala agent version from serial log...")
                 serial_console = node.features[features.SerialConsole]
                 result = serial_console.get_matched_str(WALA_VERSION_PATTERN)