From abd9081afb86944fee5c6348b6995a90ee49f89d Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Mon, 23 Feb 2026 14:46:47 -0800 Subject: [PATCH 01/19] platform: use named subnets based on address prefix --- docs/run_test/platform.rst | 7 ---- .../sut_orchestrator/azure/arm_template.bicep | 35 +++++++++++++------ lisa/sut_orchestrator/azure/common.py | 2 -- .../azure/nested_nodes_nics.bicep | 9 ++--- .../azure/nested_shared_vnet_subnets.bicep | 10 ++++++ lisa/sut_orchestrator/azure/platform_.py | 6 ---- 6 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep diff --git a/docs/run_test/platform.rst b/docs/run_test/platform.rst index 81809c24f2..ee4f80ce79 100644 --- a/docs/run_test/platform.rst +++ b/docs/run_test/platform.rst @@ -192,7 +192,6 @@ deployment. azure: virtual_network_resource_group: $(virtual_network_resource_group) virtual_network_name: $(virtual_network_name) - subnet_prefix: $(subnet_prefix) use_public_address: "" create_public_address: "" use_ipv6: "" @@ -236,12 +235,6 @@ deployment. ``. If `virtual_network_resource_group` is provided, an existing virtual network, with the name equal to `virtual_network_name`, will be used. -* **subnet_prefix**. Specify the desired subnet prefix. If - `virtual_network_resource_group` is not provided, a virtual network and - subnet will be created and the resulting subnets will look like - `0`, `1`, and so on. If - `virtual_network_resource_group` is provided, an existing virtual network and - subnet, with the name equal to `subnet_prefix`, will be used. * **use_public_address**. True means to connect to the Azure VMs with their public IP addresses. False means to connect with the private IP addresses. If not provided, the connections will default to using the public IP diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index 34ce7bd676..976b68ec78 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -22,6 +22,9 @@ param shared_resource_group_name string @description('created subnet count') param subnet_count int +@description('index of the test resource group for shared vnet subnet mapping') +param resource_group_index int + @description('options for availability sets, zones, and VMSS') param availability_options object @@ -31,9 +34,6 @@ param virtual_network_resource_group string @description('the name of vnet') param virtual_network_name string -@description('the prefix of the subnets') -param subnet_prefix string - @description('tags of virtual machine') param vm_tags object @@ -64,10 +64,14 @@ param source_address_prefixes array @description('Generate public IP address for each node') param create_public_address bool -var vnet_id = virtual_network_name_resource.id +var vnet_id = use_existing_vnet +? resourceId(virtual_network_resource_group, 'Microsoft.Network/virtualNetworks', virtual_network_name) +: virtual_network_name_resource.id var node_count = length(nodes) var availability_set_name_value = 'lisa-availabilitySet' -var existing_subnet_ref = (empty(virtual_network_resource_group) ? '' : resourceId(virtual_network_resource_group, 'Microsoft.Network/virtualNetworks/subnets', virtual_network_name, subnet_prefix)) +var wrapped_resource_group_index = resource_group_index % 256 +var use_existing_vnet = !empty(virtual_network_resource_group) +var shared_subnet_names = [for nic_index in range(0, subnet_count): '10.${wrapped_resource_group_index}.${nic_index}.0'] var availability_set_tags = availability_options.availability_set_tags var availability_set_properties = availability_options.availability_set_properties var availability_zones = availability_options.availability_zones @@ -231,8 +235,7 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): nic_count: nodes[i].nic_count location: location vnet_id: vnet_id - subnet_prefix: subnet_prefix - existing_subnet_ref: existing_subnet_ref + resource_group_index: wrapped_resource_group_index enable_sriov: nodes[i].enable_sriov tags: tags use_ipv6: use_ipv6 @@ -241,9 +244,19 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): dependsOn: [ nodes_public_ip[i] nodes_public_ip_ipv6[i] + shared_vnet_subnets ] }] +module shared_vnet_subnets './nested_shared_vnet_subnets.bicep' = if (use_existing_vnet) { + name: 'shared-vnet-subnets' + scope: resourceGroup(virtual_network_resource_group) + params: { + virtual_network_name: virtual_network_name + subnet_names: shared_subnet_names + } +} + resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-05-01' = if (empty(virtual_network_resource_group)) { name: virtual_network_name tags: tags @@ -251,16 +264,16 @@ resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-0 properties: { addressSpace: { addressPrefixes: concat( - ['10.0.0.0/16'], + ['10.0.0.0/8'], use_ipv6 ? ['2001:db8::/32'] : [] ) } subnets: [for j in range(0, subnet_count): { - name: '${subnet_prefix}${j}' + name: '10.${resource_group_index}.${j}.0/24' properties: { addressPrefixes: concat( - ['10.0.${j}.0/24'], - use_ipv6 ? ['2001:db8:${j}::/64'] : [] + ['10.${resource_group_index}.${j}.0/24'], + use_ipv6 ? ['2001:db8:${resource_group_index}:${j}::/64'] : [] ) defaultOutboundAccess: enable_vm_nat networkSecurityGroup: { diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index 2675b95e63..6f0ae4de24 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -112,7 +112,6 @@ AZURE_SHARED_RG_NAME = "lisa_shared_resource" AZURE_VIRTUAL_NETWORK_NAME = "lisa-virtualNetwork" -AZURE_SUBNET_PREFIX = "lisa-subnet-" NIC_NAME_PATTERN = re.compile(r"Microsoft.Network/networkInterfaces/(.*)", re.M) PATTERN_PUBLIC_IP_NAME = re.compile( @@ -1257,7 +1256,6 @@ class AzureArmParameter: virtual_network_resource_group: str = "" virtual_network_name: str = AZURE_VIRTUAL_NETWORK_NAME - subnet_prefix: str = AZURE_SUBNET_PREFIX is_ultradisk: bool = False is_data_disk_with_vhd: bool = False use_ipv6: bool = False diff --git a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep index 8f283ca78b..6b5543c5fe 100644 --- a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep +++ b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep @@ -2,8 +2,7 @@ param vmName string param nic_count int param location string param vnet_id string -param subnet_prefix string -param existing_subnet_ref string +param resource_group_index int param enable_sriov bool param tags object param use_ipv6 bool @@ -16,6 +15,8 @@ func getPublicIpAddress(vmName string, publicIpName string) object => { var publicIpAddress = getPublicIpAddress(vmName, '${vmName}-public-ip') var publicIpAddressV6 = getPublicIpAddress(vmName, '${vmName}-public-ipv6') +func getSubnetName(resource_group_index int, nic_index int) string => '10.${resource_group_index}.${nic_index}.0' + resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in range(0, nic_count): { name: '${vmName}-nic-${i}' location: location @@ -29,7 +30,7 @@ resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in ra privateIPAddressVersion: 'IPv4' publicIPAddress: ((0 == i && create_public_address) ? publicIpAddress : null) subnet: { - id: ((!empty(existing_subnet_ref)) ? existing_subnet_ref : '${vnet_id}/subnets/${subnet_prefix}${i}') + id: '${vnet_id}/subnets/${getSubnetName(resource_group_index, i)}' } privateIPAllocationMethod: 'Dynamic' } @@ -42,7 +43,7 @@ resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in ra privateIPAddressVersion: 'IPv6' publicIPAddress: ((0 == i && create_public_address) ? publicIpAddressV6 : null) subnet: { - id: ((!empty(existing_subnet_ref)) ? existing_subnet_ref : '${vnet_id}/subnets/${subnet_prefix}${i}') + id: '${vnet_id}/subnets/${getSubnetName(resource_group_index, i)}' } privateIPAllocationMethod: 'Dynamic' } diff --git a/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep b/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep new file mode 100644 index 0000000000..68e5061e32 --- /dev/null +++ b/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep @@ -0,0 +1,10 @@ +param virtual_network_name string +param subnet_names array + +resource shared_vnet_subnets 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = [for subnet_name in subnet_names: { + name: '${virtual_network_name}/${subnet_name}' + properties: { + addressPrefix: '${subnet_name}/24' + } +} +] diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index ab01c8342c..fb53f29526 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -103,7 +103,6 @@ from . import features from .common import ( AZURE_SHARED_RG_NAME, - AZURE_SUBNET_PREFIX, AZURE_VIRTUAL_NETWORK_NAME, SAS_URL_PATTERN, AzureArmParameter, @@ -322,7 +321,6 @@ class AzurePlatformSchema: virtual_network_resource_group: str = field(default="") virtual_network_name: str = field(default=AZURE_VIRTUAL_NETWORK_NAME) - subnet_prefix: str = field(default=AZURE_SUBNET_PREFIX) # Provisioning error causes by waagent is not ready or other reasons. In # smoke test, it can verify some points also. Other tests should use the @@ -375,7 +373,6 @@ def __post_init__(self, *args: Any, **kwargs: Any) -> None: "log_level", "virtual_network_resource_group", "virtual_network_name", - "subnet_prefix", "use_public_address", "use_ipv6", "enable_vm_nat", @@ -1216,9 +1213,6 @@ def _create_deployment_parameters( arm_parameters.virtual_network_resource_group = ( self._azure_runbook.virtual_network_resource_group ) - arm_parameters.subnet_prefix = ( - self._azure_runbook.subnet_prefix or AZURE_SUBNET_PREFIX - ) arm_parameters.virtual_network_name = ( self._azure_runbook.virtual_network_name or AZURE_VIRTUAL_NETWORK_NAME ) From d365c10197ca6ca9e956ab6eedb0f88cffc93503 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Mon, 23 Feb 2026 14:51:41 -0800 Subject: [PATCH 02/19] update autogen arm template --- .../azure/autogen_arm_template.json | 125 ++++++++++++++---- 1 file changed, 98 insertions(+), 27 deletions(-) diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index 52e1e79bb9..de9bb062bf 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.36.177.2456", - "templateHash": "18099469189437246926" + "version": "0.40.2.10011", + "templateHash": "16491672364169834990" } }, "functions": [ @@ -463,6 +463,12 @@ "description": "created subnet count" } }, + "resource_group_index": { + "type": "int", + "metadata": { + "description": "index of the test resource group for shared vnet subnet mapping" + } + }, "availability_options": { "type": "object", "metadata": { @@ -481,12 +487,6 @@ "description": "the name of vnet" } }, - "subnet_prefix": { - "type": "string", - "metadata": { - "description": "the prefix of the subnets" - } - }, "vm_tags": { "type": "object", "metadata": { @@ -553,6 +553,11 @@ }, "variables": { "copy": [ + { + "name": "shared_subnet_names", + "count": "[length(range(0, parameters('subnet_count')))]", + "input": "[format('10.{0}.{1}.0', variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')])]" + }, { "name": "ip_tags", "count": "[length(objectKeys(parameters('ip_service_tags')))]", @@ -562,10 +567,11 @@ } } ], - "vnet_id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtual_network_name'))]", + "vnet_id": "[if(variables('use_existing_vnet'), resourceId(parameters('virtual_network_resource_group'), 'Microsoft.Network/virtualNetworks', parameters('virtual_network_name')), resourceId('Microsoft.Network/virtualNetworks', parameters('virtual_network_name')))]", "node_count": "[length(parameters('nodes'))]", "availability_set_name_value": "lisa-availabilitySet", - "existing_subnet_ref": "[if(empty(parameters('virtual_network_resource_group')), '', resourceId(parameters('virtual_network_resource_group'), 'Microsoft.Network/virtualNetworks/subnets', parameters('virtual_network_name'), parameters('subnet_prefix')))]", + "wrapped_resource_group_index": "[mod(parameters('resource_group_index'), 256)]", + "use_existing_vnet": "[not(empty(parameters('virtual_network_resource_group')))]", "availability_set_tags": "[parameters('availability_options').availability_set_tags]", "availability_set_properties": "[parameters('availability_options').availability_set_properties]", "availability_zones": "[parameters('availability_options').availability_zones]", @@ -590,9 +596,9 @@ "name": "subnets", "count": "[length(range(0, parameters('subnet_count')))]", "input": { - "name": "[format('{0}{1}', parameters('subnet_prefix'), range(0, parameters('subnet_count'))[copyIndex('subnets')])]", + "name": "[format('10.{0}.{1}.0/24', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])]", "properties": { - "addressPrefixes": "[concat(createArray(format('10.0.{0}.0/24', range(0, parameters('subnet_count'))[copyIndex('subnets')])), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}::/64', range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", + "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", "defaultOutboundAccess": "[parameters('enable_vm_nat')]", "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('{0}-nsg', toLower(parameters('virtual_network_name'))))]" @@ -602,7 +608,7 @@ } ], "addressSpace": { - "addressPrefixes": "[concat(createArray('10.0.0.0/16'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray()))]" + "addressPrefixes": "[concat(createArray('10.0.0.0/8'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray()))]" } }, "dependsOn": [ @@ -883,7 +889,7 @@ "count": "[length(range(0, variables('node_count')))]" }, "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", + "apiVersion": "2025-04-01", "name": "[format('{0}-nics', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name)]", "properties": { "expressionEvaluationOptions": { @@ -903,11 +909,8 @@ "vnet_id": { "value": "[variables('vnet_id')]" }, - "subnet_prefix": { - "value": "[parameters('subnet_prefix')]" - }, - "existing_subnet_ref": { - "value": "[variables('existing_subnet_ref')]" + "resource_group_index": { + "value": "[variables('wrapped_resource_group_index')]" }, "enable_sriov": { "value": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].enable_sriov]" @@ -929,8 +932,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.36.177.2456", - "templateHash": "15654609969338604205" + "version": "0.40.2.10011", + "templateHash": "1909517540330820286" } }, "functions": [ @@ -954,6 +957,22 @@ "id": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpName'))]" } } + }, + "getSubnetName": { + "parameters": [ + { + "type": "int", + "name": "resource_group_index" + }, + { + "type": "int", + "name": "nic_index" + } + ], + "output": { + "type": "string", + "value": "[format('10.{0}.{1}.0', parameters('resource_group_index'), parameters('nic_index'))]" + } } } } @@ -971,11 +990,8 @@ "vnet_id": { "type": "string" }, - "subnet_prefix": { - "type": "string" - }, - "existing_subnet_ref": { - "type": "string" + "resource_group_index": { + "type": "int" }, "enable_sriov": { "type": "bool" @@ -1006,7 +1022,7 @@ "location": "[parameters('location')]", "tags": "[parameters('tags')]", "properties": { - "ipConfigurations": "[concat(createArray(createObject('name', 'IPv4Config', 'properties', createObject('privateIPAddressVersion', 'IPv4', 'publicIPAddress', if(and(equals(0, range(0, parameters('nic_count'))[copyIndex()]), parameters('create_public_address')), variables('publicIpAddress'), null()), 'subnet', createObject('id', if(not(empty(parameters('existing_subnet_ref'))), parameters('existing_subnet_ref'), format('{0}/subnets/{1}{2}', parameters('vnet_id'), parameters('subnet_prefix'), range(0, parameters('nic_count'))[copyIndex()]))), 'privateIPAllocationMethod', 'Dynamic'))), if(parameters('use_ipv6'), createArray(createObject('name', 'IPv6Config', 'properties', createObject('privateIPAddressVersion', 'IPv6', 'publicIPAddress', if(and(equals(0, range(0, parameters('nic_count'))[copyIndex()]), parameters('create_public_address')), variables('publicIpAddressV6'), null()), 'subnet', createObject('id', if(not(empty(parameters('existing_subnet_ref'))), parameters('existing_subnet_ref'), format('{0}/subnets/{1}{2}', parameters('vnet_id'), parameters('subnet_prefix'), range(0, parameters('nic_count'))[copyIndex()]))), 'privateIPAllocationMethod', 'Dynamic'))), createArray()))]", + "ipConfigurations": "[concat(createArray(createObject('name', 'IPv4Config', 'properties', createObject('privateIPAddressVersion', 'IPv4', 'publicIPAddress', if(and(equals(0, range(0, parameters('nic_count'))[copyIndex()]), parameters('create_public_address')), variables('publicIpAddress'), null()), 'subnet', createObject('id', format('{0}/subnets/{1}', parameters('vnet_id'), __bicep.getSubnetName(parameters('resource_group_index'), range(0, parameters('nic_count'))[copyIndex()]))), 'privateIPAllocationMethod', 'Dynamic'))), if(parameters('use_ipv6'), createArray(createObject('name', 'IPv6Config', 'properties', createObject('privateIPAddressVersion', 'IPv6', 'publicIPAddress', if(and(equals(0, range(0, parameters('nic_count'))[copyIndex()]), parameters('create_public_address')), variables('publicIpAddressV6'), null()), 'subnet', createObject('id', format('{0}/subnets/{1}', parameters('vnet_id'), __bicep.getSubnetName(parameters('resource_group_index'), range(0, parameters('nic_count'))[copyIndex()]))), 'privateIPAllocationMethod', 'Dynamic'))), createArray()))]", "enableAcceleratedNetworking": "[parameters('enable_sriov')]" } } @@ -1016,8 +1032,63 @@ "dependsOn": [ "[format('nodes_public_ip[{0}]', range(0, variables('node_count'))[copyIndex()])]", "[format('nodes_public_ip_ipv6[{0}]', range(0, variables('node_count'))[copyIndex()])]", + "shared_vnet_subnets", "virtual_network_name_resource" ] + }, + "shared_vnet_subnets": { + "condition": "[variables('use_existing_vnet')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "shared-vnet-subnets", + "resourceGroup": "[parameters('virtual_network_resource_group')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "virtual_network_name": { + "value": "[parameters('virtual_network_name')]" + }, + "subnet_names": { + "value": "[variables('shared_subnet_names')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.40.2.10011", + "templateHash": "1894805062932954870" + } + }, + "parameters": { + "virtual_network_name": { + "type": "string" + }, + "subnet_names": { + "type": "array" + } + }, + "resources": [ + { + "copy": { + "name": "shared_vnet_subnets", + "count": "[length(parameters('subnet_names'))]" + }, + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2024-05-01", + "name": "[format('{0}/{1}', parameters('virtual_network_name'), parameters('subnet_names')[copyIndex()])]", + "properties": { + "addressPrefix": "[format('{0}/24', parameters('subnet_names')[copyIndex()])]" + } + } + ] + } + } } } } \ No newline at end of file From 7fa240df3fce74f7d7f937e1cbf7eb9322a83efa Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Mon, 23 Feb 2026 14:58:47 -0800 Subject: [PATCH 03/19] update nested_node_nics to force 10.0.0.0 for nic0 on each vm --- lisa/sut_orchestrator/azure/arm_template.bicep | 6 +++--- lisa/sut_orchestrator/azure/autogen_arm_template.json | 10 +++++----- lisa/sut_orchestrator/azure/common.py | 1 + lisa/sut_orchestrator/azure/nested_nodes_nics.bicep | 2 +- lisa/sut_orchestrator/azure/platform_.py | 2 ++ 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index 976b68ec78..be07a3c7fa 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -269,11 +269,11 @@ resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-0 ) } subnets: [for j in range(0, subnet_count): { - name: '10.${resource_group_index}.${j}.0/24' + name: j == 0 ? '10.0.0.0' : '10.${resource_group_index}.${j}.0' properties: { addressPrefixes: concat( - ['10.${resource_group_index}.${j}.0/24'], - use_ipv6 ? ['2001:db8:${resource_group_index}:${j}::/64'] : [] + ['10.${j ==0 ? '0' : resource_group_index}.${j}.0/24'], + use_ipv6 ? ['2001:db8:${j == 0 ? j : resource_group_index}:${j}::/64'] : [] ) defaultOutboundAccess: enable_vm_nat networkSecurityGroup: { diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index de9bb062bf..e041c4499f 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.40.2.10011", - "templateHash": "16491672364169834990" + "templateHash": "6973919806372738793" } }, "functions": [ @@ -596,9 +596,9 @@ "name": "subnets", "count": "[length(range(0, parameters('subnet_count')))]", "input": { - "name": "[format('10.{0}.{1}.0/24', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])]", + "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '10.0.0.0', format('10.{0}.{1}.0', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", "properties": { - "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", + "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '0', parameters('resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), range(0, parameters('subnet_count'))[copyIndex('subnets')], parameters('resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", "defaultOutboundAccess": "[parameters('enable_vm_nat')]", "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('{0}-nsg', toLower(parameters('virtual_network_name'))))]" @@ -933,7 +933,7 @@ "_generator": { "name": "bicep", "version": "0.40.2.10011", - "templateHash": "1909517540330820286" + "templateHash": "18100026266609217689" } }, "functions": [ @@ -971,7 +971,7 @@ ], "output": { "type": "string", - "value": "[format('10.{0}.{1}.0', parameters('resource_group_index'), parameters('nic_index'))]" + "value": "[if(equals(parameters('nic_index'), 0), '10.0.0.0', format('10.{0}.{1}.0', parameters('resource_group_index'), parameters('nic_index')))]" } } } diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index 6f0ae4de24..4dfdff98e4 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -1262,6 +1262,7 @@ class AzureArmParameter: enable_vm_nat: bool = False create_public_address: bool = True source_address_prefixes: List[str] = field(default_factory=list) + resource_group_index: int = 0 def __post_init__(self, *args: Any, **kwargs: Any) -> None: add_secret(self.admin_username, PATTERN_HEADTAIL) diff --git a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep index 6b5543c5fe..9910116e49 100644 --- a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep +++ b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep @@ -15,7 +15,7 @@ func getPublicIpAddress(vmName string, publicIpName string) object => { var publicIpAddress = getPublicIpAddress(vmName, '${vmName}-public-ip') var publicIpAddressV6 = getPublicIpAddress(vmName, '${vmName}-public-ipv6') -func getSubnetName(resource_group_index int, nic_index int) string => '10.${resource_group_index}.${nic_index}.0' +func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : '10.${resource_group_index}.${nic_index}.0' resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in range(0, nic_count): { name: '${vmName}-nic-${i}' diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index fb53f29526..95292fcd48 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -605,6 +605,7 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: assert self._azure_runbook environment_context = get_environment_context(environment=environment) + if self._azure_runbook.resource_group_name: resource_group_name = self._azure_runbook.resource_group_name else: @@ -1217,6 +1218,7 @@ def _create_deployment_parameters( self._azure_runbook.virtual_network_name or AZURE_VIRTUAL_NETWORK_NAME ) arm_parameters.use_ipv6 = self._azure_runbook.use_ipv6 + arm_parameters.resource_group_index = int(environment.id) is_windows: bool = False arm_parameters.admin_username = self.runbook.admin_username From 0afef175b40de8e09af0d4a735e41ee137120d20 Mon Sep 17 00:00:00 2001 From: "Matthew G. McGovern" Date: Tue, 24 Feb 2026 09:53:09 -0800 Subject: [PATCH 04/19] dpdk: update subnet selection to remove hardcoded convention assumptions --- lisa/microsoft/testsuites/dpdk/dpdkutil.py | 91 ++++++++++++---------- lisa/nic.py | 28 ++++++- lisa/tools/ip.py | 8 +- 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/lisa/microsoft/testsuites/dpdk/dpdkutil.py b/lisa/microsoft/testsuites/dpdk/dpdkutil.py index 0274be7288..7db967ff55 100644 --- a/lisa/microsoft/testsuites/dpdk/dpdkutil.py +++ b/lisa/microsoft/testsuites/dpdk/dpdkutil.py @@ -7,6 +7,7 @@ from functools import partial from pathlib import PurePath from typing import Any, Dict, List, Optional, Tuple, Union +import ipaddress from assertpy import assert_that, fail from microsoft.testsuites.dpdk.common import ( @@ -259,23 +260,23 @@ def generate_testpmd_multiple_port_command( use_service_cores: int = 1, set_mtu: int = 0, ) -> Dict[DpdkTestResources, str]: - # for N senders, make a list of subnets from - # 10.0.1.0/24 to 10.0.N.0/24. - # these can be arbitrarily picked, each VM has nics on each - # subnets, so it doesn't matter which is picked for each VM - # as long as the senders are on distinct subnets. - subnets = [] - for i in range(len(senders)): - subnets += [f"10.0.{i + 1}.0/24"] + # make a list of nics on each non-primary subnet on the receiver + # these can be arbitrarily picked, each VM must have nics on each + # subnet for the test to run, so as long as we exclude the ssh + # interface, it doesn't matter which is picked as long as the mapping + # is consistent between senders and receiver. + subnets = [ + subnet for subnet in receiver.node.nics.get_node_subnets(include_primary=False) + ] sender_nics: Dict[DpdkTestResources, NicInfo] = dict() receiver_nics: Dict[str, NicInfo] = dict() for i in range(len(senders)): # pick one nic per subnet for the senders subnet = subnets[i] # defined above as "10.0.{i + 1}.0/24" sender = senders[i] - sender_nics[sender] = sender.node.nics.get_nic_by_subnet(subnet) + sender_nics[sender] = sender.node.nics.get_nic_by_subnet(str(subnet)) # and the corresponding nic on the receiver for that subnet. - receiver_nics[subnet] = receiver.node.nics.get_nic_by_subnet(subnet) + receiver_nics[subnet] = receiver.node.nics.get_nic_by_subnet(str(subnet)) # for MTU test: check that we can fetch the max MTU size for the NIC if set_mtu: @@ -298,7 +299,7 @@ def generate_testpmd_multiple_port_command( for i in range(len(senders)): # get the sender sender = senders[i] - sender_subnet = subnets[i] # defined above as "10.0.{i + 1}.0/24" + sender_subnet = subnets[i] # will be something like 10.X.Y.0/24 # get the sender nic we picked sender_nic = sender_nics[sender] # get the subnet for that nic (follows the pattern from before) @@ -484,8 +485,13 @@ def initialize_node_resources( assert_that(len(node.nics)).described_as( "Test needs at least 1 NIC on the test node." ).is_greater_than_or_equal_to(1) - - test_nic = node.nics.get_nic_by_subnet("10.0.1.0/24") + # get a sorted list of subnets for the nics on the node, + # excluding the primary nic subnet, + # and pick the one on the lowest subnet as the testing default. + subnets = sorted( + [str(subnet) for subnet in node.nics.get_node_subnets(include_primary=False)] + ) + test_nic = node.nics.get_nic_by_subnet(str(subnets[0])) # check an assumption that our nics are bound to hv_netvsc # at test start. @@ -869,13 +875,17 @@ def verify_dpdk_mutliple_ports( (f"receiver:{external_ips[0]}\nsenders:{external_ips[1]},{external_ips[2]}\n") ) receiver, sender_a, sender_b = environment.nodes.list() + subnets = receiver.nics.get_node_subnets(include_primary=False) + subnet_a, subnet_b = subnets[:2] # will be something like 10.X.Y.0/24 + # note: will assert if there are no nics on corresponding subnets, + # this is good and proper since we can't run without that setup. nic_pairings = { receiver: [ - receiver.nics.get_nic_by_subnet("10.0.1.0/24"), - receiver.nics.get_nic_by_subnet("10.0.2.0/24"), + receiver.nics.get_nic_by_subnet(str(subnet_a)), + receiver.nics.get_nic_by_subnet(str(subnet_b)), ], - sender_a: [sender_a.nics.get_nic_by_subnet("10.0.1.0/24")], - sender_b: [sender_b.nics.get_nic_by_subnet("10.0.2.0/24")], + sender_a: [sender_a.nics.get_nic_by_subnet(str(subnet_a))], + sender_b: [sender_b.nics.get_nic_by_subnet(str(subnet_b))], } # get test duration variable if set # enables long-running tests to shakeQoS and SLB issue @@ -986,10 +996,12 @@ def ipv4_to_lpm(addr: str) -> str: # enable ip forwarding for secondary and tertiary nics in this test. # run in parallel to save a bit of time on this net io step. def __enable_ip_forwarding(node: Node) -> None: - fwd_subnets = [ - node.nics.get_nic_by_index(nic_index).ip_addr for nic_index in [1, 2] - ] - for subnet_ip in fwd_subnets: + fwd_subnets = node.nics.get_node_subnets(include_primary=False) + subnet_nics = [node.nics.get_nic_by_subnet(str(subnet)) for subnet in fwd_subnets] + subnet_ips = [nic.ip_addr for nic in subnet_nics] + + for subnet_ip in subnet_ips: + node.log.debug(f"Enabling IP forwarding for nic on subnet {subnet_ip}") node.features[NetworkInterface].switch_ip_forwarding( enable=True, private_ip_addr=subnet_ip ) @@ -1115,22 +1127,8 @@ def verify_dpdk_l3fwd_ntttcp_tcp( # 3. enjoy the thrill of victory, ship a cloud net applicance. l3fwd_app_name = "l3fwd" - # pick fwd/send/receive nodes based on well known addresses in our subnets - forwarder = [ - node - for node in environment.nodes.list() - if node.nics.get_primary_nic().ip_addr.endswith("4") - ][0] - sender = [ - node - for node in environment.nodes.list() - if node.nics.get_primary_nic().ip_addr.endswith("5") - ][0] - receiver = [ - node - for node in environment.nodes.list() - if node.nics.get_primary_nic().ip_addr.endswith("6") - ][0] + + forwarder, sender, receiver = environment.nodes.list() if not ( forwarder.tools[Lscpu].get_architecture() == CpuArchitecture.X64 @@ -1172,15 +1170,22 @@ def verify_dpdk_l3fwd_ntttcp_tcp( forwarder.log.debug(f"fwd: {str(forwarder.nics)}") receiver.log.debug(f"rcv: {str(receiver.nics)}") sender.log.debug(f"snd: {str(sender.nics)}") + subnets = forwarder.nics.get_all_subnets(include_primary=False) + if len(subnets) != 2: + raise SkippedException( + "Expected exactly 2 non-primary subnets for this test. " + f"Found subnets: {subnets}" + ) + subnet_a, subnet_b = subnets subnet_a_nics = { - forwarder: forwarder.nics.get_nic_by_subnet("10.0.1.0/24"), - sender: sender.nics.get_nic_by_subnet("10.0.1.0/24"), - receiver: receiver.nics.get_nic_by_subnet("10.0.1.0/24"), + forwarder: forwarder.nics.get_nic_by_subnet(str(subnet_a)), + sender: sender.nics.get_nic_by_subnet(str(subnet_a)), + receiver: receiver.nics.get_nic_by_subnet(str(subnet_a)), } subnet_b_nics = { - forwarder: forwarder.nics.get_nic_by_subnet("10.0.2.0/24"), - receiver: receiver.nics.get_nic_by_subnet("10.0.2.0/24"), - sender: sender.nics.get_nic_by_subnet("10.0.2.0/24"), + forwarder: forwarder.nics.get_nic_by_subnet(str(subnet_b)), + receiver: receiver.nics.get_nic_by_subnet(str(subnet_b)), + sender: sender.nics.get_nic_by_subnet(str(subnet_b)), } # We use ntttcp for snd/rcv which will respect the kernel route table. diff --git a/lisa/nic.py b/lisa/nic.py index 7e91a99396..a2423158a4 100644 --- a/lisa/nic.py +++ b/lisa/nic.py @@ -8,7 +8,7 @@ from collections import OrderedDict from dataclasses import dataclass from pathlib import PurePosixPath -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from assertpy import assert_that from retry import retry @@ -44,6 +44,9 @@ def __init__( self.pci_slot = pci_slot self.dev_uuid = "" self.module_name = "" + self.subnet: Optional[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = ( + None, + ) if driver_sysfs_path is None: self.driver_sysfs_path = PurePosixPath("") else: @@ -60,6 +63,15 @@ def __str__(self) -> str: f"dev_uuid: {self.dev_uuid}\n" ) + def get_subnet(self) -> Union[ipaddress.IPv4Network, ipaddress.IPv6Network]: + # get the subnet for this nic, assuming a mask of /24 + # note: if the bicep template changes to assign different masks for subnet_prefix, + # this function will need to be updated to + if self.subnet: + return self.subnet + else: + raise LisaException(f"No subnet information available for {self.name} ") + @property def is_pci_module_enabled(self) -> bool: """ @@ -304,6 +316,19 @@ def get_nic_by_subnet(self, subnet: str) -> NicInfo: return nic raise LisaException(f"Could not find a nic for requested subnet: {subnet}") + # get a list of all subnets associated with the nics on this node, + # with the option to include the primary nic subnet or not. + def get_node_subnets( + self, include_primary: bool = True + ) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + return list( + [ + nic.subnet + for nic in self.nics.values() + if nic.subnet and (include_primary or nic.name != self.default_nic) + ] + ) + def unbind(self, nic: NicInfo) -> None: # unbind nic from current driver and return the old sysfs path tee = self._node.tools[Tee] @@ -341,6 +366,7 @@ def load_nics_info(self, nic_name: Optional[str] = None) -> None: nic_entry = self.nics[nic_name] nic_entry.ip_addr = ip_addr nic_entry.mac_addr = mac + nic_entry.subnet = nic_info.subnet found_nics.append(nic_name) if not nic_name: diff --git a/lisa/tools/ip.py b/lisa/tools/ip.py index 5b1748da35..7adb8a19be 100644 --- a/lisa/tools/ip.py +++ b/lisa/tools/ip.py @@ -17,10 +17,15 @@ class IpInfo: - def __init__(self, nic_name: str, mac_addr: str, ip_addr: str): + # subnet mask assumes /24, *this is only a convention from the deployment templates for LISA.* + # if subnet configurations change in the future, this will also need to change. + def __init__( + self, nic_name: str, mac_addr: str, ip_addr: str, subnet_mask: str = "24" + ) -> None: self.nic_name = nic_name self.mac_addr = mac_addr self.ip_addr = ip_addr + self.subnet = ipaddress.ip_network(f"{ip_addr}/{subnet_mask}", strict=False) class Ip(Tool): @@ -302,6 +307,7 @@ def get_info(self, nic_name: Optional[str] = None) -> List[IpInfo]: nic_name=matched["name"], mac_addr=matched["mac"], ip_addr=matched["ip_addr"], + subnet_mask=matched["subnet_mask"], ) ) return found_nics From 71c3003b54ab753434362de8bdfec412e2fb88df Mon Sep 17 00:00:00 2001 From: "Matthew G. McGovern" Date: Tue, 24 Feb 2026 14:05:19 -0800 Subject: [PATCH 05/19] shared subnet creation, resource exists delay --- lisa/microsoft/testsuites/dpdk/dpdkutil.py | 8 +- lisa/nic.py | 7 +- lisa/parameter_parser/runbook.py | 2 +- .../sut_orchestrator/azure/arm_template.bicep | 35 +++-- .../azure/autogen_arm_template.json | 138 ++++++++++-------- .../azure/nested_nodes_nics.bicep | 2 +- .../azure/nested_shared_vnet_subnets.bicep | 10 -- lisa/sut_orchestrator/azure/platform_.py | 9 +- lisa/tools/ip.py | 6 +- lisa/util/__init__.py | 9 ++ 10 files changed, 130 insertions(+), 96 deletions(-) delete mode 100644 lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep diff --git a/lisa/microsoft/testsuites/dpdk/dpdkutil.py b/lisa/microsoft/testsuites/dpdk/dpdkutil.py index 7db967ff55..ff75c4af4e 100644 --- a/lisa/microsoft/testsuites/dpdk/dpdkutil.py +++ b/lisa/microsoft/testsuites/dpdk/dpdkutil.py @@ -276,7 +276,7 @@ def generate_testpmd_multiple_port_command( sender = senders[i] sender_nics[sender] = sender.node.nics.get_nic_by_subnet(str(subnet)) # and the corresponding nic on the receiver for that subnet. - receiver_nics[subnet] = receiver.node.nics.get_nic_by_subnet(str(subnet)) + receiver_nics[str(subnet)] = receiver.node.nics.get_nic_by_subnet(str(subnet)) # for MTU test: check that we can fetch the max MTU size for the NIC if set_mtu: @@ -305,7 +305,7 @@ def generate_testpmd_multiple_port_command( # get the subnet for that nic (follows the pattern from before) # get the corresponding receiver nic for that subnet - receiver_nic = receiver_nics[sender_subnet] + receiver_nic = receiver_nics[str(sender_subnet)] # generate the command for the sender snd_cmd = sender.testpmd.generate_testpmd_command( [sender_nic], @@ -321,7 +321,7 @@ def generate_testpmd_multiple_port_command( kit_cmd_pairs[sender] = snd_cmd # receiver needs multiple ports, so only generate the include. receiver_include = receiver.testpmd.generate_testpmd_include( - receiver_nics[sender_subnet], i + receiver_nics[str(sender_subnet)], i ) # and save it receiver_includes += [receiver_include] @@ -1170,7 +1170,7 @@ def verify_dpdk_l3fwd_ntttcp_tcp( forwarder.log.debug(f"fwd: {str(forwarder.nics)}") receiver.log.debug(f"rcv: {str(receiver.nics)}") sender.log.debug(f"snd: {str(sender.nics)}") - subnets = forwarder.nics.get_all_subnets(include_primary=False) + subnets = forwarder.nics.get_node_subnets(include_primary=False) if len(subnets) != 2: raise SkippedException( "Expected exactly 2 non-primary subnets for this test. " diff --git a/lisa/nic.py b/lisa/nic.py index a2423158a4..b24fcbabea 100644 --- a/lisa/nic.py +++ b/lisa/nic.py @@ -44,9 +44,10 @@ def __init__( self.pci_slot = pci_slot self.dev_uuid = "" self.module_name = "" - self.subnet: Optional[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = ( - None, - ) + self.subnet: Optional[ + Union[ipaddress.IPv4Network, ipaddress.IPv6Network] + ] = None + if driver_sysfs_path is None: self.driver_sysfs_path = PurePosixPath("") else: diff --git a/lisa/parameter_parser/runbook.py b/lisa/parameter_parser/runbook.py index 5e2a7d87c4..6fbcbbe715 100644 --- a/lisa/parameter_parser/runbook.py +++ b/lisa/parameter_parser/runbook.py @@ -93,7 +93,7 @@ def from_path( runbook_name = builder.partial_resolve(constants.NAME) - constants.RUN_NAME = f"lisa-{runbook_name}-{constants.RUN_ID}" + constants.RUN_NAME = f"{runbook_name}-{constants.RUN_ID}" builder._log.info(f"run name is '{constants.RUN_NAME}'") return builder diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index be07a3c7fa..a3dd4c9718 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -71,7 +71,7 @@ var node_count = length(nodes) var availability_set_name_value = 'lisa-availabilitySet' var wrapped_resource_group_index = resource_group_index % 256 var use_existing_vnet = !empty(virtual_network_resource_group) -var shared_subnet_names = [for nic_index in range(0, subnet_count): '10.${wrapped_resource_group_index}.${nic_index}.0'] +var shared_subnet_names = [for nic_index in range(0, subnet_count): nic_index==0 ? '10.0.0.0' : 'e${resource_group_index}-10.${wrapped_resource_group_index}.${nic_index}.0'] var availability_set_tags = availability_options.availability_set_tags var availability_set_properties = availability_options.availability_set_properties var availability_zones = availability_options.availability_zones @@ -235,7 +235,7 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): nic_count: nodes[i].nic_count location: location vnet_id: vnet_id - resource_group_index: wrapped_resource_group_index + resource_group_index: resource_group_index enable_sriov: nodes[i].enable_sriov tags: tags use_ipv6: use_ipv6 @@ -248,16 +248,25 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): ] }] -module shared_vnet_subnets './nested_shared_vnet_subnets.bicep' = if (use_existing_vnet) { - name: 'shared-vnet-subnets' - scope: resourceGroup(virtual_network_resource_group) - params: { - virtual_network_name: virtual_network_name - subnet_names: shared_subnet_names +// If there is already a vnet, LISA only needs to create the test nic subnets. 10.0.0.0/24 must already exist. +// This deployment should generate an exception at runtime if two environments have overlapping address spaces; +// this is expected and will happen if the environment id mod 256 rolls over while an old environment is still active. This should be rare, but will work out so long as: +// LISA must catch this exception and retry the deployment after a timeout period to allow the old environment to be cleaned up. +// LISA must remove old subnets when an environment is not needed anymore. +// This will ensure no collisions occur where one test in a subnet can disturb another in the same subnet. + +resource shared_vnet_subnets 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = [for j in range(0, subnet_count): if (use_existing_vnet && j > 0) { + parent: virtual_network_name_resource + name: shared_subnet_names[j] + properties: { + addressPrefix: '10.${wrapped_resource_group_index}.${j}.0/24' } -} +}] + + + -resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-05-01' = if (empty(virtual_network_resource_group)) { +resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-05-01' = if (!use_existing_vnet) { name: virtual_network_name tags: tags location: location @@ -269,11 +278,11 @@ resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-0 ) } subnets: [for j in range(0, subnet_count): { - name: j == 0 ? '10.0.0.0' : '10.${resource_group_index}.${j}.0' + name: j == 0 ? '10.0.0.0' : 'e${resource_group_index}-10.${wrapped_resource_group_index}.${j}.0' properties: { addressPrefixes: concat( - ['10.${j ==0 ? '0' : resource_group_index}.${j}.0/24'], - use_ipv6 ? ['2001:db8:${j == 0 ? j : resource_group_index}:${j}::/64'] : [] + ['10.${j ==0 ? '0' : wrapped_resource_group_index}.${j}.0/${j == 0 ? 16: 24}'], + use_ipv6 ? ['2001:db8:${j == 0 ? j : wrapped_resource_group_index}:${j}::/64'] : [] ) defaultOutboundAccess: enable_vm_nat networkSecurityGroup: { diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index e041c4499f..18f064c998 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.40.2.10011", - "templateHash": "6973919806372738793" + "templateHash": "1821201376234486735" } }, "functions": [ @@ -556,7 +556,7 @@ { "name": "shared_subnet_names", "count": "[length(range(0, parameters('subnet_count')))]", - "input": "[format('10.{0}.{1}.0', variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')])]" + "input": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')], 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')]))]" }, { "name": "ip_tags", @@ -583,8 +583,24 @@ "combined_aset_tags": "[union(parameters('tags'), variables('availability_set_tags'))]" }, "resources": { + "shared_vnet_subnets": { + "copy": { + "name": "shared_vnet_subnets", + "count": "[length(range(0, parameters('subnet_count')))]" + }, + "condition": "[and(variables('use_existing_vnet'), greater(range(0, parameters('subnet_count'))[copyIndex()], 0))]", + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2024-05-01", + "name": "[format('{0}/{1}', parameters('virtual_network_name'), variables('shared_subnet_names')[range(0, parameters('subnet_count'))[copyIndex()]])]", + "properties": { + "addressPrefix": "[format('10.{0}.{1}.0/24', variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex()])]" + }, + "dependsOn": [ + "virtual_network_name_resource" + ] + }, "virtual_network_name_resource": { - "condition": "[empty(parameters('virtual_network_resource_group'))]", + "condition": "[not(variables('use_existing_vnet'))]", "type": "Microsoft.Network/virtualNetworks", "apiVersion": "2024-05-01", "name": "[parameters('virtual_network_name')]", @@ -596,9 +612,9 @@ "name": "subnets", "count": "[length(range(0, parameters('subnet_count')))]", "input": { - "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '10.0.0.0', format('10.{0}.{1}.0', parameters('resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", + "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", "properties": { - "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '0', parameters('resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), range(0, parameters('subnet_count'))[copyIndex('subnets')], parameters('resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", + "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/{2}', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '0', variables('wrapped_resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')], if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), 16, 24))), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), range(0, parameters('subnet_count'))[copyIndex('subnets')], variables('wrapped_resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", "defaultOutboundAccess": "[parameters('enable_vm_nat')]", "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('{0}-nsg', toLower(parameters('virtual_network_name'))))]" @@ -635,6 +651,58 @@ "destinationAddressPrefix": "*" } }, + { + "name": "AllowAgentPool", + "properties": { + "priority": 101, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "sourceAddressPrefix": "1ESResourceManager", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22" + } + }, + { + "name": "AllowCorpNetPublic", + "properties": { + "priority": 102, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "sourceAddressPrefix": "CorpNetPublic", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "*" + } + }, + { + "name": "VirtualNetworkIn", + "properties": { + "priority": 103, + "direction": "Inbound", + "access": "Allow", + "protocol": "*", + "sourceAddressPrefix": "VirtualNetwork", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "*" + } + }, + { + "name": "AllowManagedVPN", + "properties": { + "priority": 104, + "direction": "Inbound", + "access": "Allow", + "protocol": "*", + "sourceAddressPrefixes": ["20.120.143.192/28","20.241.227.160/28","20.31.68.160/28","20.198.165.176/28","20.83.242.194/32"], + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "*" + } + }, { "name": "LISAKVMSSH", "properties": { @@ -910,7 +978,7 @@ "value": "[variables('vnet_id')]" }, "resource_group_index": { - "value": "[variables('wrapped_resource_group_index')]" + "value": "[parameters('resource_group_index')]" }, "enable_sriov": { "value": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].enable_sriov]" @@ -933,7 +1001,7 @@ "_generator": { "name": "bicep", "version": "0.40.2.10011", - "templateHash": "18100026266609217689" + "templateHash": "2629862527207444319" } }, "functions": [ @@ -971,7 +1039,7 @@ ], "output": { "type": "string", - "value": "[if(equals(parameters('nic_index'), 0), '10.0.0.0', format('10.{0}.{1}.0', parameters('resource_group_index'), parameters('nic_index')))]" + "value": "[if(equals(parameters('nic_index'), 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), mod(parameters('resource_group_index'), 256), parameters('nic_index')))]" } } } @@ -1035,60 +1103,6 @@ "shared_vnet_subnets", "virtual_network_name_resource" ] - }, - "shared_vnet_subnets": { - "condition": "[variables('use_existing_vnet')]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2025-04-01", - "name": "shared-vnet-subnets", - "resourceGroup": "[parameters('virtual_network_resource_group')]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "virtual_network_name": { - "value": "[parameters('virtual_network_name')]" - }, - "subnet_names": { - "value": "[variables('shared_subnet_names')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.40.2.10011", - "templateHash": "1894805062932954870" - } - }, - "parameters": { - "virtual_network_name": { - "type": "string" - }, - "subnet_names": { - "type": "array" - } - }, - "resources": [ - { - "copy": { - "name": "shared_vnet_subnets", - "count": "[length(parameters('subnet_names'))]" - }, - "type": "Microsoft.Network/virtualNetworks/subnets", - "apiVersion": "2024-05-01", - "name": "[format('{0}/{1}', parameters('virtual_network_name'), parameters('subnet_names')[copyIndex()])]", - "properties": { - "addressPrefix": "[format('{0}/24', parameters('subnet_names')[copyIndex()])]" - } - } - ] - } - } } } } \ No newline at end of file diff --git a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep index 9910116e49..aed3457a7e 100644 --- a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep +++ b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep @@ -15,7 +15,7 @@ func getPublicIpAddress(vmName string, publicIpName string) object => { var publicIpAddress = getPublicIpAddress(vmName, '${vmName}-public-ip') var publicIpAddressV6 = getPublicIpAddress(vmName, '${vmName}-public-ipv6') -func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : '10.${resource_group_index}.${nic_index}.0' +func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : 'e${resource_group_index}-10.${resource_group_index%256}.${nic_index}.0' resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in range(0, nic_count): { name: '${vmName}-nic-${i}' diff --git a/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep b/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep deleted file mode 100644 index 68e5061e32..0000000000 --- a/lisa/sut_orchestrator/azure/nested_shared_vnet_subnets.bicep +++ /dev/null @@ -1,10 +0,0 @@ -param virtual_network_name string -param subnet_names array - -resource shared_vnet_subnets 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = [for subnet_name in subnet_names: { - name: '${virtual_network_name}/${subnet_name}' - properties: { - addressPrefix: '${subnet_name}/24' - } -} -] diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 95292fcd48..88172f646e 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -19,7 +19,10 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union, cast import requests -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.core.exceptions import ( + HttpResponseError, + ResourceNotFoundError, +) from azure.identity import DefaultAzureCredential from azure.mgmt.compute.models import ( CommunityGalleryImage, @@ -71,6 +74,7 @@ KernelPanicException, LisaException, LisaTimeoutException, + DeploymentActiveException, NotMeetRequirementException, ResourceAwaitableException, RootFsMountFailedException, @@ -1689,6 +1693,7 @@ def _validate_template( plugin_manager.hook.azure_deploy_failed(error_message=error_message) raise LisaException(error_message) + @retry(exceptions=DeploymentActiveException, tries=5, delay=60) # type: ignore def _deploy( self, location: str, @@ -1773,6 +1778,8 @@ def _deploy( f"provisioning failed for an internal error, try to run case. " f"Exception: {error_message}" ) + elif "ResourceExistsError" in error_message: + raise DeploymentActiveException(error_message) else: try: self._save_console_log_and_check_panic( diff --git a/lisa/tools/ip.py b/lisa/tools/ip.py index 7adb8a19be..7ab965000c 100644 --- a/lisa/tools/ip.py +++ b/lisa/tools/ip.py @@ -25,7 +25,11 @@ def __init__( self.nic_name = nic_name self.mac_addr = mac_addr self.ip_addr = ip_addr - self.subnet = ipaddress.ip_network(f"{ip_addr}/{subnet_mask}", strict=False) + self.subnet = ( + ipaddress.ip_network(f"{ip_addr}/{subnet_mask}", strict=False) + if all([ip_addr, subnet_mask]) + else None + ) class Ip(Tool): diff --git a/lisa/util/__init__.py b/lisa/util/__init__.py index 75b9584f82..e222553b7e 100644 --- a/lisa/util/__init__.py +++ b/lisa/util/__init__.py @@ -161,6 +161,15 @@ def __init__(self, *args: object) -> None: super().__init__(*args) +class DeploymentActiveException(LisaException): + """ + This exception is used to indicate that there is an active deployment with the same name. It may be caused by the previous deployment not cleaned up yet. + LISA should catch this exception and retry the deployment after a timeout period to allow the old environment to be cleaned up. + """ + + ... + + class UnsupportedOperationException(LisaException): """ An operation might not be supported. Use this exception to From 85aee4a74b0dc805e8228b3705443c5e8d0ac83e Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 25 Feb 2026 09:56:11 -0800 Subject: [PATCH 06/19] subnet fixes, merge updates --- lisa/sut_orchestrator/azure/common.py | 2 +- lisa/sut_orchestrator/azure/features.py | 1 + .../azure/nested_nodes_nics.bicep | 2 +- lisa/sut_orchestrator/azure/platform_.py | 53 +++++++++++++++++-- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index 4dfdff98e4..a5af54c7d5 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -1262,7 +1262,7 @@ class AzureArmParameter: enable_vm_nat: bool = False create_public_address: bool = True source_address_prefixes: List[str] = field(default_factory=list) - resource_group_index: int = 0 + resource_group_index: Optional[int] = None def __post_init__(self, *args: Any, **kwargs: Any) -> None: add_secret(self.admin_username, PATTERN_HEADTAIL) diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index a077b90a10..e02032494d 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import asyncio import copy +import ipaddress import json import re import string diff --git a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep index aed3457a7e..9910116e49 100644 --- a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep +++ b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep @@ -15,7 +15,7 @@ func getPublicIpAddress(vmName string, publicIpName string) object => { var publicIpAddress = getPublicIpAddress(vmName, '${vmName}-public-ip') var publicIpAddressV6 = getPublicIpAddress(vmName, '${vmName}-public-ipv6') -func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : 'e${resource_group_index}-10.${resource_group_index%256}.${nic_index}.0' +func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : '10.${resource_group_index}.${nic_index}.0' resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in range(0, nic_count): { name: '${vmName}-nic-${i}' diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 88172f646e..437b3d89d2 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import ast import copy +import ipaddress import json import logging import math @@ -24,6 +25,7 @@ ResourceNotFoundError, ) from azure.identity import DefaultAzureCredential +from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.compute.models import ( CommunityGalleryImage, CommunityGalleryImageVersion, @@ -36,6 +38,7 @@ VirtualMachineImage, ) from azure.mgmt.marketplaceordering.models import AgreementTerms +from azure.mgmt.network import NetworkManagementClient from azure.mgmt.resource import FeatureClient, SubscriptionClient from azure.mgmt.resource.resources.models import ( Deployment, @@ -129,6 +132,7 @@ get_deployable_storage_path, get_environment_context, get_marketplace_ordering_client, + get_network_client, get_node_context, get_or_create_storage_container, get_primary_ip_addresses, @@ -668,20 +672,61 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: except Exception as e: raise e + def delete_subnet(self, resource_group_name: str, subnet_address: str) -> None: + platform: AzurePlatform = self._platform # type: ignore + network_client = get_network_client(platform) + resource_client = get_resource_management_client( + platform.credential, platform.subscription_id, platform.cloud + ) + _compute_client = get_compute_client(platform) + vnets = network_client.virtual_networks.list( + resource_group_name=resource_group_name + ) + # handy convention when the subnet is named after the address prefix + virtual_network_name = vnets[0].name + subnet_az = network_client.subnets.get( + resource_group_name=resource_group_name, + virtual_network_name=virtual_network_name, + subnet_name=str( + ipaddress.ip_network(subnet_address, strict=False).network_address + ), + ) + if not subnet_az: + return + nics = network_client.network_interfaces.list( + resource_group_name=resource_group_name + ) + if not nics: + return + for nic in nics: + self._log.debug(f"Found nic: {nic.name}") + for ipconfig in nic.ip_configurations: + if ipconfig.subnet.id == subnet_az.id: + self._log.debug( + f"Found subnet: {subnet_az.name} and ipconfig: {ipconfig.name}" + ) + vm_az = resource_client.resources.get_by_id( + resource_id=nic.virtual_machine.id + ) + self._log.debug(f"found vm: {vm_az.name}") + # compute_client.virtual_machines.begin_delete(resource_group_name, vm_az.name).wait() + # network_client.network_interfaces.begin_delete(resource_group_name, nic.name).wait() + def _delete_environment(self, environment: Environment, log: Logger) -> None: environment_context = get_environment_context(environment=environment) resource_group_name = environment_context.resource_group_name # the resource group name is empty when it is not deployed for some reasons, # like capability doesn't meet case requirement. + platform: AzurePlatform = self._platform # type: ignore if not resource_group_name: return assert self._azure_runbook if not environment_context.resource_group_is_specified: - log.info( - f"skipped to delete resource group: {resource_group_name}, " - f"as it's specified in runbook." - ) + node, _ = environment.nodes.list() + subnets = node.nics.get_node_subnets() + for subnet in subnets: + self.delete_subnet(resource_group_name, str(subnet.network_address)) elif self._azure_runbook.dry_run: log.info( f"skipped to delete resource group: {resource_group_name}, " From 12d03b47e9f4fa089192cbb32a0d0dce5af175cb Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Tue, 3 Mar 2026 14:44:22 -0800 Subject: [PATCH 07/19] arm template and bicep fixes: use peering if virtual network rg is provided --- .../sut_orchestrator/azure/arm_template.bicep | 104 +++++---- .../azure/autogen_arm_template.json | 217 ++++++++++-------- .../azure/nested_nodes_nics.bicep | 2 +- .../azure/remote-peering.bicep | 27 +++ 4 files changed, 215 insertions(+), 135 deletions(-) create mode 100644 lisa/sut_orchestrator/azure/remote-peering.bicep diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index a3dd4c9718..6985b1640b 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -64,14 +64,13 @@ param source_address_prefixes array @description('Generate public IP address for each node') param create_public_address bool -var vnet_id = use_existing_vnet -? resourceId(virtual_network_resource_group, 'Microsoft.Network/virtualNetworks', virtual_network_name) -: virtual_network_name_resource.id + var node_count = length(nodes) var availability_set_name_value = 'lisa-availabilitySet' -var wrapped_resource_group_index = resource_group_index % 256 +var rg_index_mod_256 = resource_group_index % 256 +var rg_index_div_256 = resource_group_index / 256 var use_existing_vnet = !empty(virtual_network_resource_group) -var shared_subnet_names = [for nic_index in range(0, subnet_count): nic_index==0 ? '10.0.0.0' : 'e${resource_group_index}-10.${wrapped_resource_group_index}.${nic_index}.0'] +//var shared_subnet_names = [for nic_index in range(0, subnet_count): nic_index==0 ? 'default' : 'test-subnet-${nic_index}'] var availability_set_tags = availability_options.availability_set_tags var availability_set_properties = availability_options.availability_set_properties var availability_zones = availability_options.availability_zones @@ -228,13 +227,14 @@ func getAvailabilitySetId(availability_set_name string) object => { id: resourceId('Microsoft.Compute/availabilitySets', availability_set_name) } + module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): { name: '${nodes[i].name}-nics' params: { vmName: nodes[i].name nic_count: nodes[i].nic_count location: location - vnet_id: vnet_id + vnet_id: virtual_network.id resource_group_index: resource_group_index enable_sriov: nodes[i].enable_sriov tags: tags @@ -244,10 +244,15 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): dependsOn: [ nodes_public_ip[i] nodes_public_ip_ipv6[i] - shared_vnet_subnets ] }] +resource orchestrator_vnet 'Microsoft.Network/virtualNetworks@2024-01-01' existing = if (use_existing_vnet) { + scope: resourceGroup(virtual_network_resource_group) + name: virtual_network_name +} + + // If there is already a vnet, LISA only needs to create the test nic subnets. 10.0.0.0/24 must already exist. // This deployment should generate an exception at runtime if two environments have overlapping address spaces; // this is expected and will happen if the environment id mod 256 rolls over while an old environment is still active. This should be rare, but will work out so long as: @@ -255,64 +260,77 @@ module nodes_nics './nested_nodes_nics.bicep' = [for i in range(0, node_count): // LISA must remove old subnets when an environment is not needed anymore. // This will ensure no collisions occur where one test in a subnet can disturb another in the same subnet. -resource shared_vnet_subnets 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = [for j in range(0, subnet_count): if (use_existing_vnet && j > 0) { - parent: virtual_network_name_resource - name: shared_subnet_names[j] - properties: { - addressPrefix: '10.${wrapped_resource_group_index}.${j}.0/24' +module remotePeering 'remote-peering.bicep' = if (use_existing_vnet) { + name: 'remote-peering-deployment' + scope: resourceGroup(virtual_network_resource_group) + dependsOn: [ + peering + ] + params: { + remoteVnetName: virtual_network_name + localVnetId: virtual_network.id + resource_group_index: resource_group_index } -}] - +} +resource peering 'Microsoft.Network/virtualNetworks/virtualNetworkPeerings@2023-11-01' = if (use_existing_vnet) { + name: 'vnet-peering-e${resource_group_index}' + parent: virtual_network + properties: { + allowVirtualNetworkAccess: true + localSubnetNames: [ 'default' ] + peerCompleteVnets: false + remoteSubnetNames:['default'] + remoteVirtualNetwork: { + id: orchestrator_vnet.id + } + //remoteVirtualNetwork: orchestrator_vnet // reference to the orchestrator vnet + } + } -resource virtual_network_name_resource 'Microsoft.Network/virtualNetworks@2024-05-01' = if (!use_existing_vnet) { - name: virtual_network_name +resource virtual_network 'Microsoft.Network/virtualNetworks@2024-05-01' = { + name: 'test-vnet-${resource_group_index}' tags: tags location: location properties: { addressSpace: { addressPrefixes: concat( - ['10.0.0.0/8'], + ['10.${rg_index_div_256}.${rg_index_mod_256}.0/24', '192.168.0.0/16' ], use_ipv6 ? ['2001:db8::/32'] : [] ) } - subnets: [for j in range(0, subnet_count): { - name: j == 0 ? '10.0.0.0' : 'e${resource_group_index}-10.${wrapped_resource_group_index}.${j}.0' - properties: { - addressPrefixes: concat( - ['10.${j ==0 ? '0' : wrapped_resource_group_index}.${j}.0/${j == 0 ? 16: 24}'], - use_ipv6 ? ['2001:db8:${j == 0 ? j : wrapped_resource_group_index}:${j}::/64'] : [] - ) + subnets: [ for i in range(0,subnet_count): { + name: i==0 ? 'default' : 'test-subnet-${i}' + properties: { + addressPrefix: i==0 ? '10.${rg_index_div_256}.${rg_index_mod_256}.0/24' : '192.168.${i-1}.0/24' defaultOutboundAccess: enable_vm_nat - networkSecurityGroup: { - id: resourceId('Microsoft.Network/networkSecurityGroups', '${toLower(virtual_network_name)}-nsg') + networkSecurityGroup:{ + id: nsg.id + } } - } }] } - dependsOn: [ - nsg - ] } resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = { - name: '${toLower(virtual_network_name)}-nsg' + name: 'lisa-test-nsg-${resource_group_index}' location: location properties: { securityRules: [ { - name: 'LISASSH' - properties: { - priority: 100 - direction: 'Inbound' - access: 'Allow' - protocol: 'Tcp' - sourcePortRange: '*' - destinationPortRange: '22' - sourceAddressPrefixes: source_address_prefixes - destinationAddressPrefix: '*' - } + name: 'LISASSH' + properties: { + description: 'Allows nested VM SSH traffic' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '60020-60030' + destinationAddressPrefix: '*' + sourceAddressPrefixes: source_address_prefixes + access: 'Allow' + priority: 101 + direction: 'Inbound' + } } { name: 'LISAKVMSSH' @@ -513,7 +531,7 @@ resource nodes_vms 'Microsoft.Compute/virtualMachines@2024-03-01' = [for i in ra availability_set nodes_image nodes_nics - virtual_network_name_resource + virtual_network nodes_disk nodes_data_disks_with_vhds ] diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index 18f064c998..cf44284695 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.40.2.10011", - "templateHash": "1821201376234486735" + "version": "0.41.2.15936", + "templateHash": "3674176336733772196" } }, "functions": [ @@ -553,11 +553,6 @@ }, "variables": { "copy": [ - { - "name": "shared_subnet_names", - "count": "[length(range(0, parameters('subnet_count')))]", - "input": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')], 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('shared_subnet_names')]))]" - }, { "name": "ip_tags", "count": "[length(objectKeys(parameters('ip_service_tags')))]", @@ -567,10 +562,10 @@ } } ], - "vnet_id": "[if(variables('use_existing_vnet'), resourceId(parameters('virtual_network_resource_group'), 'Microsoft.Network/virtualNetworks', parameters('virtual_network_name')), resourceId('Microsoft.Network/virtualNetworks', parameters('virtual_network_name')))]", "node_count": "[length(parameters('nodes'))]", "availability_set_name_value": "lisa-availabilitySet", - "wrapped_resource_group_index": "[mod(parameters('resource_group_index'), 256)]", + "rg_index_mod_256": "[mod(parameters('resource_group_index'), 256)]", + "rg_index_div_256": "[div(parameters('resource_group_index'), 256)]", "use_existing_vnet": "[not(empty(parameters('virtual_network_resource_group')))]", "availability_set_tags": "[parameters('availability_options').availability_set_tags]", "availability_set_properties": "[parameters('availability_options').availability_set_properties]", @@ -583,27 +578,40 @@ "combined_aset_tags": "[union(parameters('tags'), variables('availability_set_tags'))]" }, "resources": { - "shared_vnet_subnets": { - "copy": { - "name": "shared_vnet_subnets", - "count": "[length(range(0, parameters('subnet_count')))]" - }, - "condition": "[and(variables('use_existing_vnet'), greater(range(0, parameters('subnet_count'))[copyIndex()], 0))]", - "type": "Microsoft.Network/virtualNetworks/subnets", - "apiVersion": "2024-05-01", - "name": "[format('{0}/{1}', parameters('virtual_network_name'), variables('shared_subnet_names')[range(0, parameters('subnet_count'))[copyIndex()]])]", + "orchestrator_vnet": { + "condition": "[variables('use_existing_vnet')]", + "existing": true, + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2024-01-01", + "resourceGroup": "[parameters('virtual_network_resource_group')]", + "name": "[parameters('virtual_network_name')]" + }, + "peering": { + "condition": "[variables('use_existing_vnet')]", + "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings", + "apiVersion": "2023-11-01", + "name": "[format('{0}/{1}', format('test-vnet-{0}', parameters('resource_group_index')), format('vnet-peering-e{0}', parameters('resource_group_index')))]", "properties": { - "addressPrefix": "[format('10.{0}.{1}.0/24', variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex()])]" + "allowVirtualNetworkAccess": true, + "localSubnetNames": [ + "default" + ], + "peerCompleteVnets": false, + "remoteSubnetNames": [ + "default" + ], + "remoteVirtualNetwork": { + "id": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('virtual_network_resource_group')), 'Microsoft.Network/virtualNetworks', parameters('virtual_network_name'))]" + } }, "dependsOn": [ - "virtual_network_name_resource" + "virtual_network" ] }, - "virtual_network_name_resource": { - "condition": "[not(variables('use_existing_vnet'))]", + "virtual_network": { "type": "Microsoft.Network/virtualNetworks", "apiVersion": "2024-05-01", - "name": "[parameters('virtual_network_name')]", + "name": "[format('test-vnet-{0}', parameters('resource_group_index'))]", "tags": "[parameters('tags')]", "location": "[parameters('location')]", "properties": { @@ -612,19 +620,19 @@ "name": "subnets", "count": "[length(range(0, parameters('subnet_count')))]", "input": { - "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), variables('wrapped_resource_group_index'), range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", + "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), 'default', format('test-subnet-{0}', range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", "properties": { - "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/{2}', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), '0', variables('wrapped_resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')], if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), 16, 24))), if(parameters('use_ipv6'), createArray(format('2001:db8:{0}:{1}::/64', if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), range(0, parameters('subnet_count'))[copyIndex('subnets')], variables('wrapped_resource_group_index')), range(0, parameters('subnet_count'))[copyIndex('subnets')])), createArray()))]", + "addressPrefix": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), format('192.168.{0}.0/24', sub(range(0, parameters('subnet_count'))[copyIndex('subnets')], 1)))]", "defaultOutboundAccess": "[parameters('enable_vm_nat')]", "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('{0}-nsg', toLower(parameters('virtual_network_name'))))]" + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('lisa-test-nsg-{0}', parameters('resource_group_index')))]" } } } } ], "addressSpace": { - "addressPrefixes": "[concat(createArray('10.0.0.0/8'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray()))]" + "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), '192.168.0.0/16'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray()))]" } }, "dependsOn": [ @@ -634,73 +642,22 @@ "nsg": { "type": "Microsoft.Network/networkSecurityGroups", "apiVersion": "2024-05-01", - "name": "[format('{0}-nsg', toLower(parameters('virtual_network_name')))]", + "name": "[format('lisa-test-nsg-{0}', parameters('resource_group_index'))]", "location": "[parameters('location')]", "properties": { "securityRules": [ { "name": "LISASSH", "properties": { - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefixes": "[parameters('source_address_prefixes')]", - "destinationAddressPrefix": "*" - } - }, - { - "name": "AllowAgentPool", - "properties": { - "priority": 101, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "sourceAddressPrefix": "1ESResourceManager", - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "22" - } - }, - { - "name": "AllowCorpNetPublic", - "properties": { - "priority": 102, - "direction": "Inbound", - "access": "Allow", + "description": "Allows nested VM SSH traffic", "protocol": "Tcp", - "sourceAddressPrefix": "CorpNetPublic", - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "*" - } - }, - { - "name": "VirtualNetworkIn", - "properties": { - "priority": 103, - "direction": "Inbound", - "access": "Allow", - "protocol": "*", - "sourceAddressPrefix": "VirtualNetwork", "sourcePortRange": "*", + "destinationPortRange": "60020-60030", "destinationAddressPrefix": "*", - "destinationPortRange": "*" - } - }, - { - "name": "AllowManagedVPN", - "properties": { - "priority": 104, - "direction": "Inbound", + "sourceAddressPrefixes": "[parameters('source_address_prefixes')]", "access": "Allow", - "protocol": "*", - "sourceAddressPrefixes": ["20.120.143.192/28","20.241.227.160/28","20.31.68.160/28","20.198.165.176/28","20.83.242.194/32"], - "sourcePortRange": "*", - "destinationAddressPrefix": "*", - "destinationPortRange": "*" + "priority": 101, + "direction": "Inbound" } }, { @@ -948,7 +905,7 @@ "nodes_disk", "nodes_image", "nodes_nics", - "virtual_network_name_resource" + "virtual_network" ] }, "nodes_nics": { @@ -975,7 +932,7 @@ "value": "[parameters('location')]" }, "vnet_id": { - "value": "[variables('vnet_id')]" + "value": "[resourceId('Microsoft.Network/virtualNetworks', format('test-vnet-{0}', parameters('resource_group_index')))]" }, "resource_group_index": { "value": "[parameters('resource_group_index')]" @@ -1000,8 +957,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.40.2.10011", - "templateHash": "2629862527207444319" + "version": "0.41.2.15936", + "templateHash": "7649300423127023996" } }, "functions": [ @@ -1039,7 +996,7 @@ ], "output": { "type": "string", - "value": "[if(equals(parameters('nic_index'), 0), '10.0.0.0', format('e{0}-10.{1}.{2}.0', parameters('resource_group_index'), mod(parameters('resource_group_index'), 256), parameters('nic_index')))]" + "value": "[if(equals(parameters('nic_index'), 0), 'default', format('test-subnet-{0}', parameters('nic_index')))]" } } } @@ -1100,8 +1057,86 @@ "dependsOn": [ "[format('nodes_public_ip[{0}]', range(0, variables('node_count'))[copyIndex()])]", "[format('nodes_public_ip_ipv6[{0}]', range(0, variables('node_count'))[copyIndex()])]", - "shared_vnet_subnets", - "virtual_network_name_resource" + "virtual_network" + ] + }, + "remotePeering": { + "condition": "[variables('use_existing_vnet')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "remote-peering-deployment", + "resourceGroup": "[parameters('virtual_network_resource_group')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "remoteVnetName": { + "value": "[parameters('virtual_network_name')]" + }, + "localVnetId": { + "value": "[resourceId('Microsoft.Network/virtualNetworks', format('test-vnet-{0}', parameters('resource_group_index')))]" + }, + "resource_group_index": { + "value": "[parameters('resource_group_index')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.41.2.15936", + "templateHash": "4298084375997684263" + } + }, + "parameters": { + "remoteVnetName": { + "type": "string", + "metadata": { + "description": "Name of the existing remote virtual network to peer with" + } + }, + "localVnetId": { + "type": "string", + "metadata": { + "description": "ID of the local VNet" + } + }, + "resource_group_index": { + "type": "int", + "metadata": { + "description": "resource group index (for unique naming)" + } + } + }, + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings", + "apiVersion": "2023-11-01", + "name": "[format('{0}/{1}', parameters('remoteVnetName'), format('vnet-peering-e{0}', parameters('resource_group_index')))]", + "properties": { + "allowVirtualNetworkAccess": true, + "localSubnetNames": [ + "default" + ], + "peerCompleteVnets": false, + "remoteSubnetNames": [ + "default" + ], + "remoteVirtualNetwork": { + "id": "[parameters('localVnetId')]" + } + } + } + ] + } + }, + "dependsOn": [ + "peering", + "virtual_network" ] } } diff --git a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep index 9910116e49..f0c41daa53 100644 --- a/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep +++ b/lisa/sut_orchestrator/azure/nested_nodes_nics.bicep @@ -15,7 +15,7 @@ func getPublicIpAddress(vmName string, publicIpName string) object => { var publicIpAddress = getPublicIpAddress(vmName, '${vmName}-public-ip') var publicIpAddressV6 = getPublicIpAddress(vmName, '${vmName}-public-ipv6') -func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? '10.0.0.0' : '10.${resource_group_index}.${nic_index}.0' +func getSubnetName(resource_group_index int, nic_index int) string => nic_index == 0 ? 'default' : 'test-subnet-${nic_index}' resource vm_nics 'Microsoft.Network/networkInterfaces@2023-06-01' = [for i in range(0, nic_count): { name: '${vmName}-nic-${i}' diff --git a/lisa/sut_orchestrator/azure/remote-peering.bicep b/lisa/sut_orchestrator/azure/remote-peering.bicep new file mode 100644 index 0000000000..61f72b0eba --- /dev/null +++ b/lisa/sut_orchestrator/azure/remote-peering.bicep @@ -0,0 +1,27 @@ +@description('Name of the existing remote virtual network to peer with') +param remoteVnetName string + +@description('ID of the local VNet') +param localVnetId string + +@description('resource group index (for unique naming)') +param resource_group_index int + +resource remoteVnet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = { + name: remoteVnetName +} + +resource peering 'Microsoft.Network/virtualNetworks/virtualNetworkPeerings@2023-11-01' = { + parent: remoteVnet + name: 'vnet-peering-e${resource_group_index}' + properties: { + allowVirtualNetworkAccess: true + localSubnetNames: [ 'default' ] + peerCompleteVnets: false + remoteSubnetNames:['default'] + remoteVirtualNetwork: { + id: localVnetId + } + //remoteVirtualNetwork: orchestrator_vnet // reference to the orchestrator vnet + } + } From 2dc5249aa126ca699a3b721907e9ff4082b1f077 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Tue, 3 Mar 2026 14:48:15 -0800 Subject: [PATCH 08/19] azure _platform.py: undo changes to platform (not needed now) --- lisa/sut_orchestrator/azure/platform_.py | 83 ++++-------------------- 1 file changed, 13 insertions(+), 70 deletions(-) diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 437b3d89d2..922fa56f77 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -2,14 +2,12 @@ # Licensed under the MIT license. import ast import copy -import ipaddress import json import logging import math import os import re import sys -import threading from copy import deepcopy from dataclasses import InitVar, dataclass, field from datetime import datetime @@ -20,12 +18,8 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union, cast import requests -from azure.core.exceptions import ( - HttpResponseError, - ResourceNotFoundError, -) +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.identity import DefaultAzureCredential -from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.compute.models import ( CommunityGalleryImage, CommunityGalleryImageVersion, @@ -38,7 +32,6 @@ VirtualMachineImage, ) from azure.mgmt.marketplaceordering.models import AgreementTerms -from azure.mgmt.network import NetworkManagementClient from azure.mgmt.resource import FeatureClient, SubscriptionClient from azure.mgmt.resource.resources.models import ( Deployment, @@ -77,7 +70,6 @@ KernelPanicException, LisaException, LisaTimeoutException, - DeploymentActiveException, NotMeetRequirementException, ResourceAwaitableException, RootFsMountFailedException, @@ -110,6 +102,7 @@ from . import features from .common import ( AZURE_SHARED_RG_NAME, + AZURE_SUBNET_PREFIX, AZURE_VIRTUAL_NETWORK_NAME, SAS_URL_PATTERN, AzureArmParameter, @@ -132,7 +125,6 @@ get_deployable_storage_path, get_environment_context, get_marketplace_ordering_client, - get_network_client, get_node_context, get_or_create_storage_container, get_primary_ip_addresses, @@ -329,6 +321,7 @@ class AzurePlatformSchema: virtual_network_resource_group: str = field(default="") virtual_network_name: str = field(default=AZURE_VIRTUAL_NETWORK_NAME) + subnet_prefix: str = field(default=AZURE_SUBNET_PREFIX) # Provisioning error causes by waagent is not ready or other reasons. In # smoke test, it can verify some points also. Other tests should use the @@ -381,6 +374,7 @@ def __post_init__(self, *args: Any, **kwargs: Any) -> None: "log_level", "virtual_network_resource_group", "virtual_network_name", + "subnet_prefix", "use_public_address", "use_ipv6", "enable_vm_nat", @@ -489,8 +483,6 @@ def __init__(self, runbook: schema.Platform) -> None: platform_utils.KEY_MSHV_VERSION: platform_utils.get_mshv_version, } - self._private_key_lock = threading.Lock() - @classmethod def type_name(cls) -> str: return AZURE @@ -613,7 +605,6 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: assert self._azure_runbook environment_context = get_environment_context(environment=environment) - if self._azure_runbook.resource_group_name: resource_group_name = self._azure_runbook.resource_group_name else: @@ -672,61 +663,20 @@ def _deploy_environment(self, environment: Environment, log: Logger) -> None: except Exception as e: raise e - def delete_subnet(self, resource_group_name: str, subnet_address: str) -> None: - platform: AzurePlatform = self._platform # type: ignore - network_client = get_network_client(platform) - resource_client = get_resource_management_client( - platform.credential, platform.subscription_id, platform.cloud - ) - _compute_client = get_compute_client(platform) - vnets = network_client.virtual_networks.list( - resource_group_name=resource_group_name - ) - # handy convention when the subnet is named after the address prefix - virtual_network_name = vnets[0].name - subnet_az = network_client.subnets.get( - resource_group_name=resource_group_name, - virtual_network_name=virtual_network_name, - subnet_name=str( - ipaddress.ip_network(subnet_address, strict=False).network_address - ), - ) - if not subnet_az: - return - nics = network_client.network_interfaces.list( - resource_group_name=resource_group_name - ) - if not nics: - return - for nic in nics: - self._log.debug(f"Found nic: {nic.name}") - for ipconfig in nic.ip_configurations: - if ipconfig.subnet.id == subnet_az.id: - self._log.debug( - f"Found subnet: {subnet_az.name} and ipconfig: {ipconfig.name}" - ) - vm_az = resource_client.resources.get_by_id( - resource_id=nic.virtual_machine.id - ) - self._log.debug(f"found vm: {vm_az.name}") - # compute_client.virtual_machines.begin_delete(resource_group_name, vm_az.name).wait() - # network_client.network_interfaces.begin_delete(resource_group_name, nic.name).wait() - def _delete_environment(self, environment: Environment, log: Logger) -> None: environment_context = get_environment_context(environment=environment) resource_group_name = environment_context.resource_group_name # the resource group name is empty when it is not deployed for some reasons, # like capability doesn't meet case requirement. - platform: AzurePlatform = self._platform # type: ignore if not resource_group_name: return assert self._azure_runbook if not environment_context.resource_group_is_specified: - node, _ = environment.nodes.list() - subnets = node.nics.get_node_subnets() - for subnet in subnets: - self.delete_subnet(resource_group_name, str(subnet.network_address)) + log.info( + f"skipped to delete resource group: {resource_group_name}, " + f"as it's specified in runbook." + ) elif self._azure_runbook.dry_run: log.info( f"skipped to delete resource group: {resource_group_name}, " @@ -1263,23 +1213,19 @@ def _create_deployment_parameters( arm_parameters.virtual_network_resource_group = ( self._azure_runbook.virtual_network_resource_group ) + arm_parameters.subnet_prefix = ( + self._azure_runbook.subnet_prefix or AZURE_SUBNET_PREFIX + ) arm_parameters.virtual_network_name = ( self._azure_runbook.virtual_network_name or AZURE_VIRTUAL_NETWORK_NAME ) arm_parameters.use_ipv6 = self._azure_runbook.use_ipv6 - arm_parameters.resource_group_index = int(environment.id) is_windows: bool = False arm_parameters.admin_username = self.runbook.admin_username # if no key or password specified, generate the key pair - with self._private_key_lock: - if ( - not self.runbook.admin_private_key_file - and not self.runbook.admin_password - ): - self.runbook.admin_private_key_file = get_or_generate_key_pairs( - self._log - ) + if not self.runbook.admin_private_key_file and not self.runbook.admin_password: + self.runbook.admin_private_key_file = get_or_generate_key_pairs(self._log) if self.runbook.admin_private_key_file: arm_parameters.admin_key_data = get_public_key_data( @@ -1738,7 +1684,6 @@ def _validate_template( plugin_manager.hook.azure_deploy_failed(error_message=error_message) raise LisaException(error_message) - @retry(exceptions=DeploymentActiveException, tries=5, delay=60) # type: ignore def _deploy( self, location: str, @@ -1823,8 +1768,6 @@ def _deploy( f"provisioning failed for an internal error, try to run case. " f"Exception: {error_message}" ) - elif "ResourceExistsError" in error_message: - raise DeploymentActiveException(error_message) else: try: self._save_console_log_and_check_panic( From a3b9f0658f51306b374d9b276f39037b2ba19e3f Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Tue, 3 Mar 2026 14:48:40 -0800 Subject: [PATCH 09/19] dpdkutil: import order --- lisa/microsoft/testsuites/dpdk/dpdkutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/microsoft/testsuites/dpdk/dpdkutil.py b/lisa/microsoft/testsuites/dpdk/dpdkutil.py index ff75c4af4e..69a444fe50 100644 --- a/lisa/microsoft/testsuites/dpdk/dpdkutil.py +++ b/lisa/microsoft/testsuites/dpdk/dpdkutil.py @@ -1,3 +1,4 @@ +import ipaddress import itertools import re import time @@ -7,7 +8,6 @@ from functools import partial from pathlib import PurePath from typing import Any, Dict, List, Optional, Tuple, Union -import ipaddress from assertpy import assert_that, fail from microsoft.testsuites.dpdk.common import ( From cc50d5ed8c29b9ddfa670efbd2c4e89caca44b6f Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Tue, 3 Mar 2026 14:59:32 -0800 Subject: [PATCH 10/19] platform: remove subnet_prefix since it's deprecated --- lisa/sut_orchestrator/azure/platform_.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 922fa56f77..3cb5916b23 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -8,6 +8,7 @@ import os import re import sys +import threading from copy import deepcopy from dataclasses import InitVar, dataclass, field from datetime import datetime @@ -483,6 +484,8 @@ def __init__(self, runbook: schema.Platform) -> None: platform_utils.KEY_MSHV_VERSION: platform_utils.get_mshv_version, } + self._private_key_lock = threading.Lock() + @classmethod def type_name(cls) -> str: return AZURE @@ -1213,9 +1216,7 @@ def _create_deployment_parameters( arm_parameters.virtual_network_resource_group = ( self._azure_runbook.virtual_network_resource_group ) - arm_parameters.subnet_prefix = ( - self._azure_runbook.subnet_prefix or AZURE_SUBNET_PREFIX - ) + arm_parameters.virtual_network_name = ( self._azure_runbook.virtual_network_name or AZURE_VIRTUAL_NETWORK_NAME ) @@ -1224,8 +1225,14 @@ def _create_deployment_parameters( is_windows: bool = False arm_parameters.admin_username = self.runbook.admin_username # if no key or password specified, generate the key pair - if not self.runbook.admin_private_key_file and not self.runbook.admin_password: - self.runbook.admin_private_key_file = get_or_generate_key_pairs(self._log) + with self._private_key_lock: + if ( + not self.runbook.admin_private_key_file + and not self.runbook.admin_password + ): + self.runbook.admin_private_key_file = get_or_generate_key_pairs( + self._log + ) if self.runbook.admin_private_key_file: arm_parameters.admin_key_data = get_public_key_data( From c5d06b2dc0bac342b3174af7777b4d32e5ea7a3f Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Tue, 3 Mar 2026 20:08:03 -0800 Subject: [PATCH 11/19] platform: re-add timeout during deployment for concurrent modifications to vnet --- lisa/sut_orchestrator/azure/autogen_arm_template.json | 2 +- lisa/sut_orchestrator/azure/platform_.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index cf44284695..64bd782028 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -652,7 +652,7 @@ "description": "Allows nested VM SSH traffic", "protocol": "Tcp", "sourcePortRange": "*", - "destinationPortRange": "60020-60030", + "destinationPortRange": "22", "destinationAddressPrefix": "*", "sourceAddressPrefixes": "[parameters('source_address_prefixes')]", "access": "Allow", diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 3cb5916b23..6ec48ac5ed 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -68,6 +68,7 @@ from lisa.tools import Hostname, KernelConfig, Modinfo, Whoami from lisa.tools.lsinitrd import Lsinitrd from lisa.util import ( + DeploymentActiveException, KernelPanicException, LisaException, LisaTimeoutException, @@ -103,7 +104,6 @@ from . import features from .common import ( AZURE_SHARED_RG_NAME, - AZURE_SUBNET_PREFIX, AZURE_VIRTUAL_NETWORK_NAME, SAS_URL_PATTERN, AzureArmParameter, @@ -322,7 +322,6 @@ class AzurePlatformSchema: virtual_network_resource_group: str = field(default="") virtual_network_name: str = field(default=AZURE_VIRTUAL_NETWORK_NAME) - subnet_prefix: str = field(default=AZURE_SUBNET_PREFIX) # Provisioning error causes by waagent is not ready or other reasons. In # smoke test, it can verify some points also. Other tests should use the @@ -1247,6 +1246,9 @@ def _create_deployment_parameters( arm_parameters.vm_tags["lisa_username"] = local().tools[Whoami].get_username() arm_parameters.vm_tags["lisa_hostname"] = local().tools[Hostname].get_hostname() + # pass the rg id to the arm template + arm_parameters.resource_group_index = int(environment.id) + nodes_parameters: List[AzureNodeArmParameter] = [] features_settings: Dict[str, schema.FeatureSettings] = {} @@ -1268,11 +1270,13 @@ def _create_deployment_parameters( azure_node_runbook = self._create_node_runbook( len(nodes_parameters), node_space, log, resource_group_name ) + # save parsed runbook back, for example, the version of marketplace may be # parsed from latest to a specified version. node.capability.set_extended_runbook(azure_node_runbook) node_arm_parameters = self._create_node_arm_parameters(node.capability, log) + nodes_parameters.append(node_arm_parameters) arm_parameters.is_ultradisk = any( @@ -1691,6 +1695,7 @@ def _validate_template( plugin_manager.hook.azure_deploy_failed(error_message=error_message) raise LisaException(error_message) + @retry(DeploymentActiveException, tries=5, delay=30, jitter=(0, 10)) def _deploy( self, location: str, @@ -1775,6 +1780,8 @@ def _deploy( f"provisioning failed for an internal error, try to run case. " f"Exception: {error_message}" ) + elif "DeploymentActive" in error_message: + raise DeploymentActiveException(e) else: try: self._save_console_log_and_check_panic( From 31a3cdf7c4dbe55acc210b05d609376e12778b26 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 02:10:25 -0800 Subject: [PATCH 12/19] paramter parser: undo naming hack --- lisa/parameter_parser/runbook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/parameter_parser/runbook.py b/lisa/parameter_parser/runbook.py index 6fbcbbe715..5e2a7d87c4 100644 --- a/lisa/parameter_parser/runbook.py +++ b/lisa/parameter_parser/runbook.py @@ -93,7 +93,7 @@ def from_path( runbook_name = builder.partial_resolve(constants.NAME) - constants.RUN_NAME = f"{runbook_name}-{constants.RUN_ID}" + constants.RUN_NAME = f"lisa-{runbook_name}-{constants.RUN_ID}" builder._log.info(f"run name is '{constants.RUN_NAME}'") return builder From 8c608d8cc4e66d13dc3b100c658ea9dcafac4468 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 02:12:29 -0800 Subject: [PATCH 13/19] bicep: reset lisassh rule --- .../sut_orchestrator/azure/arm_template.bicep | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index 6985b1640b..9beb0a4b04 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -318,19 +318,18 @@ resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = { location: location properties: { securityRules: [ - { - name: 'LISASSH' - properties: { - description: 'Allows nested VM SSH traffic' - protocol: 'Tcp' - sourcePortRange: '*' - destinationPortRange: '60020-60030' - destinationAddressPrefix: '*' - sourceAddressPrefixes: source_address_prefixes - access: 'Allow' - priority: 101 - direction: 'Inbound' - } + { + name: 'LISASSH' + properties: { + priority: 100 + direction: 'Inbound' + access: 'Allow' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '22' + sourceAddressPrefixes: source_address_prefixes + destinationAddressPrefix: '*' + } } { name: 'LISAKVMSSH' From 0c9f5275383b030a1f1d148dc31678fa31807623 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 02:13:54 -0800 Subject: [PATCH 14/19] arm template: regenerate --- lisa/sut_orchestrator/azure/autogen_arm_template.json | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index 64bd782028..a28f148a08 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.41.2.15936", - "templateHash": "3674176336733772196" + "templateHash": "4261739258007286340" } }, "functions": [ @@ -649,15 +649,14 @@ { "name": "LISASSH", "properties": { - "description": "Allows nested VM SSH traffic", + "priority": 100, + "direction": "Inbound", + "access": "Allow", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "22", - "destinationAddressPrefix": "*", "sourceAddressPrefixes": "[parameters('source_address_prefixes')]", - "access": "Allow", - "priority": 101, - "direction": "Inbound" + "destinationAddressPrefix": "*" } }, { From 34514e8291a48f42c5358c4f7d7e32a5fb7a1d8d Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 02:18:47 -0800 Subject: [PATCH 15/19] comment DeploymentActiveException --- lisa/util/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lisa/util/__init__.py b/lisa/util/__init__.py index e222553b7e..b25d0ef478 100644 --- a/lisa/util/__init__.py +++ b/lisa/util/__init__.py @@ -163,8 +163,10 @@ def __init__(self, *args: object) -> None: class DeploymentActiveException(LisaException): """ - This exception is used to indicate that there is an active deployment with the same name. It may be caused by the previous deployment not cleaned up yet. - LISA should catch this exception and retry the deployment after a timeout period to allow the old environment to be cleaned up. + This exception is used to indicate that there is already an active deployment on a resource. + It may be caused by the previous deployment not cleaned up yet, a deployment adding a resource to an existing resource, + or a deployment with the same name as another deployment. + This is a retryable exception: LISA can catch this exception retry the deployment after a timeout period. """ ... From 59fca67d7c49de592f501bb3d3afa77ef5c56fc7 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 03:08:35 -0800 Subject: [PATCH 16/19] docs: update docs for virtual_network_resource_group --- docs/run_test/platform.rst | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/run_test/platform.rst b/docs/run_test/platform.rst index ee4f80ce79..89ac85060c 100644 --- a/docs/run_test/platform.rst +++ b/docs/run_test/platform.rst @@ -192,6 +192,7 @@ deployment. azure: virtual_network_resource_group: $(virtual_network_resource_group) virtual_network_name: $(virtual_network_name) + subnet_prefix: $(subnet_prefix) use_public_address: "" create_public_address: "" use_ipv6: "" @@ -234,8 +235,35 @@ deployment. created and the resulting virtual network name will be ``. If `virtual_network_resource_group` is provided, an existing virtual network, with the name equal to `virtual_network_name`, - will be used. -* **use_public_address**. True means to connect to the Azure VMs with their + will be used. `virtual_network_resource_group` does not change the behavior for + test resource group creation. Instead, the subnets in the test RGs vnet will be + peered to a subnet within the RG `virtual_network_resource_group`. Peering requires + the subnets and vnet have no address space collisions; a default schema + assuming a remote subnet named 'default' with the address space 10.255.255.0/24 is used + to allow a large number of test resource groups to be created without collisions. + Virtual networks in the test environments will set the default subnet prefix of: + 10.$(environment_id/256).$(environment_id%256).0/24; any additional subnets will use the prefix + 192.168.${nic_index}.0/24. + This scheme allows for 0xFFFE test environments with up to 256 nics per VM; however, + Azure will likely restrict these numbers to something smaller than the maximum. + LISA does not remove the subnet peerings from the remote vnet in `virtual_network_resource_group`. + The total number of test environments will be limited based on the allowed active subnet peerings per vnet. + This complex behavior is intended to enable testing without exposing a public IP address; + since a VM in `virtual_network_resource_group` will be able to access the test environments + via SSH on the private network. This assumes an automated deployment of this orchestrator resource group, + VM and virtual network with the expected default values. '`` and `` are respected. + Note that usage of `` with `virtual_network_resource_group` will likely result in address space + collisions and failed deployments. Similarly; use of `` with this option will likely + result in failed test resource deployments. +* **subnet_prefix**. Specify the desired subnet prefix. If + `virtual_network_resource_group` is not provided, a virtual network and + subnet will be created and the resulting subnets will look like + `0`, `1`, and so on. + The '`` option will likely conflict with the use of ``. + LISA will warn of this configuration but allow it's use, see notes for `` + for more details. + + * **use_public_address**. True means to connect to the Azure VMs with their public IP addresses. False means to connect with the private IP addresses. If not provided, the connections will default to using the public IP addresses. From 6941f2561e26fcbbbcb8cbea81333ccab448ea05 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 08:15:47 -0800 Subject: [PATCH 17/19] clear old subnets and handle deployment issues --- lisa/microsoft/runbook/azure.yml | 4 ++-- .../sut_orchestrator/azure/arm_template.bicep | 13 +++++++----- .../azure/autogen_arm_template.json | 14 +++++++++---- lisa/sut_orchestrator/azure/common.py | 21 +++++++++++++++++++ lisa/sut_orchestrator/azure/platform_.py | 17 ++++++++++++++- 5 files changed, 57 insertions(+), 12 deletions(-) diff --git a/lisa/microsoft/runbook/azure.yml b/lisa/microsoft/runbook/azure.yml index 257f8ee3c3..17f3a80475 100644 --- a/lisa/microsoft/runbook/azure.yml +++ b/lisa/microsoft/runbook/azure.yml @@ -45,14 +45,14 @@ variable: value: "" - name: virtual_network_name value: "" - - name: subnet_prefix - value: "" - name: use_public_address value: true - name: create_public_address value: true - name: resource_group_tags value: null + - name: subnet_prefix + value: "" # Example usage: # resource_group_tags: # Environment: Testing diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index 9beb0a4b04..127cbc1ea2 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -22,6 +22,9 @@ param shared_resource_group_name string @description('created subnet count') param subnet_count int +@description('user supplied subnet prefix (incompatible with virtual_network_resource_group)') +param subnet_prefix string + @description('index of the test resource group for shared vnet subnet mapping') param resource_group_index int @@ -295,15 +298,15 @@ resource virtual_network 'Microsoft.Network/virtualNetworks@2024-05-01' = { location: location properties: { addressSpace: { - addressPrefixes: concat( + addressPrefixes: empty(subnet_prefix) ? (concat( ['10.${rg_index_div_256}.${rg_index_mod_256}.0/24', '192.168.0.0/16' ], use_ipv6 ? ['2001:db8::/32'] : [] - ) - } + )) : [ subnet_prefix ] + } subnets: [ for i in range(0,subnet_count): { - name: i==0 ? 'default' : 'test-subnet-${i}' + name: empty(subnet_prefix) ? (i==0 ? 'default' : 'test-subnet-${i}') : subnet_prefix properties: { - addressPrefix: i==0 ? '10.${rg_index_div_256}.${rg_index_mod_256}.0/24' : '192.168.${i-1}.0/24' + addressPrefix: empty(subnet_prefix) ? (i==0 ? '10.${rg_index_div_256}.${rg_index_mod_256}.0/24' : '192.168.${i-1}.0/24' ) : subnet_prefix defaultOutboundAccess: enable_vm_nat networkSecurityGroup:{ id: nsg.id diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index a28f148a08..ad90122aa7 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.41.2.15936", - "templateHash": "4261739258007286340" + "templateHash": "8018757189124818305" } }, "functions": [ @@ -463,6 +463,12 @@ "description": "created subnet count" } }, + "subnet_prefix": { + "type": "string", + "metadata": { + "description": "user supplied subnet prefix (incompatible with virtual_network_resource_group)" + } + }, "resource_group_index": { "type": "int", "metadata": { @@ -620,9 +626,9 @@ "name": "subnets", "count": "[length(range(0, parameters('subnet_count')))]", "input": { - "name": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), 'default', format('test-subnet-{0}', range(0, parameters('subnet_count'))[copyIndex('subnets')]))]", + "name": "[if(empty(parameters('subnet_prefix')), if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), 'default', format('test-subnet-{0}', range(0, parameters('subnet_count'))[copyIndex('subnets')])), parameters('subnet_prefix'))]", "properties": { - "addressPrefix": "[if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), format('192.168.{0}.0/24', sub(range(0, parameters('subnet_count'))[copyIndex('subnets')], 1)))]", + "addressPrefix": "[if(empty(parameters('subnet_prefix')), if(equals(range(0, parameters('subnet_count'))[copyIndex('subnets')], 0), format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), format('192.168.{0}.0/24', sub(range(0, parameters('subnet_count'))[copyIndex('subnets')], 1))), parameters('subnet_prefix'))]", "defaultOutboundAccess": "[parameters('enable_vm_nat')]", "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', format('lisa-test-nsg-{0}', parameters('resource_group_index')))]" @@ -632,7 +638,7 @@ } ], "addressSpace": { - "addressPrefixes": "[concat(createArray(format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), '192.168.0.0/16'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray()))]" + "addressPrefixes": "[if(empty(parameters('subnet_prefix')), concat(createArray(format('10.{0}.{1}.0/24', variables('rg_index_div_256'), variables('rg_index_mod_256')), '192.168.0.0/16'), if(parameters('use_ipv6'), createArray('2001:db8::/32'), createArray())), createArray(parameters('subnet_prefix')))]" } }, "dependsOn": [ diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index a5af54c7d5..63234ae763 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -1263,6 +1263,7 @@ class AzureArmParameter: create_public_address: bool = True source_address_prefixes: List[str] = field(default_factory=list) resource_group_index: Optional[int] = None + subnet_prefix: str = field(default="") def __post_init__(self, *args: Any, **kwargs: Any) -> None: add_secret(self.admin_username, PATTERN_HEADTAIL) @@ -1710,6 +1711,26 @@ def get_virtual_networks( ] return virtual_network_dict +def remove_vnet_peering(platform: "AzurePlatform", resource_group_name:str)-> None: + # delete both the vnet peering for a resource group and the corresponding link + # in the remote group. + network_client = get_network_client(platform) + vnets = network_client.virtual_networks.list( + resource_group_name=resource_group_name + ) + for vnet in vnets: + peerings = network_client.virtual_network_peerings.get(resource_group_name, name=vnet.name) + for peering in peerings: + network_client.virtual_network_peerings.begin_delete(resource_group_name, vnet.name, peering.name).wait() + + if not delete_op.done(): + platform._log.debug(f"Delete peering operation failed for peering: {peering.name}") + + remote_peering = self._rm_client.get(id: peering.remote_virtual_network) + remote_peering = network_client.virtual_network_peerings.begin_delete(resource_group_name, peering.remote_virtual_network.split('/')[-1], peering.name).wait() + for subnet in vnet.subnets: + network_client.subnets.begin_delete(resource_group_name, vnet.name, subnet.name).wait() + def get_network_client(platform: "AzurePlatform") -> NetworkManagementClient: return NetworkManagementClient( diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 6ec48ac5ed..857345470a 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -131,6 +131,8 @@ get_primary_ip_addresses, get_resource_management_client, get_static_access_token, + get_virtual_networks, + remove_vnet_peerings, get_storage_account_name, get_vhd_details, get_vm, @@ -309,6 +311,7 @@ class AzurePlatformSchema: ) vm_tags: Optional[Dict[str, Any]] = field(default=None) tags: Optional[Dict[str, Any]] = field(default=None) + subnet_prefix: Optional[str] = field(default=None) use_public_address: bool = field(default=True) create_public_address: bool = field(default=True) use_ipv6: bool = field(default=False) @@ -700,6 +703,8 @@ def _delete_environment(self, environment: Environment, log: Logger) -> None: delete_operation = self._rm_client.resource_groups.begin_delete( resource_group_name ) + if self._azure_runbook.virtual_network_resource_group: + remove_vnet_peerings(self._platform, resource_group_name) except Exception as e: log.debug(f"exception on delete resource group: {e}") if delete_operation and self._azure_runbook.wait_delete: @@ -1219,6 +1224,16 @@ def _create_deployment_parameters( arm_parameters.virtual_network_name = ( self._azure_runbook.virtual_network_name or AZURE_VIRTUAL_NETWORK_NAME ) + arm_parameters.subnet_prefix = self._azure_runbook.subnet_prefix or "" + if ( + arm_parameters.subnet_prefix + and arm_parameters.virtual_network_resource_group + ): + log.warn( + "subnet_prefix and virtual_network_resource_group runbook options " + "may introduce unexpected failures due to network peering " + "address prefix collisions." + ) arm_parameters.use_ipv6 = self._azure_runbook.use_ipv6 is_windows: bool = False @@ -1695,7 +1710,7 @@ def _validate_template( plugin_manager.hook.azure_deploy_failed(error_message=error_message) raise LisaException(error_message) - @retry(DeploymentActiveException, tries=5, delay=30, jitter=(0, 10)) + @retry(DeploymentActiveException, tries=5, delay=30, jitter=(0, 10)) # type: ignore def _deploy( self, location: str, From c68df0d0c40011c2163ba0a2b043c4343fb1bb5b Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 13:04:02 -0800 Subject: [PATCH 18/19] cleanup platform changes, still needs a lock --- lisa/sut_orchestrator/azure/common.py | 37 ++++++++++++++++-------- lisa/sut_orchestrator/azure/platform_.py | 8 ++--- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index 63234ae763..a198793e6a 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -1711,26 +1711,39 @@ def get_virtual_networks( ] return virtual_network_dict -def remove_vnet_peering(platform: "AzurePlatform", resource_group_name:str)-> None: + +def remove_vnet_peerings(platform: "AzurePlatform", resource_group_name: str) -> None: # delete both the vnet peering for a resource group and the corresponding link # in the remote group. network_client = get_network_client(platform) vnets = network_client.virtual_networks.list( - resource_group_name=resource_group_name - ) + resource_group_name=resource_group_name + ) for vnet in vnets: - peerings = network_client.virtual_network_peerings.get(resource_group_name, name=vnet.name) + peerings = network_client.virtual_network_peerings.get( + resource_group_name, name=vnet.name + ) for peering in peerings: - network_client.virtual_network_peerings.begin_delete(resource_group_name, vnet.name, peering.name).wait() - + network_client.virtual_network_peerings.begin_delete( + resource_group_name, vnet.name, peering.name + ).wait() + if not delete_op.done(): - platform._log.debug(f"Delete peering operation failed for peering: {peering.name}") - - remote_peering = self._rm_client.get(id: peering.remote_virtual_network) - remote_peering = network_client.virtual_network_peerings.begin_delete(resource_group_name, peering.remote_virtual_network.split('/')[-1], peering.name).wait() + platform._log.debug( + f"Delete peering operation failed for peering: {peering.name}" + ) + + remote_peering = self._rm_client.get(peering.remote_virtual_network) + remote_peering = network_client.virtual_network_peerings.begin_delete( + resource_group_name, + peering.remote_virtual_network.split("/")[-1], + peering.name, + ).wait() for subnet in vnet.subnets: - network_client.subnets.begin_delete(resource_group_name, vnet.name, subnet.name).wait() - + network_client.subnets.begin_delete( + resource_group_name, vnet.name, subnet.name + ).wait() + def get_network_client(platform: "AzurePlatform") -> NetworkManagementClient: return NetworkManagementClient( diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 857345470a..bb6c554d14 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -131,13 +131,13 @@ get_primary_ip_addresses, get_resource_management_client, get_static_access_token, - get_virtual_networks, - remove_vnet_peerings, get_storage_account_name, get_vhd_details, + get_virtual_networks, get_vm, global_credential_access_lock, load_location_info_from_file, + remove_vnet_peerings, save_console_log, wait_operation, ) @@ -703,8 +703,8 @@ def _delete_environment(self, environment: Environment, log: Logger) -> None: delete_operation = self._rm_client.resource_groups.begin_delete( resource_group_name ) - if self._azure_runbook.virtual_network_resource_group: - remove_vnet_peerings(self._platform, resource_group_name) + if self._azure_runbook.virtual_network_resource_group: + remove_vnet_peerings(self._platform, resource_group_name) except Exception as e: log.debug(f"exception on delete resource group: {e}") if delete_operation and self._azure_runbook.wait_delete: From 4e1fbad1780fdd3ba0e333ed153365c4b5f07160 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 4 Mar 2026 13:09:51 -0800 Subject: [PATCH 19/19] fix remote vnet peering name --- lisa/sut_orchestrator/azure/common.py | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index a198793e6a..d6b922b615 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -1712,9 +1712,14 @@ def get_virtual_networks( return virtual_network_dict -def remove_vnet_peerings(platform: "AzurePlatform", resource_group_name: str) -> None: +def remove_vnet_peerings( + platform: "AzurePlatform", resource_group_name: str, environment_id: int +) -> None: # delete both the vnet peering for a resource group and the corresponding link # in the remote group. + # peerings for test resources in our vnet sharing scheme will only be + # peered with one remote VM vnet for orchestration; so we only need to find + # one corresponding peering on the remote side. network_client = get_network_client(platform) vnets = network_client.virtual_networks.list( resource_group_name=resource_group_name @@ -1724,25 +1729,20 @@ def remove_vnet_peerings(platform: "AzurePlatform", resource_group_name: str) -> resource_group_name, name=vnet.name ) for peering in peerings: + # remove the local peerings network_client.virtual_network_peerings.begin_delete( resource_group_name, vnet.name, peering.name ).wait() - - if not delete_op.done(): - platform._log.debug( - f"Delete peering operation failed for peering: {peering.name}" - ) - - remote_peering = self._rm_client.get(peering.remote_virtual_network) - remote_peering = network_client.virtual_network_peerings.begin_delete( - resource_group_name, + # remove the remote peering + network_client.virtual_network_peerings.begin_delete( + peering.remote_virtual_network.split("/")[4], peering.remote_virtual_network.split("/")[-1], - peering.name, - ).wait() - for subnet in vnet.subnets: - network_client.subnets.begin_delete( - resource_group_name, vnet.name, subnet.name + f"vnet-peering-e{environment_id}", ).wait() + # for subnet in vnet.subnets: + # network_client.subnets.begin_delete( + # resource_group_name, vnet.name, subnet.name + # ).wait() def get_network_client(platform: "AzurePlatform") -> NetworkManagementClient: