From 6fb102c0008258dd72da03016e958c62a4409adc Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 17 Mar 2026 14:33:36 -0700 Subject: [PATCH 1/8] Add MAUI desktop BenchmarkDotNet benchmarks scenario New scenario that clones dotnet/maui, builds dependencies, patches in PerfLabExporter via BenchmarkDotNet.Extensions, and runs the Core, XAML, and Graphics BDN benchmark suites on desktop (Windows). Key implementation details: - Sparse-clone of dotnet/maui with depth 1 (only needed directories) - Patches Directory.Build.props to disable non-desktop TFMs so all builds (including BDN internal builds) work without mobile workloads - Removes MAUI BDN PackageReference, injects BDN.Extensions ProjectRef - Patches Program.cs with ManualConfig + PerfLabExporter - Branch mapping: net11.0 to net11.0, net10.0 to net10.0 - All heavy work in test.py to keep correlation payload small - Pipeline entries disabled until validated in CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_desktop_benchmarks.proj | 19 + eng/pipelines/sdk-perf-jobs.yml | 32 ++ src/scenarios/mauiDesktopBenchmarks/post.py | 30 ++ src/scenarios/mauiDesktopBenchmarks/pre.py | 19 + src/scenarios/mauiDesktopBenchmarks/test.py | 403 +++++++++++++++++++ 5 files changed, 503 insertions(+) create mode 100644 eng/performance/maui_desktop_benchmarks.proj create mode 100644 src/scenarios/mauiDesktopBenchmarks/post.py create mode 100644 src/scenarios/mauiDesktopBenchmarks/pre.py create mode 100644 src/scenarios/mauiDesktopBenchmarks/test.py diff --git a/eng/performance/maui_desktop_benchmarks.proj b/eng/performance/maui_desktop_benchmarks.proj new file mode 100644 index 00000000000..00aa89b64a8 --- /dev/null +++ b/eng/performance/maui_desktop_benchmarks.proj @@ -0,0 +1,19 @@ + + + + + + + + + $(ScenariosDir)mauiDesktopBenchmarks + $(Python) pre.py -f $(PERFLAB_Framework) + $(Python) test.py -f $(PERFLAB_Framework) --suite all + $(Python) post.py + + + + + diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..4ee4268a2f5 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -56,6 +56,22 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # MAUI Desktop BenchmarkDotNet benchmarks + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + isPublic: true + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Blazor scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: @@ -586,6 +602,22 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # MAUI Desktop BDN benchmarks (private) + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + isPublic: false + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # NativeAOT scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: diff --git a/src/scenarios/mauiDesktopBenchmarks/post.py b/src/scenarios/mauiDesktopBenchmarks/post.py new file mode 100644 index 00000000000..69cd572c68b --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/post.py @@ -0,0 +1,30 @@ +''' +Post-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Cleans up the cloned maui repo and temporary artifacts. +''' +import os +import shutil +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +MAUI_REPO_DIR = 'maui_repo' + + +def cleanup(): + """Remove the cloned maui repository and any leftover artifacts.""" + if os.path.exists(MAUI_REPO_DIR): + log.info(f'Removing cloned MAUI repo: {MAUI_REPO_DIR}') + shutil.rmtree(MAUI_REPO_DIR, ignore_errors=True) + + # Clean up combined report if still in working directory + combined = 'combined-perf-lab-report.json' + if os.path.exists(combined): + os.remove(combined) + + log.info('Post-commands cleanup complete.') + + +if __name__ == '__main__': + cleanup() diff --git a/src/scenarios/mauiDesktopBenchmarks/pre.py b/src/scenarios/mauiDesktopBenchmarks/pre.py new file mode 100644 index 00000000000..99c7e37499f --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/pre.py @@ -0,0 +1,19 @@ +''' +Pre-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Kept minimal — all heavy lifting (clone, build, patch, run) is in test.py +to keep the correlation payload small. +''' +import argparse +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='MAUI Desktop BDN Benchmarks - Pre-commands') + parser.add_argument('-f', '--framework', default='net11.0', + help='Target .NET framework (determines MAUI branch)') + args = parser.parse_args() + log.info(f'MAUI Desktop BDN Benchmarks pre-commands (framework={args.framework})') + log.info('Setup deferred to test.py to minimize correlation payload.') + diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py new file mode 100644 index 00000000000..ca8132f3b41 --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -0,0 +1,403 @@ +''' +Test runner for MAUI Desktop BenchmarkDotNet benchmarks. + +Handles the full lifecycle: clone dotnet/maui, build dependencies, patch +benchmark projects for PerfLabExporter, run BDN suites, and collect results. +All heavy work lives here (not in pre.py) to keep the correlation payload small. +''' +import os +import sys +import glob +import json +import shutil +import subprocess +import xml.etree.ElementTree as ET +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +MAUI_REPO_DIR = 'maui_repo' +MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' + +BENCHMARK_PROJECTS = { + 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', + 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', + 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', +} + +# MSBuild properties to disable non-desktop target frameworks. +# Used by patch_directory_build_props() to replace true→false in-place +# in MAUI's Directory.Build.props. This ensures ALL builds (including BDN's +# internal auto-generated project builds) are desktop-only. +DESKTOP_ONLY_PROPS = { + 'IncludeAndroidTargetFrameworks': 'false', + 'IncludeIosTargetFrameworks': 'false', + 'IncludeMacCatalystTargetFrameworks': 'false', + 'IncludeMacOSTargetFrameworks': 'false', + 'IncludeTizenTargetFrameworks': 'false', +} + +# ── Branch mapping ────────────────────────────────────────────────────────── + +def get_maui_branch(framework: str) -> str: + """Map .NET framework version to MAUI repo branch name.""" + if framework and framework.startswith('net'): + return framework # net11.0 -> net11.0, net10.0 -> net10.0 + return 'net11.0' + +# ── Clone & build ─────────────────────────────────────────────────────────── + +def clone_maui_repo(branch: str): + """Sparse-clone the maui repo with only the directories needed for benchmarks.""" + log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') + + if os.path.exists(MAUI_REPO_DIR): + shutil.rmtree(MAUI_REPO_DIR) + + subprocess.run([ + 'git', 'clone', + '-c', 'core.longpaths=true', + '--depth', '1', + '--filter=blob:none', + '--sparse', + '--branch', branch, + MAUI_REPO_URL, + MAUI_REPO_DIR + ], check=True) + + subprocess.run([ + 'git', 'sparse-checkout', 'set', + 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', + 'src/Workload', 'src/Essentials', + 'eng', '.config' + ], cwd=MAUI_REPO_DIR, check=True) + + log.info('Clone complete.') + + +def patch_directory_build_props(): + """Disable non-desktop TFMs in Directory.Build.props. + + MAUI's props set Include*TargetFrameworks to true at multiple points. + MauiPlatforms (which controls TargetFrameworks) is computed from these. + We must replace ALL true→false in-place so the platform lists are never + populated, not just append overrides at the end (too late for evaluation). + """ + import re + props_path = os.path.join(MAUI_REPO_DIR, 'Directory.Build.props') + log.info('Patching Directory.Build.props to disable non-desktop TFMs...') + + with open(props_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + for prop_name in DESKTOP_ONLY_PROPS: + # Replace all occurrences of true with false + pattern = rf'(<{prop_name}\b[^>]*>)true()' + content, count = re.subn(pattern, r'\g<1>false\g<2>', content) + if count > 0: + log.info(f' {prop_name}: replaced {count} occurrence(s)') + + with open(props_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Directory.Build.props patched.') + + +def build_maui_dependencies(): + """Build MAUI BuildTasks solution filter — the core libraries benchmarks depend on.""" + log.info('Restoring dotnet tools...') + subprocess.run(['dotnet', 'tool', 'restore'], cwd=MAUI_REPO_DIR, check=True) + + log.info('Building Microsoft.Maui.BuildTasks.slnf (desktop TFMs only)...') + subprocess.run([ + 'dotnet', 'build', + 'Microsoft.Maui.BuildTasks.slnf', + '-c', 'Release', + ], cwd=MAUI_REPO_DIR, check=True) + + log.info('MAUI dependencies built successfully.') + +# ── BDN.Extensions injection ──────────────────────────────────────────────── + +def _find_bdn_extensions() -> str: + """ + Return the absolute path to BenchmarkDotNet.Extensions.csproj. + On Helix the full perf repo lives under the correlation payload; + for local runs we walk up from this script's location. + """ + correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') + if correlation: + # Helix: repo is at /performance/ + candidate = os.path.join( + correlation, 'performance', 'src', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') + else: + # Local: navigate from this script's directory + scenario_dir = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.normpath(os.path.join( + scenario_dir, '..', '..', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) + + if not os.path.exists(candidate): + raise FileNotFoundError( + f'BenchmarkDotNet.Extensions.csproj not found at {candidate}. ' + f'HELIX_CORRELATION_PAYLOAD={correlation!r}') + + log.info(f'BDN.Extensions located at: {candidate}') + return candidate + + +def inject_bdn_extensions(csproj_path: str, bdn_ext_abs: str): + """Add a ProjectReference to BenchmarkDotNet.Extensions into a benchmark csproj. + Also removes the existing BenchmarkDotNet PackageReference to avoid version conflicts + (BDN.Extensions brings in the correct version transitively).""" + log.info(f'Injecting BDN.Extensions reference into {os.path.basename(csproj_path)}') + + csproj_dir = os.path.dirname(os.path.abspath(csproj_path)) + bdn_ext_rel = os.path.relpath(bdn_ext_abs, csproj_dir) + + tree = ET.parse(csproj_path) + root = tree.getroot() + + # SDK-style projects typically have no XML namespace + ns = '' + if root.tag.startswith('{'): + ns = root.tag.split('}')[0] + '}' + + # Remove existing BenchmarkDotNet PackageReference to avoid NU1605 downgrade error + for item_group in root.findall(f'{ns}ItemGroup'): + for pkg_ref in item_group.findall(f'{ns}PackageReference'): + include = pkg_ref.get('Include', '') + if include.startswith('BenchmarkDotNet'): + item_group.remove(pkg_ref) + log.info(f' Removed PackageReference: {include}') + + # Add ProjectReference to BDN.Extensions + item_group = ET.SubElement(root, f'{ns}ItemGroup') + item_group.set('Label', 'PerfLabInjected') + proj_ref = ET.SubElement(item_group, f'{ns}ProjectReference') + proj_ref.set('Include', bdn_ext_rel) + + tree.write(csproj_path, xml_declaration=True, encoding='utf-8') + log.info(f' Added ProjectReference: {bdn_ext_rel}') + + +def patch_program_cs(program_cs_path: str): + """Patch Program.cs to add PerfLabExporter via ManualConfig.""" + log.info(f'Patching {os.path.basename(program_cs_path)} for PerfLabExporter') + + with open(program_cs_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + # Add required using statements at the top + usings_to_add = [ + 'using BenchmarkDotNet.Configs;', + 'using BenchmarkDotNet.Extensions;', + 'using System;', + 'using System.IO;', + ] + insert_block = '' + for u in usings_to_add: + if u not in content: + insert_block += u + '\n' + if insert_block: + content = insert_block + content + + # Build a minimal config that adds PerfLabExporter without the + # MandatoryCategoryValidator (MAUI benchmarks don't use BenchmarkCategory). + new_run_call = ( + 'var config = ManualConfig.Create(DefaultConfig.Instance)\n' + ' .WithArtifactsPath(Path.Combine(\n' + ' Path.GetDirectoryName(typeof(Program).Assembly.Location),\n' + ' "BenchmarkDotNet.Artifacts"));\n' + ' if (Environment.GetEnvironmentVariable("PERFLAB_INLAB") == "1")\n' + ' config = config.AddExporter(new PerfLabExporter());\n' + ' BenchmarkSwitcher\n' + ' .FromAssembly(typeof(Program).Assembly)\n' + ' .Run(args, config)' + ) + + # Known patterns from MAUI benchmark Program.cs files + patterns = [ + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);', + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args)', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args);', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args)', + ] + + replaced = False + for pattern in patterns: + if pattern in content: + suffix = ';' if pattern.endswith(';') else '' + content = content.replace(pattern, new_run_call + suffix) + replaced = True + break + + if not replaced: + log.warning(f' Could not find BenchmarkSwitcher.Run pattern — may need manual patching') + return + + with open(program_cs_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(f' Patched successfully.') + + +def patch_benchmark_projects(): + """Inject BDN.Extensions and RecommendedConfig into all benchmark projects.""" + bdn_ext_abs = _find_bdn_extensions() + + for name, csproj_rel in BENCHMARK_PROJECTS.items(): + csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) + project_dir = os.path.dirname(csproj_path) + program_cs = os.path.join(project_dir, 'Program.cs') + + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found: {csproj_path}') + continue + + inject_bdn_extensions(csproj_path, bdn_ext_abs) + + if os.path.exists(program_cs): + patch_program_cs(program_cs) + else: + log.warning(f'Program.cs not found for {name}') + +# ── Build benchmarks ──────────────────────────────────────────────────────── + +def build_benchmark_projects(): + """Build each benchmark project in Release mode.""" + for name, csproj_rel in BENCHMARK_PROJECTS.items(): + csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) + if not os.path.exists(csproj_path): + continue + + log.info(f'Building benchmark: {name}') + subprocess.run([ + 'dotnet', 'build', + csproj_rel, + '-c', 'Release', + ], cwd=MAUI_REPO_DIR, check=True) + + log.info('All benchmark projects built successfully.') + +# ── Run benchmarks ────────────────────────────────────────────────────────── + +def run_benchmark(name: str, csproj_rel: str, extra_bdn_args: list): + """Run a single BDN benchmark suite.""" + csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found: {csproj_path}') + return False + + log.info(f'Running benchmark suite: {name}') + + cmd = [ + 'dotnet', 'run', + '-c', 'Release', + '--no-build', + '--project', csproj_rel, + '--', + '--filter', '*', + ] + extra_bdn_args + + result = subprocess.run(cmd, cwd=MAUI_REPO_DIR) + if result.returncode != 0: + log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') + return False + + log.info(f'Benchmark suite {name} completed successfully.') + return True + +# ── Result collection ─────────────────────────────────────────────────────── + +def collect_results(): + """ + Collect perf-lab-report.json files from BDN artifacts and copy them + to HELIX_WORKITEM_UPLOAD_ROOT for the infrastructure to pick up. + """ + upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') + + # Search recursively under maui_repo for perf-lab-report files + report_pattern = os.path.join(MAUI_REPO_DIR, '**', '*-perf-lab-report.json') + report_files = glob.glob(report_pattern, recursive=True) + + if not report_files: + log.warning('No perf-lab-report.json files found. ' + 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') + return + + log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') + + # Combine all reports into a single file + combined = [] + for report_file in report_files: + log.info(f' Collecting: {report_file}') + try: + with open(report_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + combined.extend(data) + else: + combined.append(data) + except (json.JSONDecodeError, IOError) as e: + log.warning(f' Failed to read {report_file}: {e}') + + if combined: + combined_path = 'combined-perf-lab-report.json' + with open(combined_path, 'w', encoding='utf-8') as f: + json.dump(combined, f, indent=2) + log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') + + if upload_root: + dest = os.path.join(upload_root, combined_path) + shutil.copy2(combined_path, dest) + log.info(f'Copied combined report to {dest}') + + for report_file in report_files: + basename = os.path.basename(report_file) + dest = os.path.join(upload_root, basename) + shutil.copy2(report_file, dest) + log.info(f' Copied {basename} to upload root') + +# ── Main ──────────────────────────────────────────────────────────────────── + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description='MAUI Desktop BDN Benchmarks - Test runner') + parser.add_argument('-f', '--framework', default='net11.0', + help='Target .NET framework (determines MAUI branch)') + parser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], + default='all', help='Which benchmark suite to run') + parser.add_argument('--bdn-args', nargs='*', default=[], + help='Additional arguments to pass to BenchmarkDotNet') + args = parser.parse_args() + + # ── Setup ─────────────────────────────────────────────────────────────── + branch = get_maui_branch(args.framework) + clone_maui_repo(branch) + patch_directory_build_props() + build_maui_dependencies() + patch_benchmark_projects() + build_benchmark_projects() + + # ── Run ───────────────────────────────────────────────────────────────── + if args.suite == 'all': + suites = BENCHMARK_PROJECTS.items() + else: + suites = [(args.suite, BENCHMARK_PROJECTS[args.suite])] + + all_passed = True + for name, csproj_rel in suites: + if not run_benchmark(name, csproj_rel, args.bdn_args): + all_passed = False + + # ── Collect ───────────────────────────────────────────────────────────── + collect_results() + + if not all_passed: + log.error('One or more benchmark suites failed.') + sys.exit(1) + + log.info('All benchmark suites completed.') From aea1c27e2ed73ea813a39fe9f8b7f58d0feae4a1 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 17 Mar 2026 16:14:06 -0700 Subject: [PATCH 2/8] Refactor MAUI desktop BDN benchmarks to use shared Runner infrastructure Move lifecycle logic (clone, patch, build, run, collect) from standalone test.py into shared/bdndesktop.py BDNDesktopHelper class, following the same pattern as AndroidInstrumentationHelper. Add BDNDESKTOP test type to const.py and runner.py subparser routing. test.py is now 16 lines, delegating to Runner(traits).run(). Tested locally: Graphics.Benchmarks ran successfully via runner path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_desktop_benchmarks.proj | 2 +- src/scenarios/mauiDesktopBenchmarks/test.py | 407 +----------------- src/scenarios/shared/bdndesktop.py | 412 +++++++++++++++++++ src/scenarios/shared/const.py | 1 + src/scenarios/shared/runner.py | 20 + 5 files changed, 444 insertions(+), 398 deletions(-) create mode 100644 src/scenarios/shared/bdndesktop.py diff --git a/eng/performance/maui_desktop_benchmarks.proj b/eng/performance/maui_desktop_benchmarks.proj index 00aa89b64a8..6233b7b722a 100644 --- a/eng/performance/maui_desktop_benchmarks.proj +++ b/eng/performance/maui_desktop_benchmarks.proj @@ -10,7 +10,7 @@ $(ScenariosDir)mauiDesktopBenchmarks $(Python) pre.py -f $(PERFLAB_Framework) - $(Python) test.py -f $(PERFLAB_Framework) --suite all + $(Python) test.py bdndesktop --framework $(PERFLAB_Framework) --suite all $(Python) post.py diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index ca8132f3b41..88d148148e7 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -1,403 +1,16 @@ ''' -Test runner for MAUI Desktop BenchmarkDotNet benchmarks. +MAUI Desktop BenchmarkDotNet benchmarks. -Handles the full lifecycle: clone dotnet/maui, build dependencies, patch -benchmark projects for PerfLabExporter, run BDN suites, and collect results. -All heavy work lives here (not in pre.py) to keep the correlation payload small. -''' -import os -import sys -import glob -import json -import shutil -import subprocess -import xml.etree.ElementTree as ET -from performance.logger import setup_loggers, getLogger - -setup_loggers(True) -log = getLogger(__name__) - -MAUI_REPO_DIR = 'maui_repo' -MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' - -BENCHMARK_PROJECTS = { - 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', - 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', - 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', -} - -# MSBuild properties to disable non-desktop target frameworks. -# Used by patch_directory_build_props() to replace true→false in-place -# in MAUI's Directory.Build.props. This ensures ALL builds (including BDN's -# internal auto-generated project builds) are desktop-only. -DESKTOP_ONLY_PROPS = { - 'IncludeAndroidTargetFrameworks': 'false', - 'IncludeIosTargetFrameworks': 'false', - 'IncludeMacCatalystTargetFrameworks': 'false', - 'IncludeMacOSTargetFrameworks': 'false', - 'IncludeTizenTargetFrameworks': 'false', -} - -# ── Branch mapping ────────────────────────────────────────────────────────── - -def get_maui_branch(framework: str) -> str: - """Map .NET framework version to MAUI repo branch name.""" - if framework and framework.startswith('net'): - return framework # net11.0 -> net11.0, net10.0 -> net10.0 - return 'net11.0' - -# ── Clone & build ─────────────────────────────────────────────────────────── - -def clone_maui_repo(branch: str): - """Sparse-clone the maui repo with only the directories needed for benchmarks.""" - log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') - - if os.path.exists(MAUI_REPO_DIR): - shutil.rmtree(MAUI_REPO_DIR) - - subprocess.run([ - 'git', 'clone', - '-c', 'core.longpaths=true', - '--depth', '1', - '--filter=blob:none', - '--sparse', - '--branch', branch, - MAUI_REPO_URL, - MAUI_REPO_DIR - ], check=True) - - subprocess.run([ - 'git', 'sparse-checkout', 'set', - 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', - 'src/Workload', 'src/Essentials', - 'eng', '.config' - ], cwd=MAUI_REPO_DIR, check=True) - - log.info('Clone complete.') - - -def patch_directory_build_props(): - """Disable non-desktop TFMs in Directory.Build.props. - - MAUI's props set Include*TargetFrameworks to true at multiple points. - MauiPlatforms (which controls TargetFrameworks) is computed from these. - We must replace ALL true→false in-place so the platform lists are never - populated, not just append overrides at the end (too late for evaluation). - """ - import re - props_path = os.path.join(MAUI_REPO_DIR, 'Directory.Build.props') - log.info('Patching Directory.Build.props to disable non-desktop TFMs...') - - with open(props_path, 'r', encoding='utf-8-sig') as f: - content = f.read() - - for prop_name in DESKTOP_ONLY_PROPS: - # Replace all occurrences of true with false - pattern = rf'(<{prop_name}\b[^>]*>)true()' - content, count = re.subn(pattern, r'\g<1>false\g<2>', content) - if count > 0: - log.info(f' {prop_name}: replaced {count} occurrence(s)') - - with open(props_path, 'w', encoding='utf-8') as f: - f.write(content) - - log.info(' Directory.Build.props patched.') - - -def build_maui_dependencies(): - """Build MAUI BuildTasks solution filter — the core libraries benchmarks depend on.""" - log.info('Restoring dotnet tools...') - subprocess.run(['dotnet', 'tool', 'restore'], cwd=MAUI_REPO_DIR, check=True) - - log.info('Building Microsoft.Maui.BuildTasks.slnf (desktop TFMs only)...') - subprocess.run([ - 'dotnet', 'build', - 'Microsoft.Maui.BuildTasks.slnf', - '-c', 'Release', - ], cwd=MAUI_REPO_DIR, check=True) - - log.info('MAUI dependencies built successfully.') - -# ── BDN.Extensions injection ──────────────────────────────────────────────── - -def _find_bdn_extensions() -> str: - """ - Return the absolute path to BenchmarkDotNet.Extensions.csproj. - On Helix the full perf repo lives under the correlation payload; - for local runs we walk up from this script's location. - """ - correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') - if correlation: - # Helix: repo is at /performance/ - candidate = os.path.join( - correlation, 'performance', 'src', 'harness', - 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') - else: - # Local: navigate from this script's directory - scenario_dir = os.path.dirname(os.path.abspath(__file__)) - candidate = os.path.normpath(os.path.join( - scenario_dir, '..', '..', 'harness', - 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) - - if not os.path.exists(candidate): - raise FileNotFoundError( - f'BenchmarkDotNet.Extensions.csproj not found at {candidate}. ' - f'HELIX_CORRELATION_PAYLOAD={correlation!r}') - - log.info(f'BDN.Extensions located at: {candidate}') - return candidate - - -def inject_bdn_extensions(csproj_path: str, bdn_ext_abs: str): - """Add a ProjectReference to BenchmarkDotNet.Extensions into a benchmark csproj. - Also removes the existing BenchmarkDotNet PackageReference to avoid version conflicts - (BDN.Extensions brings in the correct version transitively).""" - log.info(f'Injecting BDN.Extensions reference into {os.path.basename(csproj_path)}') - - csproj_dir = os.path.dirname(os.path.abspath(csproj_path)) - bdn_ext_rel = os.path.relpath(bdn_ext_abs, csproj_dir) - - tree = ET.parse(csproj_path) - root = tree.getroot() - - # SDK-style projects typically have no XML namespace - ns = '' - if root.tag.startswith('{'): - ns = root.tag.split('}')[0] + '}' - - # Remove existing BenchmarkDotNet PackageReference to avoid NU1605 downgrade error - for item_group in root.findall(f'{ns}ItemGroup'): - for pkg_ref in item_group.findall(f'{ns}PackageReference'): - include = pkg_ref.get('Include', '') - if include.startswith('BenchmarkDotNet'): - item_group.remove(pkg_ref) - log.info(f' Removed PackageReference: {include}') - - # Add ProjectReference to BDN.Extensions - item_group = ET.SubElement(root, f'{ns}ItemGroup') - item_group.set('Label', 'PerfLabInjected') - proj_ref = ET.SubElement(item_group, f'{ns}ProjectReference') - proj_ref.set('Include', bdn_ext_rel) - - tree.write(csproj_path, xml_declaration=True, encoding='utf-8') - log.info(f' Added ProjectReference: {bdn_ext_rel}') - - -def patch_program_cs(program_cs_path: str): - """Patch Program.cs to add PerfLabExporter via ManualConfig.""" - log.info(f'Patching {os.path.basename(program_cs_path)} for PerfLabExporter') - - with open(program_cs_path, 'r', encoding='utf-8-sig') as f: - content = f.read() +Delegates to the shared Runner infrastructure which dispatches to +BDNDesktopHelper for the full lifecycle: clone dotnet/maui, build +dependencies, patch for PerfLabExporter, run BDN suites, collect results. - # Add required using statements at the top - usings_to_add = [ - 'using BenchmarkDotNet.Configs;', - 'using BenchmarkDotNet.Extensions;', - 'using System;', - 'using System.IO;', - ] - insert_block = '' - for u in usings_to_add: - if u not in content: - insert_block += u + '\n' - if insert_block: - content = insert_block + content - - # Build a minimal config that adds PerfLabExporter without the - # MandatoryCategoryValidator (MAUI benchmarks don't use BenchmarkCategory). - new_run_call = ( - 'var config = ManualConfig.Create(DefaultConfig.Instance)\n' - ' .WithArtifactsPath(Path.Combine(\n' - ' Path.GetDirectoryName(typeof(Program).Assembly.Location),\n' - ' "BenchmarkDotNet.Artifacts"));\n' - ' if (Environment.GetEnvironmentVariable("PERFLAB_INLAB") == "1")\n' - ' config = config.AddExporter(new PerfLabExporter());\n' - ' BenchmarkSwitcher\n' - ' .FromAssembly(typeof(Program).Assembly)\n' - ' .Run(args, config)' - ) - - # Known patterns from MAUI benchmark Program.cs files - patterns = [ - 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);', - 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args)', - 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args);', - 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args)', - ] - - replaced = False - for pattern in patterns: - if pattern in content: - suffix = ';' if pattern.endswith(';') else '' - content = content.replace(pattern, new_run_call + suffix) - replaced = True - break - - if not replaced: - log.warning(f' Could not find BenchmarkSwitcher.Run pattern — may need manual patching') - return - - with open(program_cs_path, 'w', encoding='utf-8') as f: - f.write(content) - - log.info(f' Patched successfully.') - - -def patch_benchmark_projects(): - """Inject BDN.Extensions and RecommendedConfig into all benchmark projects.""" - bdn_ext_abs = _find_bdn_extensions() - - for name, csproj_rel in BENCHMARK_PROJECTS.items(): - csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) - project_dir = os.path.dirname(csproj_path) - program_cs = os.path.join(project_dir, 'Program.cs') - - if not os.path.exists(csproj_path): - log.warning(f'Benchmark project not found: {csproj_path}') - continue - - inject_bdn_extensions(csproj_path, bdn_ext_abs) - - if os.path.exists(program_cs): - patch_program_cs(program_cs) - else: - log.warning(f'Program.cs not found for {name}') - -# ── Build benchmarks ──────────────────────────────────────────────────────── - -def build_benchmark_projects(): - """Build each benchmark project in Release mode.""" - for name, csproj_rel in BENCHMARK_PROJECTS.items(): - csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) - if not os.path.exists(csproj_path): - continue - - log.info(f'Building benchmark: {name}') - subprocess.run([ - 'dotnet', 'build', - csproj_rel, - '-c', 'Release', - ], cwd=MAUI_REPO_DIR, check=True) - - log.info('All benchmark projects built successfully.') - -# ── Run benchmarks ────────────────────────────────────────────────────────── - -def run_benchmark(name: str, csproj_rel: str, extra_bdn_args: list): - """Run a single BDN benchmark suite.""" - csproj_path = os.path.join(MAUI_REPO_DIR, csproj_rel) - if not os.path.exists(csproj_path): - log.warning(f'Benchmark project not found: {csproj_path}') - return False - - log.info(f'Running benchmark suite: {name}') - - cmd = [ - 'dotnet', 'run', - '-c', 'Release', - '--no-build', - '--project', csproj_rel, - '--', - '--filter', '*', - ] + extra_bdn_args - - result = subprocess.run(cmd, cwd=MAUI_REPO_DIR) - if result.returncode != 0: - log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') - return False - - log.info(f'Benchmark suite {name} completed successfully.') - return True - -# ── Result collection ─────────────────────────────────────────────────────── - -def collect_results(): - """ - Collect perf-lab-report.json files from BDN artifacts and copy them - to HELIX_WORKITEM_UPLOAD_ROOT for the infrastructure to pick up. - """ - upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') - - # Search recursively under maui_repo for perf-lab-report files - report_pattern = os.path.join(MAUI_REPO_DIR, '**', '*-perf-lab-report.json') - report_files = glob.glob(report_pattern, recursive=True) - - if not report_files: - log.warning('No perf-lab-report.json files found. ' - 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') - return - - log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') - - # Combine all reports into a single file - combined = [] - for report_file in report_files: - log.info(f' Collecting: {report_file}') - try: - with open(report_file, 'r', encoding='utf-8') as f: - data = json.load(f) - if isinstance(data, list): - combined.extend(data) - else: - combined.append(data) - except (json.JSONDecodeError, IOError) as e: - log.warning(f' Failed to read {report_file}: {e}') - - if combined: - combined_path = 'combined-perf-lab-report.json' - with open(combined_path, 'w', encoding='utf-8') as f: - json.dump(combined, f, indent=2) - log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') - - if upload_root: - dest = os.path.join(upload_root, combined_path) - shutil.copy2(combined_path, dest) - log.info(f'Copied combined report to {dest}') - - for report_file in report_files: - basename = os.path.basename(report_file) - dest = os.path.join(upload_root, basename) - shutil.copy2(report_file, dest) - log.info(f' Copied {basename} to upload root') +Usage: test.py bdndesktop --framework net11.0 --suite all +''' +from shared.runner import TestTraits, Runner -# ── Main ──────────────────────────────────────────────────────────────────── +EXENAME = 'MauiDesktopBDNBenchmarks' if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser(description='MAUI Desktop BDN Benchmarks - Test runner') - parser.add_argument('-f', '--framework', default='net11.0', - help='Target .NET framework (determines MAUI branch)') - parser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], - default='all', help='Which benchmark suite to run') - parser.add_argument('--bdn-args', nargs='*', default=[], - help='Additional arguments to pass to BenchmarkDotNet') - args = parser.parse_args() - - # ── Setup ─────────────────────────────────────────────────────────────── - branch = get_maui_branch(args.framework) - clone_maui_repo(branch) - patch_directory_build_props() - build_maui_dependencies() - patch_benchmark_projects() - build_benchmark_projects() - - # ── Run ───────────────────────────────────────────────────────────────── - if args.suite == 'all': - suites = BENCHMARK_PROJECTS.items() - else: - suites = [(args.suite, BENCHMARK_PROJECTS[args.suite])] - - all_passed = True - for name, csproj_rel in suites: - if not run_benchmark(name, csproj_rel, args.bdn_args): - all_passed = False - - # ── Collect ───────────────────────────────────────────────────────────── - collect_results() - - if not all_passed: - log.error('One or more benchmark suites failed.') - sys.exit(1) - - log.info('All benchmark suites completed.') + traits = TestTraits(exename=EXENAME, guiapp='false') + Runner(traits).run() diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py new file mode 100644 index 00000000000..6f1097e4f92 --- /dev/null +++ b/src/scenarios/shared/bdndesktop.py @@ -0,0 +1,412 @@ +''' +Helper/Runner for desktop BenchmarkDotNet scenarios that build from external repos. + +Currently supports MAUI desktop benchmarks from dotnet/maui. The pattern +(clone → patch → build → run BDN → collect results) can be extended to +other repos by passing different configuration. +''' +import os +import re +import sys +import glob +import json +import shutil +import subprocess +import xml.etree.ElementTree as ET +from logging import getLogger +from performance.common import runninginlab +from shared.const import TRACEDIR + +# ── Default MAUI configuration ────────────────────────────────────────────── + +MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' + +MAUI_BENCHMARK_PROJECTS = { + 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', + 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', + 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', +} + +MAUI_BUILD_SOLUTION_FILTER = 'Microsoft.Maui.BuildTasks.slnf' + +MAUI_SPARSE_CHECKOUT_DIRS = [ + 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', + 'src/Workload', 'src/Essentials', + 'eng', '.config', +] + +# MSBuild properties to disable non-desktop target frameworks. +# Used by _patch_directory_build_props() to replace true→false in-place +# so ALL builds (including BDN's internal auto-generated project builds) +# are desktop-only. +DESKTOP_ONLY_PROPS = { + 'IncludeAndroidTargetFrameworks': 'false', + 'IncludeIosTargetFrameworks': 'false', + 'IncludeMacCatalystTargetFrameworks': 'false', + 'IncludeMacOSTargetFrameworks': 'false', + 'IncludeTizenTargetFrameworks': 'false', +} + + +class BDNDesktopHelper(object): + + def __init__(self): + self.repo_dir = 'maui_repo' + + # ── Public entry point ────────────────────────────────────────────────── + + def runtests(self, framework: str, suite: str, bdn_args: list, + upload_to_perflab_container: bool): + ''' + Full lifecycle: clone dotnet/maui, build dependencies, patch benchmark + projects for PerfLabExporter, run BDN suites, and collect results. + ''' + log = getLogger() + branch = self._get_branch(framework) + + # Setup + self._clone_repo(branch) + self._patch_directory_build_props() + self._build_dependencies() + self._patch_benchmark_projects() + self._build_benchmark_projects(suite) + + # Run + if suite == 'all': + suites = MAUI_BENCHMARK_PROJECTS.items() + else: + suites = [(suite, MAUI_BENCHMARK_PROJECTS[suite])] + + all_passed = True + for name, csproj_rel in suites: + if not self._run_benchmark(name, csproj_rel, bdn_args): + all_passed = False + + # Collect + self._collect_results(upload_to_perflab_container) + + if not all_passed: + log.error('One or more benchmark suites failed.') + sys.exit(1) + + log.info('All benchmark suites completed.') + + # ── Branch mapping ────────────────────────────────────────────────────── + + @staticmethod + def _get_branch(framework: str) -> str: + if framework and framework.startswith('net'): + return framework # net11.0 -> net11.0, net10.0 -> net10.0 + return 'net11.0' + + # ── Clone & build ─────────────────────────────────────────────────────── + + def _clone_repo(self, branch: str): + log = getLogger() + log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') + + if os.path.exists(self.repo_dir): + shutil.rmtree(self.repo_dir) + + subprocess.run([ + 'git', 'clone', + '-c', 'core.longpaths=true', + '--depth', '1', + '--filter=blob:none', + '--sparse', + '--branch', branch, + MAUI_REPO_URL, + self.repo_dir + ], check=True) + + subprocess.run( + ['git', 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, + cwd=self.repo_dir, check=True) + + log.info('Clone complete.') + + def _patch_directory_build_props(self): + '''Disable non-desktop TFMs in Directory.Build.props. + + MAUI's props set Include*TargetFrameworks to true at multiple points. + MauiPlatforms (which controls TargetFrameworks) is computed from these. + We must replace ALL true→false in-place so the platform lists are never + populated. + ''' + log = getLogger() + props_path = os.path.join(self.repo_dir, 'Directory.Build.props') + log.info('Patching Directory.Build.props to disable non-desktop TFMs...') + + with open(props_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + for prop_name in DESKTOP_ONLY_PROPS: + pattern = rf'(<{prop_name}\b[^>]*>)true()' + content, count = re.subn(pattern, r'\g<1>false\g<2>', content) + if count > 0: + log.info(f' {prop_name}: replaced {count} occurrence(s)') + + with open(props_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Directory.Build.props patched.') + + def _build_dependencies(self): + log = getLogger() + log.info('Restoring dotnet tools...') + subprocess.run(['dotnet', 'tool', 'restore'], cwd=self.repo_dir, check=True) + + log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') + subprocess.run([ + 'dotnet', 'build', + MAUI_BUILD_SOLUTION_FILTER, + '-c', 'Release', + ], cwd=self.repo_dir, check=True) + + log.info('MAUI dependencies built successfully.') + + # ── BDN.Extensions injection ──────────────────────────────────────────── + + def _find_bdn_extensions(self) -> str: + '''Return the absolute path to BenchmarkDotNet.Extensions.csproj.''' + correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') + if correlation: + candidate = os.path.join( + correlation, 'performance', 'src', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') + else: + scenario_dir = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.normpath(os.path.join( + scenario_dir, '..', '..', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) + + if not os.path.exists(candidate): + raise FileNotFoundError( + f'BenchmarkDotNet.Extensions.csproj not found at {candidate}. ' + f'HELIX_CORRELATION_PAYLOAD={correlation!r}') + + getLogger().info(f'BDN.Extensions located at: {candidate}') + return candidate + + def _inject_bdn_extensions(self, csproj_path: str, bdn_ext_abs: str): + '''Add a ProjectReference to BDN.Extensions and remove existing BDN PackageRef.''' + log = getLogger() + log.info(f'Injecting BDN.Extensions reference into {os.path.basename(csproj_path)}') + + csproj_dir = os.path.dirname(os.path.abspath(csproj_path)) + bdn_ext_rel = os.path.relpath(bdn_ext_abs, csproj_dir) + + tree = ET.parse(csproj_path) + root = tree.getroot() + + ns = '' + if root.tag.startswith('{'): + ns = root.tag.split('}')[0] + '}' + + # Remove existing BenchmarkDotNet PackageReference to avoid version conflicts + for item_group in root.findall(f'{ns}ItemGroup'): + for pkg_ref in item_group.findall(f'{ns}PackageReference'): + include = pkg_ref.get('Include', '') + if include.startswith('BenchmarkDotNet'): + item_group.remove(pkg_ref) + log.info(f' Removed PackageReference: {include}') + + # Add ProjectReference to BDN.Extensions + item_group = ET.SubElement(root, f'{ns}ItemGroup') + item_group.set('Label', 'PerfLabInjected') + proj_ref = ET.SubElement(item_group, f'{ns}ProjectReference') + proj_ref.set('Include', bdn_ext_rel) + + tree.write(csproj_path, xml_declaration=True, encoding='utf-8') + log.info(f' Added ProjectReference: {bdn_ext_rel}') + + def _patch_program_cs(self, program_cs_path: str): + '''Patch Program.cs to add PerfLabExporter via ManualConfig.''' + log = getLogger() + log.info(f'Patching {os.path.basename(program_cs_path)} for PerfLabExporter') + + with open(program_cs_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + usings_to_add = [ + 'using BenchmarkDotNet.Configs;', + 'using BenchmarkDotNet.Extensions;', + 'using System;', + 'using System.IO;', + ] + insert_block = '' + for u in usings_to_add: + if u not in content: + insert_block += u + '\n' + if insert_block: + content = insert_block + content + + # ManualConfig without MandatoryCategoryValidator (MAUI benchmarks + # don't use [BenchmarkCategory]) + new_run_call = ( + 'var config = ManualConfig.Create(DefaultConfig.Instance)\n' + ' .WithArtifactsPath(Path.Combine(\n' + ' Path.GetDirectoryName(typeof(Program).Assembly.Location),\n' + ' "BenchmarkDotNet.Artifacts"));\n' + ' if (Environment.GetEnvironmentVariable("PERFLAB_INLAB") == "1")\n' + ' config = config.AddExporter(new PerfLabExporter());\n' + ' BenchmarkSwitcher\n' + ' .FromAssembly(typeof(Program).Assembly)\n' + ' .Run(args, config)' + ) + + patterns = [ + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);', + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args)', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args);', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args)', + ] + + replaced = False + for pattern in patterns: + if pattern in content: + suffix = ';' if pattern.endswith(';') else '' + content = content.replace(pattern, new_run_call + suffix) + replaced = True + break + + if not replaced: + log.warning(' Could not find BenchmarkSwitcher.Run pattern — may need manual patching') + return + + with open(program_cs_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Patched successfully.') + + def _patch_benchmark_projects(self): + '''Inject BDN.Extensions and PerfLabExporter into all benchmark projects.''' + bdn_ext_abs = self._find_bdn_extensions() + + for name, csproj_rel in MAUI_BENCHMARK_PROJECTS.items(): + csproj_path = os.path.join(self.repo_dir, csproj_rel) + project_dir = os.path.dirname(csproj_path) + program_cs = os.path.join(project_dir, 'Program.cs') + + if not os.path.exists(csproj_path): + getLogger().warning(f'Benchmark project not found: {csproj_path}') + continue + + self._inject_bdn_extensions(csproj_path, bdn_ext_abs) + + if os.path.exists(program_cs): + self._patch_program_cs(program_cs) + else: + getLogger().warning(f'Program.cs not found for {name}') + + # ── Build benchmarks ──────────────────────────────────────────────────── + + def _build_benchmark_projects(self, suite: str): + log = getLogger() + if suite == 'all': + projects = MAUI_BENCHMARK_PROJECTS.items() + else: + projects = [(suite, MAUI_BENCHMARK_PROJECTS[suite])] + + for name, csproj_rel in projects: + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + continue + + log.info(f'Building benchmark: {name}') + subprocess.run([ + 'dotnet', 'build', + csproj_rel, + '-c', 'Release', + ], cwd=self.repo_dir, check=True) + + log.info('All benchmark projects built successfully.') + + # ── Run benchmarks ────────────────────────────────────────────────────── + + def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bool: + log = getLogger() + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found: {csproj_path}') + return False + + log.info(f'Running benchmark suite: {name}') + + cmd = [ + 'dotnet', 'run', + '-c', 'Release', + '--no-build', + '--project', csproj_rel, + '--', + '--filter', '*', + ] + extra_bdn_args + + result = subprocess.run(cmd, cwd=self.repo_dir) + if result.returncode != 0: + log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') + return False + + log.info(f'Benchmark suite {name} completed successfully.') + return True + + # ── Result collection ─────────────────────────────────────────────────── + + def _collect_results(self, upload_to_perflab_container: bool): + '''Collect perf-lab-report.json files from BDN artifacts.''' + log = getLogger() + upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') + + report_pattern = os.path.join(self.repo_dir, '**', '*-perf-lab-report.json') + report_files = glob.glob(report_pattern, recursive=True) + + if not report_files: + log.warning('No perf-lab-report.json files found. ' + 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') + return + + log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') + + # Combine all reports into a single file + combined = [] + for report_file in report_files: + log.info(f' Collecting: {report_file}') + try: + with open(report_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + combined.extend(data) + else: + combined.append(data) + except (json.JSONDecodeError, IOError) as e: + log.warning(f' Failed to read {report_file}: {e}') + + if combined: + combined_path = 'combined-perf-lab-report.json' + with open(combined_path, 'w', encoding='utf-8') as f: + json.dump(combined, f, indent=2) + log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') + + if upload_root: + dest = os.path.join(upload_root, combined_path) + shutil.copy2(combined_path, dest) + log.info(f'Copied combined report to {dest}') + + for report_file in report_files: + basename = os.path.basename(report_file) + dest = os.path.join(upload_root, basename) + shutil.copy2(report_file, dest) + log.info(f' Copied {basename} to upload root') + + # Also upload via perflab container if requested + if upload_to_perflab_container and runninginlab(): + try: + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + import upload + globpath = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + upload_code = upload.upload(globpath, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + log.info(f'BDN Desktop Benchmarks Upload Code: {upload_code}') + if upload_code != 0: + sys.exit(upload_code) + except ImportError: + log.warning('Upload module not available — skipping perflab container upload') diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 074fa7fdf89..31c77cc09b7 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -18,6 +18,7 @@ DEVICEMEMORYCONSUMPTION = "devicememoryconsumption" ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" +BDNDESKTOP = "bdndesktop" BUILDTIME = "buildtime" SCENARIO_NAMES = {STARTUP: 'Startup', diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 165f0e882ba..73af2fb27e2 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -18,6 +18,7 @@ from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper +from shared.bdndesktop import BDNDesktopHelper from shared.devicepowerconsumption import DevicePowerConsumptionHelper from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper @@ -174,6 +175,16 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + bdndesktopparser = subparsers.add_parser(const.BDNDESKTOP, + description='Run BenchmarkDotNet benchmarks from an external repo on desktop') + bdndesktopparser.add_argument('--framework', '-f', default='net11.0', + help='Target .NET framework (determines repo branch)', dest='framework') + bdndesktopparser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], + default='all', help='Which benchmark suite to run', dest='suite') + bdndesktopparser.add_argument('--bdn-args', nargs='*', default=[], + help='Additional arguments to pass to BenchmarkDotNet', dest='bdnargs') + self.add_common_arguments(bdndesktopparser) + args = parser.parse_args() if not args.testtype: @@ -231,6 +242,11 @@ def parseargs(self): self.runtimeseconds = args.runtimeseconds self.closeToStartDelay = args.closeToStartDelay + if self.testtype == const.BDNDESKTOP: + self.framework = args.framework + self.suite = args.suite + self.bdnargs = args.bdnargs + if args.scenarioname: self.scenarioname = args.scenarioname @@ -428,6 +444,10 @@ def run(self): androidInstrumentation = AndroidInstrumentationHelper() androidInstrumentation.runtests(self.packagepath, self.packagename, self.instrumentationname, self.upload_to_perflab_container) + elif self.testtype == const.BDNDESKTOP: + bdnDesktop = BDNDesktopHelper() + bdnDesktop.runtests(self.framework, self.suite, self.bdnargs, self.upload_to_perflab_container) + elif self.testtype == const.DEVICEPOWERCONSUMPTION: devicePowerConsumption = DevicePowerConsumptionHelper() self.traits.add_traits(overwrite=True, apptorun="app", powerconsumptionmetric=const.POWERCONSUMPTION_ANDROID, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) From 0a112ec40b2bdd9303fc594387a4e8093cb9d0d4 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 17 Mar 2026 17:01:33 -0700 Subject: [PATCH 3/8] Split MAUI-specific logic into test.py, make BDNDesktopHelper reusable Move clone, branch mapping, sparse checkout, and dependency build into test.py (MAUI-specific orchestrator). BDNDesktopHelper now accepts repo_dir, benchmark_projects, and disable_props as constructor params, making it reusable for any repo's BDN desktop benchmarks. Remove bdndesktop routing from runner.py and const.py since test.py drives the helper directly with its own arg parsing. Tested locally: Graphics.Benchmarks ran successfully via refactored path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_desktop_benchmarks.proj | 2 +- src/scenarios/mauiDesktopBenchmarks/test.py | 135 +++++++++++++- src/scenarios/shared/bdndesktop.py | 176 +++++++------------ src/scenarios/shared/const.py | 1 - src/scenarios/shared/runner.py | 19 -- 5 files changed, 190 insertions(+), 143 deletions(-) diff --git a/eng/performance/maui_desktop_benchmarks.proj b/eng/performance/maui_desktop_benchmarks.proj index 6233b7b722a..047a4f9f9b5 100644 --- a/eng/performance/maui_desktop_benchmarks.proj +++ b/eng/performance/maui_desktop_benchmarks.proj @@ -10,7 +10,7 @@ $(ScenariosDir)mauiDesktopBenchmarks $(Python) pre.py -f $(PERFLAB_Framework) - $(Python) test.py bdndesktop --framework $(PERFLAB_Framework) --suite all + $(Python) test.py --framework $(PERFLAB_Framework) --suite all $(Python) post.py diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index 88d148148e7..f94470f3e14 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -1,16 +1,135 @@ ''' MAUI Desktop BenchmarkDotNet benchmarks. -Delegates to the shared Runner infrastructure which dispatches to -BDNDesktopHelper for the full lifecycle: clone dotnet/maui, build -dependencies, patch for PerfLabExporter, run BDN suites, collect results. +Handles MAUI-specific setup (clone, branch mapping, dependency build) then +delegates to the shared BDNDesktopHelper for the generic BDN workflow +(patch, build benchmarks, run, collect results). -Usage: test.py bdndesktop --framework net11.0 --suite all +Usage: test.py --framework net11.0 --suite all ''' -from shared.runner import TestTraits, Runner +import os +import shutil +import subprocess +from argparse import ArgumentParser +from logging import getLogger +from performance.logger import setup_loggers +from shared.bdndesktop import BDNDesktopHelper + +# ── MAUI-specific configuration ───────────────────────────────────────────── + +MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' +MAUI_REPO_DIR = 'maui_repo' + +MAUI_BENCHMARK_PROJECTS = { + 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', + 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', + 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', +} + +MAUI_SPARSE_CHECKOUT_DIRS = [ + 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', + 'src/Workload', 'src/Essentials', + 'eng', '.config', +] + +MAUI_BUILD_SOLUTION_FILTER = 'Microsoft.Maui.BuildTasks.slnf' + +# MSBuild properties to disable non-desktop target frameworks. +# MAUI's Directory.Build.props sets these to true unconditionally at multiple +# points; MauiPlatforms is computed from them. In-place replacement is +# required because appending overrides at the end doesn't work (MSBuild +# evaluates top-to-bottom). +DESKTOP_ONLY_PROPS = { + 'IncludeAndroidTargetFrameworks': 'false', + 'IncludeIosTargetFrameworks': 'false', + 'IncludeMacCatalystTargetFrameworks': 'false', + 'IncludeMacOSTargetFrameworks': 'false', + 'IncludeTizenTargetFrameworks': 'false', +} + + +def get_branch(framework: str) -> str: + '''Map framework moniker to MAUI repo branch.''' + if framework and framework.startswith('net'): + return framework # net11.0 -> net11.0, net10.0 -> net10.0 + return 'net11.0' + + +def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): + '''Sparse-clone dotnet/maui at the given branch.''' + log = getLogger() + log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') + + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + + subprocess.run([ + 'git', 'clone', + '-c', 'core.longpaths=true', + '--depth', '1', + '--filter=blob:none', + '--sparse', + '--branch', branch, + MAUI_REPO_URL, + repo_dir + ], check=True) + + subprocess.run( + ['git', 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, + cwd=repo_dir, check=True) + + log.info('Clone complete.') + + +def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): + '''Restore dotnet tools and build MAUI's BuildTasks solution filter.''' + log = getLogger() + log.info('Restoring dotnet tools...') + subprocess.run(['dotnet', 'tool', 'restore'], cwd=repo_dir, check=True) + + log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') + subprocess.run([ + 'dotnet', 'build', + MAUI_BUILD_SOLUTION_FILTER, + '-c', 'Release', + ], cwd=repo_dir, check=True) + + log.info('MAUI dependencies built successfully.') + + +def parse_args(): + parser = ArgumentParser(description='Run MAUI desktop BDN benchmarks') + parser.add_argument('--framework', '-f', default='net11.0', + help='Target .NET framework (determines MAUI repo branch)') + parser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], + default='all', help='Which benchmark suite to run') + parser.add_argument('--bdn-args', nargs='*', default=[], + help='Additional arguments to pass to BenchmarkDotNet') + parser.add_argument('--upload-to-perflab-container', action='store_true', + help='Upload results to perflab container') + return parser.parse_args() -EXENAME = 'MauiDesktopBDNBenchmarks' if __name__ == '__main__': - traits = TestTraits(exename=EXENAME, guiapp='false') - Runner(traits).run() + setup_loggers(True) + args = parse_args() + + # MAUI-specific: clone repo and build dependencies + branch = get_branch(args.framework) + clone_maui_repo(branch) + + # Generic BDN desktop workflow: patch, build benchmarks, run, collect + helper = BDNDesktopHelper( + repo_dir=MAUI_REPO_DIR, + benchmark_projects=MAUI_BENCHMARK_PROJECTS, + disable_props=DESKTOP_ONLY_PROPS, + ) + + # Patch Directory.Build.props BEFORE any builds (including MAUI deps) + helper.patch_directory_build_props() + + # MAUI-specific: build BuildTasks solution filter + build_maui_dependencies() + + # Run the generic BDN workflow + helper.runtests(args.suite, args.bdn_args, args.upload_to_perflab_container) diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py index 6f1097e4f92..9fa1568b06f 100644 --- a/src/scenarios/shared/bdndesktop.py +++ b/src/scenarios/shared/bdndesktop.py @@ -1,9 +1,16 @@ ''' -Helper/Runner for desktop BenchmarkDotNet scenarios that build from external repos. - -Currently supports MAUI desktop benchmarks from dotnet/maui. The pattern -(clone → patch → build → run BDN → collect results) can be extended to -other repos by passing different configuration. +Reusable helper for running BenchmarkDotNet benchmarks from external repos +on desktop. + +Handles the generic workflow: + 1. Patch Directory.Build.props to disable unwanted target frameworks + 2. Inject BDN.Extensions (PerfLabExporter) into benchmark projects + 3. Build benchmark projects + 4. Run BDN suites + 5. Collect and upload results + +Callers (e.g. test.py in each scenario) are responsible for repo-specific +setup such as cloning, branch selection, and building repo dependencies. ''' import os import re @@ -15,67 +22,50 @@ import xml.etree.ElementTree as ET from logging import getLogger from performance.common import runninginlab -from shared.const import TRACEDIR - -# ── Default MAUI configuration ────────────────────────────────────────────── - -MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' - -MAUI_BENCHMARK_PROJECTS = { - 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', - 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', - 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', -} - -MAUI_BUILD_SOLUTION_FILTER = 'Microsoft.Maui.BuildTasks.slnf' - -MAUI_SPARSE_CHECKOUT_DIRS = [ - 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', - 'src/Workload', 'src/Essentials', - 'eng', '.config', -] - -# MSBuild properties to disable non-desktop target frameworks. -# Used by _patch_directory_build_props() to replace true→false in-place -# so ALL builds (including BDN's internal auto-generated project builds) -# are desktop-only. -DESKTOP_ONLY_PROPS = { - 'IncludeAndroidTargetFrameworks': 'false', - 'IncludeIosTargetFrameworks': 'false', - 'IncludeMacCatalystTargetFrameworks': 'false', - 'IncludeMacOSTargetFrameworks': 'false', - 'IncludeTizenTargetFrameworks': 'false', -} class BDNDesktopHelper(object): - - def __init__(self): - self.repo_dir = 'maui_repo' + '''Generic helper for running BDN desktop benchmarks from a local repo checkout. + + Args: + repo_dir: Path to the cloned repository root. + benchmark_projects: Dict mapping suite name to csproj relative path + (e.g. {'graphics': 'src/Graphics/.../Graphics.Benchmarks.csproj'}). + disable_props: Optional dict of MSBuild property names to replacement values + to patch in Directory.Build.props (e.g. disable mobile TFMs). + ''' + + def __init__(self, repo_dir: str, benchmark_projects: dict, + disable_props: dict = None): + self.repo_dir = repo_dir + self.benchmark_projects = benchmark_projects + self.disable_props = disable_props or {} # ── Public entry point ────────────────────────────────────────────────── - def runtests(self, framework: str, suite: str, bdn_args: list, + def runtests(self, suite: str, bdn_args: list, upload_to_perflab_container: bool): ''' - Full lifecycle: clone dotnet/maui, build dependencies, patch benchmark - projects for PerfLabExporter, run BDN suites, and collect results. + Patch benchmark projects, build, run, and collect BDN results. + + Assumes the caller has already: + - Cloned the repo and built repo-specific dependencies + - Called patch_directory_build_props() if needed (must happen + before any builds, including dependency builds) ''' log = getLogger() - branch = self._get_branch(framework) - # Setup - self._clone_repo(branch) - self._patch_directory_build_props() - self._build_dependencies() - self._patch_benchmark_projects() - self._build_benchmark_projects(suite) + # Patch benchmark csprojs + Program.cs for PerfLabExporter + self.patch_benchmark_projects() + + # Build + self.build_benchmark_projects(suite) # Run if suite == 'all': - suites = MAUI_BENCHMARK_PROJECTS.items() + suites = self.benchmark_projects.items() else: - suites = [(suite, MAUI_BENCHMARK_PROJECTS[suite])] + suites = [(suite, self.benchmark_projects[suite])] all_passed = True for name, csproj_rel in suites: @@ -91,58 +81,30 @@ def runtests(self, framework: str, suite: str, bdn_args: list, log.info('All benchmark suites completed.') - # ── Branch mapping ────────────────────────────────────────────────────── + # ── Patch Directory.Build.props ───────────────────────────────────────── - @staticmethod - def _get_branch(framework: str) -> str: - if framework and framework.startswith('net'): - return framework # net11.0 -> net11.0, net10.0 -> net10.0 - return 'net11.0' + def patch_directory_build_props(self): + '''Disable unwanted TFMs by replacing property values in-place. - # ── Clone & build ─────────────────────────────────────────────────────── - - def _clone_repo(self, branch: str): - log = getLogger() - log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') - - if os.path.exists(self.repo_dir): - shutil.rmtree(self.repo_dir) - - subprocess.run([ - 'git', 'clone', - '-c', 'core.longpaths=true', - '--depth', '1', - '--filter=blob:none', - '--sparse', - '--branch', branch, - MAUI_REPO_URL, - self.repo_dir - ], check=True) - - subprocess.run( - ['git', 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, - cwd=self.repo_dir, check=True) - - log.info('Clone complete.') - - def _patch_directory_build_props(self): - '''Disable non-desktop TFMs in Directory.Build.props. - - MAUI's props set Include*TargetFrameworks to true at multiple points. - MauiPlatforms (which controls TargetFrameworks) is computed from these. - We must replace ALL true→false in-place so the platform lists are never - populated. + Handles repos (like MAUI) that set Include*TargetFrameworks=true at + multiple points before computing TargetFrameworks. Appending + overrides at the end doesn't work because MSBuild evaluates + top-to-bottom, so we regex-replace ALL occurrences in-place. ''' log = getLogger() props_path = os.path.join(self.repo_dir, 'Directory.Build.props') - log.info('Patching Directory.Build.props to disable non-desktop TFMs...') + if not os.path.exists(props_path): + log.info('No Directory.Build.props found — skipping TFM patching.') + return + + log.info('Patching Directory.Build.props to disable unwanted TFMs...') with open(props_path, 'r', encoding='utf-8-sig') as f: content = f.read() - for prop_name in DESKTOP_ONLY_PROPS: + for prop_name, new_value in self.disable_props.items(): pattern = rf'(<{prop_name}\b[^>]*>)true()' - content, count = re.subn(pattern, r'\g<1>false\g<2>', content) + content, count = re.subn(pattern, rf'\g<1>{new_value}\g<2>', content) if count > 0: log.info(f' {prop_name}: replaced {count} occurrence(s)') @@ -151,20 +113,6 @@ def _patch_directory_build_props(self): log.info(' Directory.Build.props patched.') - def _build_dependencies(self): - log = getLogger() - log.info('Restoring dotnet tools...') - subprocess.run(['dotnet', 'tool', 'restore'], cwd=self.repo_dir, check=True) - - log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') - subprocess.run([ - 'dotnet', 'build', - MAUI_BUILD_SOLUTION_FILTER, - '-c', 'Release', - ], cwd=self.repo_dir, check=True) - - log.info('MAUI dependencies built successfully.') - # ── BDN.Extensions injection ──────────────────────────────────────────── def _find_bdn_extensions(self) -> str: @@ -241,8 +189,8 @@ def _patch_program_cs(self, program_cs_path: str): if insert_block: content = insert_block + content - # ManualConfig without MandatoryCategoryValidator (MAUI benchmarks - # don't use [BenchmarkCategory]) + # ManualConfig without MandatoryCategoryValidator (external benchmarks + # may not use [BenchmarkCategory]) new_run_call = ( 'var config = ManualConfig.Create(DefaultConfig.Instance)\n' ' .WithArtifactsPath(Path.Combine(\n' @@ -279,11 +227,11 @@ def _patch_program_cs(self, program_cs_path: str): log.info(' Patched successfully.') - def _patch_benchmark_projects(self): + def patch_benchmark_projects(self): '''Inject BDN.Extensions and PerfLabExporter into all benchmark projects.''' bdn_ext_abs = self._find_bdn_extensions() - for name, csproj_rel in MAUI_BENCHMARK_PROJECTS.items(): + for name, csproj_rel in self.benchmark_projects.items(): csproj_path = os.path.join(self.repo_dir, csproj_rel) project_dir = os.path.dirname(csproj_path) program_cs = os.path.join(project_dir, 'Program.cs') @@ -301,12 +249,12 @@ def _patch_benchmark_projects(self): # ── Build benchmarks ──────────────────────────────────────────────────── - def _build_benchmark_projects(self, suite: str): + def build_benchmark_projects(self, suite: str): log = getLogger() if suite == 'all': - projects = MAUI_BENCHMARK_PROJECTS.items() + projects = self.benchmark_projects.items() else: - projects = [(suite, MAUI_BENCHMARK_PROJECTS[suite])] + projects = [(suite, self.benchmark_projects[suite])] for name, csproj_rel in projects: csproj_path = os.path.join(self.repo_dir, csproj_rel) diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 31c77cc09b7..074fa7fdf89 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -18,7 +18,6 @@ DEVICEMEMORYCONSUMPTION = "devicememoryconsumption" ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" -BDNDESKTOP = "bdndesktop" BUILDTIME = "buildtime" SCENARIO_NAMES = {STARTUP: 'Startup', diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 73af2fb27e2..e54e9536e2c 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -18,7 +18,6 @@ from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper -from shared.bdndesktop import BDNDesktopHelper from shared.devicepowerconsumption import DevicePowerConsumptionHelper from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper @@ -175,16 +174,6 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) - bdndesktopparser = subparsers.add_parser(const.BDNDESKTOP, - description='Run BenchmarkDotNet benchmarks from an external repo on desktop') - bdndesktopparser.add_argument('--framework', '-f', default='net11.0', - help='Target .NET framework (determines repo branch)', dest='framework') - bdndesktopparser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], - default='all', help='Which benchmark suite to run', dest='suite') - bdndesktopparser.add_argument('--bdn-args', nargs='*', default=[], - help='Additional arguments to pass to BenchmarkDotNet', dest='bdnargs') - self.add_common_arguments(bdndesktopparser) - args = parser.parse_args() if not args.testtype: @@ -242,10 +231,6 @@ def parseargs(self): self.runtimeseconds = args.runtimeseconds self.closeToStartDelay = args.closeToStartDelay - if self.testtype == const.BDNDESKTOP: - self.framework = args.framework - self.suite = args.suite - self.bdnargs = args.bdnargs if args.scenarioname: self.scenarioname = args.scenarioname @@ -444,10 +429,6 @@ def run(self): androidInstrumentation = AndroidInstrumentationHelper() androidInstrumentation.runtests(self.packagepath, self.packagename, self.instrumentationname, self.upload_to_perflab_container) - elif self.testtype == const.BDNDESKTOP: - bdnDesktop = BDNDesktopHelper() - bdnDesktop.runtests(self.framework, self.suite, self.bdnargs, self.upload_to_perflab_container) - elif self.testtype == const.DEVICEPOWERCONSUMPTION: devicePowerConsumption = DevicePowerConsumptionHelper() self.traits.add_traits(overwrite=True, apptorun="app", powerconsumptionmetric=const.POWERCONSUMPTION_ANDROID, tracefolder='PerfTest/', tracename='runoutput.trace', scenarioname=self.scenarioname) From 33da2b9be9d18c46e2452041c2c25b212af213b6 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Mar 2026 14:38:43 -0700 Subject: [PATCH 4/8] Address PR review feedback: fix long-path cleanup, BDN injection safety, semicolon handling - Use performance.common.remove_directory() instead of shutil.rmtree for Windows long-path handling in test.py and post.py (C4, C5) - Narrow BDN PackageReference removal to exact package names instead of prefix match to preserve optional subpackages (C1) - Always emit trailing semicolon in patched Program.cs to avoid invalid C# when source pattern lacks semicolon (C2) - Add whitespace tolerance to Directory.Build.props regex and log when a requested property has zero matches (C9) - Harmonize report glob pattern to *perf-lab-report.json across collection and upload paths (C11) - Remove whitespace-only change in runner.py (C8) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/post.py | 4 ++-- src/scenarios/mauiDesktopBenchmarks/test.py | 4 ++-- src/scenarios/shared/bdndesktop.py | 20 ++++++++++++-------- src/scenarios/shared/runner.py | 1 - 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/post.py b/src/scenarios/mauiDesktopBenchmarks/post.py index 69cd572c68b..42b86ae153a 100644 --- a/src/scenarios/mauiDesktopBenchmarks/post.py +++ b/src/scenarios/mauiDesktopBenchmarks/post.py @@ -3,7 +3,7 @@ Cleans up the cloned maui repo and temporary artifacts. ''' import os -import shutil +from performance.common import remove_directory from performance.logger import setup_loggers, getLogger setup_loggers(True) @@ -16,7 +16,7 @@ def cleanup(): """Remove the cloned maui repository and any leftover artifacts.""" if os.path.exists(MAUI_REPO_DIR): log.info(f'Removing cloned MAUI repo: {MAUI_REPO_DIR}') - shutil.rmtree(MAUI_REPO_DIR, ignore_errors=True) + remove_directory(MAUI_REPO_DIR) # Clean up combined report if still in working directory combined = 'combined-perf-lab-report.json' diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index f94470f3e14..76282de1411 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -8,10 +8,10 @@ Usage: test.py --framework net11.0 --suite all ''' import os -import shutil import subprocess from argparse import ArgumentParser from logging import getLogger +from performance.common import remove_directory from performance.logger import setup_loggers from shared.bdndesktop import BDNDesktopHelper @@ -61,7 +61,7 @@ def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') if os.path.exists(repo_dir): - shutil.rmtree(repo_dir) + remove_directory(repo_dir) subprocess.run([ 'git', 'clone', diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py index 9fa1568b06f..c5c5fec13b9 100644 --- a/src/scenarios/shared/bdndesktop.py +++ b/src/scenarios/shared/bdndesktop.py @@ -103,10 +103,12 @@ def patch_directory_build_props(self): content = f.read() for prop_name, new_value in self.disable_props.items(): - pattern = rf'(<{prop_name}\b[^>]*>)true()' + pattern = rf'(<{prop_name}\b[^>]*>)\s*true\s*()' content, count = re.subn(pattern, rf'\g<1>{new_value}\g<2>', content) if count > 0: log.info(f' {prop_name}: replaced {count} occurrence(s)') + else: + log.warning(f' {prop_name}: no occurrences of "true" found to replace') with open(props_path, 'w', encoding='utf-8') as f: f.write(content) @@ -151,11 +153,14 @@ def _inject_bdn_extensions(self, csproj_path: str, bdn_ext_abs: str): if root.tag.startswith('{'): ns = root.tag.split('}')[0] + '}' - # Remove existing BenchmarkDotNet PackageReference to avoid version conflicts + # Remove the primary BenchmarkDotNet PackageReference to avoid version conflicts. + # Only remove exact 'BenchmarkDotNet' to preserve optional subpackages + # (e.g. BenchmarkDotNet.Annotations, BenchmarkDotNet.Diagnostics.*). + bdn_packages_to_remove = {'BenchmarkDotNet', 'BenchmarkDotNet.Annotations'} for item_group in root.findall(f'{ns}ItemGroup'): for pkg_ref in item_group.findall(f'{ns}PackageReference'): include = pkg_ref.get('Include', '') - if include.startswith('BenchmarkDotNet'): + if include in bdn_packages_to_remove: item_group.remove(pkg_ref) log.info(f' Removed PackageReference: {include}') @@ -200,7 +205,7 @@ def _patch_program_cs(self, program_cs_path: str): ' config = config.AddExporter(new PerfLabExporter());\n' ' BenchmarkSwitcher\n' ' .FromAssembly(typeof(Program).Assembly)\n' - ' .Run(args, config)' + ' .Run(args, config);' ) patterns = [ @@ -213,8 +218,7 @@ def _patch_program_cs(self, program_cs_path: str): replaced = False for pattern in patterns: if pattern in content: - suffix = ';' if pattern.endswith(';') else '' - content = content.replace(pattern, new_run_call + suffix) + content = content.replace(pattern, new_run_call) replaced = True break @@ -305,11 +309,11 @@ def _collect_results(self, upload_to_perflab_container: bool): log = getLogger() upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') - report_pattern = os.path.join(self.repo_dir, '**', '*-perf-lab-report.json') + report_pattern = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') report_files = glob.glob(report_pattern, recursive=True) if not report_files: - log.warning('No perf-lab-report.json files found. ' + log.warning('No *perf-lab-report.json files found. ' 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') return diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index e54e9536e2c..165f0e882ba 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -231,7 +231,6 @@ def parseargs(self): self.runtimeseconds = args.runtimeseconds self.closeToStartDelay = args.closeToStartDelay - if args.scenarioname: self.scenarioname = args.scenarioname From 9e3228588c3f3d1ddb8bf62ce9f3ee84d128772f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Mar 2026 14:49:29 -0700 Subject: [PATCH 5/8] Enable MAUI Desktop BDN pipeline jobs (public and private) Remove the if false guards from both the public (win-x64) and private (win-x64-viper) MAUI Desktop BenchmarkDotNet pipeline entries. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 54 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4ee4268a2f5..f07da074131 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -57,20 +57,19 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # MAUI Desktop BenchmarkDotNet benchmarks - - ${{ if false }}: - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - isPublic: true - jobParameters: - runKind: maui_desktop_benchmarks - projectFileName: maui_desktop_benchmarks.proj - channels: - - main - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + isPublic: true + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Blazor scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -603,20 +602,19 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # MAUI Desktop BDN benchmarks (private) - - ${{ if false }}: - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - isPublic: false - jobParameters: - runKind: maui_desktop_benchmarks - projectFileName: maui_desktop_benchmarks.proj - channels: - - main - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + isPublic: false + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml From 9d9b3cad2b8163107302f0df0a1bce92aac92d50 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 23 Mar 2026 15:15:20 -0700 Subject: [PATCH 6/8] Fix git not found on Helix: search common paths + zip fallback Helix work items don't have git on PATH. Add _find_git() to check common Windows install locations (Program Files\Git\cmd\git.exe). If git is truly unavailable, fall back to downloading the GitHub archive as a zip and extracting only the sparse checkout directories plus root-level files (Directory.Build.props, NuGet.config, etc.). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/test.py | 101 ++++++++++++++++++-- 1 file changed, 92 insertions(+), 9 deletions(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index 76282de1411..96aa6635603 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -8,7 +8,11 @@ Usage: test.py --framework net11.0 --suite all ''' import os +import shutil import subprocess +import sys +import urllib.request +import zipfile from argparse import ArgumentParser from logging import getLogger from performance.common import remove_directory @@ -55,16 +59,31 @@ def get_branch(framework: str) -> str: return 'net11.0' -def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): - '''Sparse-clone dotnet/maui at the given branch.''' - log = getLogger() - log.info(f'Cloning dotnet/maui branch {branch} (sparse, depth 1)...') +def _find_git() -> str: + '''Find the git executable on PATH or at common Windows locations.''' + git = shutil.which('git') + if git: + return git - if os.path.exists(repo_dir): - remove_directory(repo_dir) + if sys.platform == 'win32': + for candidate in [ + os.path.join(os.environ.get('ProgramFiles', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramFiles(x86)', r'C:\Program Files (x86)'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramW6432', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + ]: + if os.path.isfile(candidate): + return candidate + + return None + + +def _git_sparse_clone(git: str, branch: str, repo_dir: str): + '''Clone using git sparse checkout (preferred — smaller download).''' + log = getLogger() + log.info(f'Using git at: {git}') subprocess.run([ - 'git', 'clone', + git, 'clone', '-c', 'core.longpaths=true', '--depth', '1', '--filter=blob:none', @@ -75,10 +94,74 @@ def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): ], check=True) subprocess.run( - ['git', 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, + [git, 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, cwd=repo_dir, check=True) - log.info('Clone complete.') + +def _zip_download(branch: str, repo_dir: str): + '''Download the repo as a zip archive and extract needed directories. + + Fallback when git is not available (e.g. Helix work items where git is + not on PATH and not installed). + ''' + log = getLogger() + archive_url = f'https://github.com/dotnet/maui/archive/refs/heads/{branch}.zip' + zip_path = 'maui_download.zip' + + log.info(f'git not found — downloading archive from {archive_url}') + urllib.request.urlretrieve(archive_url, zip_path) + log.info(f'Downloaded {os.path.getsize(zip_path) / (1024*1024):.1f} MB') + + os.makedirs(repo_dir, exist_ok=True) + + # Directories to extract (sparse checkout equivalent + root-level files) + sparse_prefixes = [d.rstrip('/') + '/' for d in MAUI_SPARSE_CHECKOUT_DIRS] + + with zipfile.ZipFile(zip_path) as zf: + # GitHub archives have a top-level dir like "maui-net11.0/" + top_dir = zf.namelist()[0].split('/')[0] + '/' + + for member in zf.namelist(): + if not member.startswith(top_dir): + continue + rel_path = member[len(top_dir):] + if not rel_path: + continue + + # Include root-level files and our sparse directories + is_root_file = '/' not in rel_path + in_sparse_dir = any(rel_path.startswith(p) for p in sparse_prefixes) + + if not is_root_file and not in_sparse_dir: + continue + + target = os.path.join(repo_dir, rel_path) + if member.endswith('/'): + os.makedirs(target, exist_ok=True) + else: + os.makedirs(os.path.dirname(target), exist_ok=True) + with zf.open(member) as src, open(target, 'wb') as dst: + dst.write(src.read()) + + os.remove(zip_path) + log.info('Archive extracted.') + + +def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): + '''Clone or download dotnet/maui at the given branch.''' + log = getLogger() + log.info(f'Acquiring dotnet/maui branch {branch}...') + + if os.path.exists(repo_dir): + remove_directory(repo_dir) + + git = _find_git() + if git: + _git_sparse_clone(git, branch, repo_dir) + else: + _zip_download(branch, repo_dir) + + log.info('MAUI source acquired.') def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): From 2ddfd3e748cfcfd2d0873485c399543e47213d84 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 24 Mar 2026 12:34:23 -0700 Subject: [PATCH 7/8] Fix SSL cert error on Helix: use curl.exe for zip download Python's bundled SSL certificates don't include the CA certs trusted by Helix machines, causing CERTIFICATE_VERIFY_FAILED. Use curl.exe (built into Windows 10+/Server 2016+) which uses the Windows certificate store instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/test.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index 96aa6635603..ef2ac773c00 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -103,13 +103,26 @@ def _zip_download(branch: str, repo_dir: str): Fallback when git is not available (e.g. Helix work items where git is not on PATH and not installed). + + Uses curl.exe (built into Windows 10+) for the download because Python's + bundled SSL certificates may not include the CA certs trusted by the + machine (common on Helix/corporate environments). ''' log = getLogger() archive_url = f'https://github.com/dotnet/maui/archive/refs/heads/{branch}.zip' zip_path = 'maui_download.zip' log.info(f'git not found — downloading archive from {archive_url}') - urllib.request.urlretrieve(archive_url, zip_path) + + # Use curl.exe (ships with Windows 10+/Server 2016+) which uses the + # Windows certificate store, avoiding Python SSL cert issues on Helix. + curl = shutil.which('curl') or shutil.which('curl.exe') + if curl: + subprocess.run([curl, '-L', '-o', zip_path, '--fail', '-s', '-S', archive_url], check=True) + else: + # Last resort: try urllib with default certs + urllib.request.urlretrieve(archive_url, zip_path) + log.info(f'Downloaded {os.path.getsize(zip_path) / (1024*1024):.1f} MB') os.makedirs(repo_dir, exist_ok=True) From 54f1ed1d4d6c07f74b3604574eae5dde32b09d7d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 24 Mar 2026 14:56:01 -0700 Subject: [PATCH 8/8] Fix zip extraction: include parent directory files (src/*.targets) Git sparse-checkout automatically includes files in parent directories of checked-out paths. The zip fallback wasn't extracting files like src/MultiTargeting.targets and src/PublicAPI.targets because they live at the src/ level, outside the specific sparse checkout subdirectories. Compute parent directories from the sparse checkout list and include files directly in those parent dirs (non-recursively). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/test.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index ef2ac773c00..d7686bbdbc6 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -127,8 +127,17 @@ def _zip_download(branch: str, repo_dir: str): os.makedirs(repo_dir, exist_ok=True) - # Directories to extract (sparse checkout equivalent + root-level files) + # Directories to extract (sparse checkout equivalent + root-level files). + # Also include files directly in parent directories of sparse dirs + # (e.g. src/MultiTargeting.targets, src/PublicAPI.targets) since git + # sparse-checkout includes parent-level files automatically. sparse_prefixes = [d.rstrip('/') + '/' for d in MAUI_SPARSE_CHECKOUT_DIRS] + parent_dirs = set() + for d in MAUI_SPARSE_CHECKOUT_DIRS: + parts = d.strip('/').split('/') + for i in range(1, len(parts)): + parent_dirs.add('/'.join(parts[:i]) + '/') + parent_dirs = list(parent_dirs) # e.g. ['src/'] with zipfile.ZipFile(zip_path) as zf: # GitHub archives have a top-level dir like "maui-net11.0/" @@ -141,11 +150,16 @@ def _zip_download(branch: str, repo_dir: str): if not rel_path: continue - # Include root-level files and our sparse directories + # Include: root-level files, sparse directories, and files + # directly in parent directories (not recursing into subdirs) is_root_file = '/' not in rel_path in_sparse_dir = any(rel_path.startswith(p) for p in sparse_prefixes) + in_parent_dir = any( + rel_path.startswith(p) and '/' not in rel_path[len(p):] + for p in parent_dirs + ) - if not is_root_file and not in_sparse_dir: + if not is_root_file and not in_sparse_dir and not in_parent_dir: continue target = os.path.join(repo_dir, rel_path)