diff --git a/eng/performance/maui_desktop_benchmarks.proj b/eng/performance/maui_desktop_benchmarks.proj new file mode 100644 index 00000000000..047a4f9f9b5 --- /dev/null +++ b/eng/performance/maui_desktop_benchmarks.proj @@ -0,0 +1,19 @@ + + + + + + + + + $(ScenariosDir)mauiDesktopBenchmarks + $(Python) pre.py -f $(PERFLAB_Framework) + $(Python) test.py --framework $(PERFLAB_Framework) --suite all + $(Python) post.py + + + + + diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..760d18e2cc9 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -19,306 +19,321 @@ jobs: - ${{ if parameters.runPublicJobs }}: - # Scenario benchmarks + # # Scenario benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: scenarios + # projectFileName: scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # MAUI scenario benchmarks + # - ${{ if false }}: + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: maui_scenarios + # projectFileName: maui_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # MAUI Desktop BenchmarkDotNet benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # MAUI scenario benchmarks - - ${{ if false }}: - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: maui_scenarios - projectFileName: maui_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Blazor scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: blazor_scenarios - projectFileName: blazor_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # SDK scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - #- win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: true - jobParameters: - runKind: sdk_scenarios - projectFileName: sdk_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # micro benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - win-x86 - isPublic: true - jobParameters: - runKind: micro - targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - 9.0 - - 8.0 - - # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: micro - targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - # - nativeaot9.0 # Disable until I have time to properly fix the issues and can merge https://github.com/dotnet/performance/pull/4741 - # - nativeaot8.0 - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # ML.NET benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: mlnet - targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: fsharp - targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: fsharpmicro - targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # bepuphysics benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 isPublic: true jobParameters: - runKind: bepuphysics - targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj channels: - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # ImageSharp benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: imagesharp - targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Akade.IndexedSet benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: akadeindexedset - targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Roslyn benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: roslyn - targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 9.0 - - 8.0 ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - - ${{ if false }}: - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: illink - targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # NativeAOT scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Powershell benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: powershell - targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # # Blazor scenario benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: blazor_scenarios + # projectFileName: blazor_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # SDK scenario benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + # buildMachines: + # #- win-x86 + # #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + # isPublic: true + # jobParameters: + # runKind: sdk_scenarios + # projectFileName: sdk_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # micro benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - win-x86 + # isPublic: true + # jobParameters: + # runKind: micro + # targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: + # - main + # - 9.0 + # - 8.0 + + # # Ubuntux64 Default and NativeAOT micro benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: micro + # targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + # runCategories: 'runtime libraries' + # channels: + # - main + # # - nativeaot9.0 # Disable until I have time to properly fix the issues and can merge https://github.com/dotnet/performance/pull/4741 + # # - nativeaot8.0 + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # ML.NET benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: mlnet + # targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + # runCategories: 'mldotnet' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # F# benchmarks + # - ${{ if false }}: # skipping, no useful benchmarks there currently + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: fsharp + # targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + # runCategories: 'fsharp' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: fsharpmicro + # targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + # runCategories: 'FSharpMicro' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # bepuphysics benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: bepuphysics + # targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + # runCategories: 'BepuPhysics' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # ImageSharp benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: imagesharp + # targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + # runCategories: 'ImageSharp' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # Akade.IndexedSet benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: akadeindexedset + # targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + # runCategories: 'AkadeIndexedSet' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # Roslyn benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: roslyn + # targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + # runCategories: 'roslyn' + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # ILLink benchmarks + # # disabled because of: https://github.com/dotnet/performance/issues/3569 + # - ${{ if false }}: + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: illink + # targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + # runCategories: 'illink' + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # NativeAOT scenario benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: nativeaot_scenarios + # projectFileName: nativeaot_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # Powershell benchmarks + # - template: /eng/pipelines/templates/build-machine-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/templates/run-performance-job.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # runKind: powershell + # targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + # runCategories: 'Public' + # channels: + # - main + # - 9.0 + # - 8.0 + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} ########################################### # Private Jobs @@ -586,6 +601,21 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # MAUI Desktop BDN benchmarks (private) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + isPublic: false + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # NativeAOT scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: diff --git a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs index c60d3cb1db4..4d1cd029c24 100644 --- a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Extensions { - class ExclusionFilter : IFilter + public class ExclusionFilter : IFilter { private readonly GlobFilter? globFilter; diff --git a/src/scenarios/mauiDesktopBenchmarks/post.py b/src/scenarios/mauiDesktopBenchmarks/post.py new file mode 100644 index 00000000000..42b86ae153a --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/post.py @@ -0,0 +1,30 @@ +''' +Post-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Cleans up the cloned maui repo and temporary artifacts. +''' +import os +from performance.common import remove_directory +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +MAUI_REPO_DIR = 'maui_repo' + + +def cleanup(): + """Remove the cloned maui repository and any leftover artifacts.""" + if os.path.exists(MAUI_REPO_DIR): + log.info(f'Removing cloned MAUI repo: {MAUI_REPO_DIR}') + remove_directory(MAUI_REPO_DIR) + + # Clean up combined report if still in working directory + combined = 'combined-perf-lab-report.json' + if os.path.exists(combined): + os.remove(combined) + + log.info('Post-commands cleanup complete.') + + +if __name__ == '__main__': + cleanup() diff --git a/src/scenarios/mauiDesktopBenchmarks/pre.py b/src/scenarios/mauiDesktopBenchmarks/pre.py new file mode 100644 index 00000000000..99c7e37499f --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/pre.py @@ -0,0 +1,19 @@ +''' +Pre-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Kept minimal — all heavy lifting (clone, build, patch, run) is in test.py +to keep the correlation payload small. +''' +import argparse +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='MAUI Desktop BDN Benchmarks - Pre-commands') + parser.add_argument('-f', '--framework', default='net11.0', + help='Target .NET framework (determines MAUI branch)') + args = parser.parse_args() + log.info(f'MAUI Desktop BDN Benchmarks pre-commands (framework={args.framework})') + log.info('Setup deferred to test.py to minimize correlation payload.') + diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py new file mode 100644 index 00000000000..66cb04d6455 --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -0,0 +1,254 @@ +''' +MAUI Desktop BenchmarkDotNet benchmarks. + +Handles MAUI-specific setup (clone, branch mapping, dependency build) then +delegates to the shared BDNDesktopHelper for the generic BDN workflow +(patch, build benchmarks, run, collect results). + +Usage: test.py --framework net11.0 --suite all +''' +import os +import shutil +import subprocess +import sys +import urllib.request +import zipfile +from argparse import ArgumentParser +from logging import getLogger +from performance.common import remove_directory +from performance.logger import setup_loggers +from shared.bdndesktop import BDNDesktopHelper + +# ── MAUI-specific configuration ───────────────────────────────────────────── + +MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' +MAUI_REPO_DIR = 'maui_repo' + +MAUI_BENCHMARK_PROJECTS = { + 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', + 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', + 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', +} + +MAUI_SPARSE_CHECKOUT_DIRS = [ + 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', + 'src/Workload', 'src/Essentials', + 'eng', '.config', +] + +MAUI_BUILD_SOLUTION_FILTER = 'Microsoft.Maui.BuildTasks.slnf' + +# MSBuild properties to disable non-desktop target frameworks. +# MAUI's Directory.Build.props sets these to true unconditionally at multiple +# points; MauiPlatforms is computed from them. In-place replacement is +# required because appending overrides at the end doesn't work (MSBuild +# evaluates top-to-bottom). +DESKTOP_ONLY_PROPS = { + 'IncludeAndroidTargetFrameworks': 'false', + 'IncludeIosTargetFrameworks': 'false', + 'IncludeMacCatalystTargetFrameworks': 'false', + 'IncludeMacOSTargetFrameworks': 'false', + 'IncludeTizenTargetFrameworks': 'false', +} + +# Benchmarks to exclude: these emit millions of log lines per iteration, +# bloating output and slowing runs (BindableProperty readonly errors). +EXCLUDED_BENCHMARKS = [ + '*MauiLoggerWithLoggerMinLevelErrorBenchmarker*', +] + + +def get_branch(framework: str) -> str: + '''Map framework moniker to MAUI repo branch.''' + if framework and framework.startswith('net'): + return framework # net11.0 -> net11.0, net10.0 -> net10.0 + return 'net11.0' + + +def _find_git() -> str: + '''Find the git executable on PATH or at common Windows locations.''' + git = shutil.which('git') + if git: + return git + + if sys.platform == 'win32': + for candidate in [ + os.path.join(os.environ.get('ProgramFiles', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramFiles(x86)', r'C:\Program Files (x86)'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramW6432', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + ]: + if os.path.isfile(candidate): + return candidate + + return None + + +def _git_sparse_clone(git: str, branch: str, repo_dir: str): + '''Clone using git sparse checkout (preferred — smaller download).''' + log = getLogger() + log.info(f'Using git at: {git}') + + subprocess.run([ + git, 'clone', + '-c', 'core.longpaths=true', + '--depth', '1', + '--filter=blob:none', + '--sparse', + '--branch', branch, + MAUI_REPO_URL, + repo_dir + ], check=True) + + subprocess.run( + [git, 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, + cwd=repo_dir, check=True) + + +def _zip_download(branch: str, repo_dir: str): + '''Download the repo as a zip archive and extract needed directories. + + Fallback when git is not available (e.g. Helix work items where git is + not on PATH and not installed). + + Uses curl.exe (built into Windows 10+) for the download because Python's + bundled SSL certificates may not include the CA certs trusted by the + machine (common on Helix/corporate environments). + ''' + log = getLogger() + archive_url = f'https://github.com/dotnet/maui/archive/refs/heads/{branch}.zip' + zip_path = 'maui_download.zip' + + log.info(f'git not found — downloading archive from {archive_url}') + + # Use curl.exe (ships with Windows 10+/Server 2016+) which uses the + # Windows certificate store, avoiding Python SSL cert issues on Helix. + curl = shutil.which('curl') or shutil.which('curl.exe') + if curl: + subprocess.run([curl, '-L', '-o', zip_path, '--fail', '-s', '-S', archive_url], check=True) + else: + # Last resort: try urllib with default certs + urllib.request.urlretrieve(archive_url, zip_path) + + log.info(f'Downloaded {os.path.getsize(zip_path) / (1024*1024):.1f} MB') + + os.makedirs(repo_dir, exist_ok=True) + + # Directories to extract (sparse checkout equivalent + root-level files). + # Also include files directly in parent directories of sparse dirs + # (e.g. src/MultiTargeting.targets, src/PublicAPI.targets) since git + # sparse-checkout includes parent-level files automatically. + sparse_prefixes = [d.rstrip('/') + '/' for d in MAUI_SPARSE_CHECKOUT_DIRS] + parent_dirs = set() + for d in MAUI_SPARSE_CHECKOUT_DIRS: + parts = d.strip('/').split('/') + for i in range(1, len(parts)): + parent_dirs.add('/'.join(parts[:i]) + '/') + parent_dirs = list(parent_dirs) # e.g. ['src/'] + + with zipfile.ZipFile(zip_path) as zf: + # GitHub archives have a top-level dir like "maui-net11.0/" + top_dir = zf.namelist()[0].split('/')[0] + '/' + + for member in zf.namelist(): + if not member.startswith(top_dir): + continue + rel_path = member[len(top_dir):] + if not rel_path: + continue + + # Include: root-level files, sparse directories, and files + # directly in parent directories (not recursing into subdirs) + is_root_file = '/' not in rel_path + in_sparse_dir = any(rel_path.startswith(p) for p in sparse_prefixes) + in_parent_dir = any( + rel_path.startswith(p) and '/' not in rel_path[len(p):] + for p in parent_dirs + ) + + if not is_root_file and not in_sparse_dir and not in_parent_dir: + continue + + target = os.path.join(repo_dir, rel_path) + if member.endswith('/'): + os.makedirs(target, exist_ok=True) + else: + os.makedirs(os.path.dirname(target), exist_ok=True) + with zf.open(member) as src, open(target, 'wb') as dst: + dst.write(src.read()) + + os.remove(zip_path) + log.info('Archive extracted.') + + +def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): + '''Clone or download dotnet/maui at the given branch.''' + log = getLogger() + log.info(f'Acquiring dotnet/maui branch {branch}...') + + if os.path.exists(repo_dir): + remove_directory(repo_dir) + + git = _find_git() + if git: + _git_sparse_clone(git, branch, repo_dir) + else: + _zip_download(branch, repo_dir) + + log.info('MAUI source acquired.') + + +def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): + '''Restore dotnet tools and build MAUI's BuildTasks solution filter.''' + log = getLogger() + log.info('Restoring dotnet tools...') + subprocess.run(['dotnet', 'tool', 'restore'], cwd=repo_dir, check=True) + + log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') + subprocess.run([ + 'dotnet', 'build', + MAUI_BUILD_SOLUTION_FILTER, + '-c', 'Release', + ], cwd=repo_dir, check=True) + + log.info('MAUI dependencies built successfully.') + + +def parse_args(): + parser = ArgumentParser(description='Run MAUI desktop BDN benchmarks') + parser.add_argument('--framework', '-f', default='net11.0', + help='Target .NET framework (determines MAUI repo branch)') + parser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], + default='all', help='Which benchmark suite to run') + parser.add_argument('--bdn-args', nargs='*', default=[], + help='Additional arguments to pass to BenchmarkDotNet') + parser.add_argument('--upload-to-perflab-container', action='store_true', + help='Upload results to perflab container') + return parser.parse_args() + + +if __name__ == '__main__': + setup_loggers(True) + args = parse_args() + + # MAUI-specific: clone repo and build dependencies + branch = get_branch(args.framework) + clone_maui_repo(branch) + + # Generic BDN desktop workflow: patch, build benchmarks, run, collect + helper = BDNDesktopHelper( + repo_dir=MAUI_REPO_DIR, + benchmark_projects=MAUI_BENCHMARK_PROJECTS, + disable_props=DESKTOP_ONLY_PROPS, + ) + + # Patch Directory.Build.props BEFORE any builds (including MAUI deps) + helper.patch_directory_build_props() + + # MAUI-specific: build BuildTasks solution filter + build_maui_dependencies() + + # Run the generic BDN workflow + bdn_args = list(args.bdn_args) + if EXCLUDED_BENCHMARKS: + bdn_args.extend(['--exclusion-filter'] + EXCLUDED_BENCHMARKS) + helper.runtests(args.suite, bdn_args, args.upload_to_perflab_container) diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py new file mode 100644 index 00000000000..b7f647044ea --- /dev/null +++ b/src/scenarios/shared/bdndesktop.py @@ -0,0 +1,392 @@ +''' +Reusable helper for running BenchmarkDotNet benchmarks from external repos +on desktop. + +Handles the generic workflow: + 1. Patch Directory.Build.props to disable unwanted target frameworks + 2. Inject BDN.Extensions (PerfLabExporter) into benchmark projects + 3. Build benchmark projects + 4. Run BDN suites + 5. Collect and upload results + +Callers (e.g. test.py in each scenario) are responsible for repo-specific +setup such as cloning, branch selection, and building repo dependencies. +''' +import os +import re +import sys +import glob +import json +import shutil +import subprocess +import xml.etree.ElementTree as ET +from logging import getLogger +from performance.common import runninginlab + + +class BDNDesktopHelper(object): + '''Generic helper for running BDN desktop benchmarks from a local repo checkout. + + Args: + repo_dir: Path to the cloned repository root. + benchmark_projects: Dict mapping suite name to csproj relative path + (e.g. {'graphics': 'src/Graphics/.../Graphics.Benchmarks.csproj'}). + disable_props: Optional dict of MSBuild property names to replacement values + to patch in Directory.Build.props (e.g. disable mobile TFMs). + ''' + + def __init__(self, repo_dir: str, benchmark_projects: dict, + disable_props: dict = None): + self.repo_dir = repo_dir + self.benchmark_projects = benchmark_projects + self.disable_props = disable_props or {} + + # ── Public entry point ────────────────────────────────────────────────── + + def runtests(self, suite: str, bdn_args: list, + upload_to_perflab_container: bool): + ''' + Patch benchmark projects, build, run, and collect BDN results. + + Assumes the caller has already: + - Cloned the repo and built repo-specific dependencies + - Called patch_directory_build_props() if needed (must happen + before any builds, including dependency builds) + ''' + log = getLogger() + + # Patch benchmark csprojs + Program.cs for PerfLabExporter + self.patch_benchmark_projects() + + # Build + built_suites = self.build_benchmark_projects(suite) + + # Run (only suites that built successfully) + if suite == 'all': + suites = [(n, p) for n, p in self.benchmark_projects.items() if n in built_suites] + else: + suites = [(suite, self.benchmark_projects[suite])] + + all_passed = True + for name, csproj_rel in suites: + if not self._run_benchmark(name, csproj_rel, bdn_args): + all_passed = False + + # Collect + self._collect_results(upload_to_perflab_container) + + if not all_passed: + log.error('One or more benchmark suites failed.') + sys.exit(1) + + log.info('All benchmark suites completed.') + + # ── Patch Directory.Build.props ───────────────────────────────────────── + + def patch_directory_build_props(self): + '''Disable unwanted TFMs by replacing property values in-place. + + Handles repos (like MAUI) that set Include*TargetFrameworks=true at + multiple points before computing TargetFrameworks. Appending + overrides at the end doesn't work because MSBuild evaluates + top-to-bottom, so we regex-replace ALL occurrences in-place. + ''' + log = getLogger() + props_path = os.path.join(self.repo_dir, 'Directory.Build.props') + if not os.path.exists(props_path): + log.info('No Directory.Build.props found — skipping TFM patching.') + return + + log.info('Patching Directory.Build.props to disable unwanted TFMs...') + + with open(props_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + for prop_name, new_value in self.disable_props.items(): + pattern = rf'(<{prop_name}\b[^>]*>)\s*true\s*()' + content, count = re.subn(pattern, rf'\g<1>{new_value}\g<2>', content) + if count > 0: + log.info(f' {prop_name}: replaced {count} occurrence(s)') + else: + log.warning(f' {prop_name}: no occurrences of "true" found to replace') + + with open(props_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Directory.Build.props patched.') + + # ── BDN.Extensions injection ──────────────────────────────────────────── + + def _find_bdn_extensions(self) -> str: + '''Return the absolute path to BenchmarkDotNet.Extensions.csproj.''' + correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') + if correlation: + candidate = os.path.join( + correlation, 'performance', 'src', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') + else: + scenario_dir = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.normpath(os.path.join( + scenario_dir, '..', '..', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) + + if not os.path.exists(candidate): + raise FileNotFoundError( + f'BenchmarkDotNet.Extensions.csproj not found at {candidate}. ' + f'HELIX_CORRELATION_PAYLOAD={correlation!r}') + + getLogger().info(f'BDN.Extensions located at: {candidate}') + return candidate + + def _inject_bdn_extensions(self, csproj_path: str, bdn_ext_abs: str): + '''Add a ProjectReference to BDN.Extensions and remove existing BDN PackageRef.''' + log = getLogger() + log.info(f'Injecting BDN.Extensions reference into {os.path.basename(csproj_path)}') + + csproj_dir = os.path.dirname(os.path.abspath(csproj_path)) + bdn_ext_rel = os.path.relpath(bdn_ext_abs, csproj_dir) + + tree = ET.parse(csproj_path) + root = tree.getroot() + + ns = '' + if root.tag.startswith('{'): + ns = root.tag.split('}')[0] + '}' + + # Remove the primary BenchmarkDotNet PackageReference to avoid version conflicts. + # Only remove exact 'BenchmarkDotNet' to preserve optional subpackages + # (e.g. BenchmarkDotNet.Annotations, BenchmarkDotNet.Diagnostics.*). + bdn_packages_to_remove = {'BenchmarkDotNet', 'BenchmarkDotNet.Annotations'} + for item_group in root.findall(f'{ns}ItemGroup'): + for pkg_ref in item_group.findall(f'{ns}PackageReference'): + include = pkg_ref.get('Include', '') + if include in bdn_packages_to_remove: + item_group.remove(pkg_ref) + log.info(f' Removed PackageReference: {include}') + + # Add ProjectReference to BDN.Extensions + item_group = ET.SubElement(root, f'{ns}ItemGroup') + item_group.set('Label', 'PerfLabInjected') + proj_ref = ET.SubElement(item_group, f'{ns}ProjectReference') + proj_ref.set('Include', bdn_ext_rel) + + tree.write(csproj_path, xml_declaration=True, encoding='utf-8') + log.info(f' Added ProjectReference: {bdn_ext_rel}') + + def _patch_program_cs(self, program_cs_path: str): + '''Patch Program.cs to add PerfLabExporter via ManualConfig.''' + log = getLogger() + log.info(f'Patching {os.path.basename(program_cs_path)} for PerfLabExporter') + + with open(program_cs_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + usings_to_add = [ + 'using BenchmarkDotNet.Configs;', + 'using BenchmarkDotNet.Extensions;', + 'using System;', + 'using System.Collections.Generic;', + 'using System.IO;', + 'using System.Linq;', + ] + insert_block = '' + for u in usings_to_add: + if u not in content: + insert_block += u + '\n' + if insert_block: + content = insert_block + content + + # ManualConfig without MandatoryCategoryValidator (external benchmarks + # may not use [BenchmarkCategory]) + new_run_call = ( + 'var argsList = args.ToList();\n' + ' argsList = CommandLineOptions.ParseAndRemoveStringsParameter(\n' + ' argsList, "--exclusion-filter", out var exclusionFilterValue);\n' + ' var config = ManualConfig.Create(DefaultConfig.Instance)\n' + ' .WithArtifactsPath(Path.Combine(\n' + ' Path.GetDirectoryName(typeof(Program).Assembly.Location),\n' + ' "BenchmarkDotNet.Artifacts"))\n' + ' .AddFilter(new ExclusionFilter(exclusionFilterValue));\n' + ' if (Environment.GetEnvironmentVariable("PERFLAB_INLAB") == "1")\n' + ' config = config.AddExporter(new PerfLabExporter());\n' + ' BenchmarkSwitcher\n' + ' .FromAssembly(typeof(Program).Assembly)\n' + ' .Run(argsList.ToArray(), config);' + ) + + patterns = [ + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);', + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args)', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args);', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args)', + ] + + replaced = False + for pattern in patterns: + if pattern in content: + content = content.replace(pattern, new_run_call) + replaced = True + break + + if not replaced: + log.warning(' Could not find BenchmarkSwitcher.Run pattern — may need manual patching') + return + + with open(program_cs_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Patched successfully.') + + def patch_benchmark_projects(self): + '''Inject BDN.Extensions and PerfLabExporter into all benchmark projects.''' + bdn_ext_abs = self._find_bdn_extensions() + + for name, csproj_rel in self.benchmark_projects.items(): + csproj_path = os.path.join(self.repo_dir, csproj_rel) + project_dir = os.path.dirname(csproj_path) + program_cs = os.path.join(project_dir, 'Program.cs') + + if not os.path.exists(csproj_path): + getLogger().warning(f'Benchmark project not found: {csproj_path}') + continue + + self._inject_bdn_extensions(csproj_path, bdn_ext_abs) + + if os.path.exists(program_cs): + self._patch_program_cs(program_cs) + else: + getLogger().warning(f'Program.cs not found for {name}') + + # ── Build benchmarks ──────────────────────────────────────────────────── + + def build_benchmark_projects(self, suite: str) -> set: + '''Build benchmark projects. Returns the set of suite names that built successfully.''' + log = getLogger() + if suite == 'all': + projects = list(self.benchmark_projects.items()) + else: + projects = [(suite, self.benchmark_projects[suite])] + + built = set() + for name, csproj_rel in projects: + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found, skipping: {csproj_path}') + continue + + log.info(f'Building benchmark: {name}') + result = subprocess.run([ + 'dotnet', 'build', + csproj_rel, + '-c', 'Release', + ], cwd=self.repo_dir) + + if result.returncode == 0: + built.add(name) + else: + log.warning(f'Build failed for {name} (exit code {result.returncode}) — skipping this suite') + + if built: + log.info(f'Successfully built: {", ".join(sorted(built))}') + else: + log.error('No benchmark projects built successfully.') + sys.exit(1) + + failed = set(name for name, _ in projects) - built + if failed: + log.warning(f'The following suites failed to build and will be skipped: {", ".join(sorted(failed))}') + + return built + + # ── Run benchmarks ────────────────────────────────────────────────────── + + def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bool: + log = getLogger() + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found: {csproj_path}') + return False + + log.info(f'Running benchmark suite: {name}') + + cmd = [ + 'dotnet', 'run', + '-c', 'Release', + '--no-build', + '--project', csproj_rel, + '--', + '--filter', '*', + '--warmupCount', '1', + '--minIterationCount', '15', + '--maxIterationCount', '20', + '--iterationTime', '250', + ] + extra_bdn_args + + result = subprocess.run(cmd, cwd=self.repo_dir) + if result.returncode != 0: + log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') + return False + + log.info(f'Benchmark suite {name} completed successfully.') + return True + + # ── Result collection ─────────────────────────────────────────────────── + + def _collect_results(self, upload_to_perflab_container: bool): + '''Collect perf-lab-report.json files from BDN artifacts.''' + log = getLogger() + upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') + + report_pattern = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + report_files = glob.glob(report_pattern, recursive=True) + + if not report_files: + log.warning('No *perf-lab-report.json files found. ' + 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') + return + + log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') + + # Combine all reports into a single file + combined = [] + for report_file in report_files: + log.info(f' Collecting: {report_file}') + try: + with open(report_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + combined.extend(data) + else: + combined.append(data) + except (json.JSONDecodeError, IOError) as e: + log.warning(f' Failed to read {report_file}: {e}') + + if combined: + combined_path = 'combined-perf-lab-report.json' + with open(combined_path, 'w', encoding='utf-8') as f: + json.dump(combined, f, indent=2) + log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') + + if upload_root: + dest = os.path.join(upload_root, combined_path) + shutil.copy2(combined_path, dest) + log.info(f'Copied combined report to {dest}') + + for report_file in report_files: + basename = os.path.basename(report_file) + dest = os.path.join(upload_root, basename) + shutil.copy2(report_file, dest) + log.info(f' Copied {basename} to upload root') + + # Also upload via perflab container if requested + if upload_to_perflab_container and runninginlab(): + try: + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + import upload + globpath = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + upload_code = upload.upload(globpath, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + log.info(f'BDN Desktop Benchmarks Upload Code: {upload_code}') + if upload_code != 0: + sys.exit(upload_code) + except ImportError: + log.warning('Upload module not available — skipping perflab container upload')