Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .agents/skills/cosmos-provider/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,13 @@ Non-relational provider with its own parallel query pipeline. Uses JSON for docu
- Partition key configuration required for performance
- `ETag` for optimistic concurrency
- No cross-container joins

## Azure Cosmos DB Emulator in Docker

Cosmos tests on Helix start the emulator from the work item via `PreCommands` that run a Docker container using:
- `eng/testing/run-cosmos-container.ps1`
- `eng/testing/run-cosmos-container.sh`

These scripts can be invoked locally for testing on machines that don't have the emulator installed, but have docker available.

The `Test__Cosmos__SkipConnectionCheck=true` env var is set to prevent tests from being skipped when the emulator failed to start.
2 changes: 2 additions & 0 deletions .github/workflows/TestCosmos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ jobs:
- name: Test on Cosmos
run: dotnet test test/EFCore.Cosmos.FunctionalTests/EFCore.Cosmos.FunctionalTests.csproj
shell: cmd
env:
Test__Cosmos__SkipConnectionCheck: true

- name: Publish Test Results
uses: actions/upload-artifact@v7
Expand Down
20 changes: 18 additions & 2 deletions .github/workflows/copilot-setup-steps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,22 @@ jobs:
--health-retries=30
--health-timeout=5s

cosmosdb:
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
ports:
- 8081:8081
env:
PROTOCOL: https
ENABLE_EXPLORER: "false"
options: >-
--health-cmd="curl -k https://localhost:8081/ || exit 1"
--health-start-period=120s
--health-interval=5s
--health-retries=60
--health-timeout=5s

steps:
- name: Export connection string for the agent's session
run: echo 'Test__SqlServer__DefaultConnection=Server=localhost;Database=test;User=SA;Password=PLACEHOLDERPass$$w0rd;Connect Timeout=60;ConnectRetryCount=0;Trust Server Certificate=true' >> "$GITHUB_ENV"
- name: Export environment variables for the agent's session
run: |
echo 'Test__SqlServer__DefaultConnection=Server=localhost;Database=master;User=SA;Password=PLACEHOLDERPass$$w0rd;Connect Timeout=60;ConnectRetryCount=0;Trust Server Certificate=true' >> "$GITHUB_ENV"
echo 'Test__Cosmos__EmulatorType=linux' >> "$GITHUB_ENV"
2 changes: 1 addition & 1 deletion azure-pipelines-internal-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ extends:
- name: _HelixBuildConfig
value: $(_BuildConfig)
- name: HelixTargetQueues
value: Windows.10.Amd64;OSX.15.Amd64;OSX.15.ARM64;Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64
value: Windows.10.Amd64;OSX.15.Amd64;OSX.15.ARM64;Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64;Ubuntu.2204.Amd64.XL;Ubuntu.2204.Amd64
- name: _HelixAccessToken
# Needed for internal queues
value: $(HelixApiAccessToken)
Expand Down
2 changes: 1 addition & 1 deletion azure-pipelines-public.yml
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ stages:
- name: _HelixBuildConfig
value: $(_BuildConfig)
- name: HelixTargetQueues
value: Windows.10.Amd64.Open;OSX.15.Amd64.Open;Ubuntu.2204.Amd64.XL.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64
value: Windows.10.Amd64.Open;OSX.15.Amd64.Open;OSX.15.ARM64.Open;Ubuntu.2204.Amd64.XL.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64;Ubuntu.2204.Amd64.XL.Open;Ubuntu.2204.Amd64.Open
- name: _HelixAccessToken
value: '' # Needed for public queues
steps:
Expand Down
46 changes: 37 additions & 9 deletions eng/helix.proj
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@

<EnableAzurePipelinesReporter>true</EnableAzurePipelinesReporter>
<FailOnTestFailure>true</FailOnTestFailure>
<SqlServerTests>$(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj</SqlServerTests>
<SqlServerTests>$(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj</SqlServerTests>
<CosmosTests>$(RepoRoot)/test/EFCore.Cosmos.FunctionalTests/*.csproj</CosmosTests>
</PropertyGroup>

<PropertyGroup Condition = "'$(SYSTEM_ACCESSTOKEN)' == ''">
<!-- Local build outside of Azure Pipeline -->
<HelixTargetQueues Condition = "'$(HelixTargetQueues)' == ''">Windows.10.Amd64.Open;OSX.1200.Amd64.Open;OSX.1200.ARM64.Open;Ubuntu.2204.Amd64.XL.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64</HelixTargetQueues>
<HelixTargetQueues Condition = "'$(HelixTargetQueues)' == ''">Windows.10.Amd64.Open;OSX.15.Amd64.Open;OSX.15.ARM64.Open;Ubuntu.2204.Amd64.XL.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64;Ubuntu.2204.Amd64.XL.Open;Ubuntu.2204.Amd64.Open</HelixTargetQueues>
<EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
<HelixSource>efcore/localbuild/</HelixSource>
<HelixBuild>t001</HelixBuild>
Expand All @@ -33,6 +34,10 @@
<AdditionalDotNetPackage Include="$(MicrosoftNETCorePlatformsVersion)">
<PackageType>runtime</PackageType>
</AdditionalDotNetPackage>

<HelixCorrelationPayload Include="$(RepoRoot)/eng/testing">
<Destination>testing</Destination>
</HelixCorrelationPayload>
</ItemGroup>

<ItemGroup>
Expand All @@ -53,25 +58,48 @@
</XUnitProject>
</ItemGroup>

<!-- Start LocalDb instance for test projects which uses SqlServer on windows -->
<!-- Start LocalDb instance for test projects which uses SqlServer on Windows -->
<ItemGroup Condition = "'$(HelixTargetQueue.StartsWith(`Windows`))'">
<XUnitProject Update="$(SqlServerTests);$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj">
<XUnitProject Update="$(SqlServerTests)">
<PreCommands>$(PreCommands); SqlLocalDB start</PreCommands>
</XUnitProject>
</ItemGroup>

<ItemGroup Condition = "'$(HelixTargetQueue.StartsWith(`Windows.11`))'">
<XUnitProject Update="$(CosmosTests)">
<PreCommands>$(PreCommands); set Test__Cosmos__SkipConnectionCheck=true</PreCommands>
</XUnitProject>
</ItemGroup>

<!-- Start SqlServer instance for test projects which uses SqlServer on docker. Also remove other projects as they will be run outside of docker. -->
<ItemGroup Condition = "'$(HelixTargetQueue.Contains(`ubuntu-22.04-helix-sqlserver-amd64`))'">
<!-- Start SqlServer instance for test projects which uses SqlServer on docker. Only run SqlServer tests. -->
<ItemGroup Condition = "'$(HelixTargetQueue.Contains(`helix-sqlserver`))'">
<XUnitProject Remove="$(RepoRoot)/test/**/*.csproj"/>
<XUnitProject Remove="$(RepoRoot)/test/**/*.fsproj"/>
<XUnitProject Include="$(SqlServerTests);$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj">
<XUnitProject Include="$(SqlServerTests)">
<PreCommands>$(PreCommands); export MSSQL_SA_PASSWORD=$(MSSQL_SA_PASSWORD); export Test__SqlServer__DefaultConnection="Server=localhost;;Database=master;;User=sa;;Password=$(MSSQL_SA_PASSWORD);;Connect Timeout=60;;ConnectRetryCount=0;;TrustServerCertificate=True"; /opt/mssql/bin/sqlservr --accept-eula &amp;; sleep 120; </PreCommands>
</XUnitProject>
</ItemGroup>

<!-- Remove test projects which requires SqlServer from Ubuntu/OSX. -->
<ItemGroup Condition = "'$(HelixTargetQueue.StartsWith(`OSX`))' OR '$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64.XL.Open' OR '$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64.XL'">
<!-- Start Cosmos emulator in Docker on Ubuntu and only run Cosmos tests -->
<ItemGroup Condition = "'$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64.XL.Open' OR '$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64.XL'">
<XUnitProject Remove="$(RepoRoot)/test/**/*.csproj"/>
<XUnitProject Remove="$(RepoRoot)/test/**/*.fsproj"/>
<XUnitProject Include="$(CosmosTests)">
<PreCommands>$(PreCommands); chmod +x $HELIX_CORRELATION_PAYLOAD/testing/run-cosmos-container.sh; $HELIX_CORRELATION_PAYLOAD/testing/run-cosmos-container.sh; export Test__Cosmos__DefaultConnection=https://localhost:8081; export Test__Cosmos__SkipConnectionCheck=true; export Test__Cosmos__EmulatorType=linux</PreCommands>
<PostCommands>$(PostCommands); docker stop cosmos-emulator || true; docker rm -f cosmos-emulator || true</PostCommands>
</XUnitProject>
</ItemGroup>

<!-- Run tests that don't need SqlServer or Cosmos on bare Ubuntu -->
<ItemGroup Condition = "'$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64.Open' OR '$(HelixTargetQueue)' == 'Ubuntu.2204.Amd64'">
<XUnitProject Remove="$(SqlServerTests)"/>
<XUnitProject Remove="$(CosmosTests)"/>
</ItemGroup>

<!-- Remove test projects which require SqlServer or Cosmos from OSX -->
<ItemGroup Condition = "'$(HelixTargetQueue.StartsWith(`OSX`))'">
<XUnitProject Remove="$(SqlServerTests)"/>
<XUnitProject Remove="$(CosmosTests)"/>
</ItemGroup>

<PropertyGroup>
Expand Down
61 changes: 61 additions & 0 deletions eng/testing/run-cosmos-container.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Starts the Azure Cosmos DB Emulator in a Windows Docker container and waits for it to be ready.
# Tests run on the host machine connecting to the emulator via https://localhost:8081.
# Usage: .\run-cosmos-container.ps1
param()

$ErrorActionPreference = 'Stop'
$image = 'mcr.microsoft.com/cosmosdb/windows/azure-cosmos-emulator'
$containerName = 'cosmos-emulator'
$port = 8081
$maxRetries = 90
$retryDelaySec = 2

Write-Host "Pulling image: $image"
docker pull $image
if ($LASTEXITCODE -ne 0) { throw "docker pull failed with exit code $LASTEXITCODE" }

Write-Host "Checking for existing container named $containerName..."
$existingContainerId = docker ps -a --filter "name=^${containerName}$" --format '{{.ID}}'
if ($LASTEXITCODE -ne 0) { throw "docker ps failed with exit code $LASTEXITCODE" }
if ($existingContainerId) {
Write-Host "Existing container '$containerName' found. Stopping and removing it..."
docker stop $containerName 2>$null
docker rm -f $containerName
if ($LASTEXITCODE -ne 0) { throw "docker rm failed with exit code $LASTEXITCODE" }
}

# -t is required because Start.ps1 sets [Console]::BufferWidth which needs a TTY handle.
Write-Host "Starting Cosmos DB Emulator container on port $port..."
docker run -d -t `
--name $containerName `
--publish "${port}:8081" `
--memory 2G `
$image
if ($LASTEXITCODE -ne 0) { throw "docker run failed with exit code $LASTEXITCODE" }

Write-Host "Waiting for emulator to be ready (up to $($maxRetries * $retryDelaySec)s)..."
$ready = $false
for ($i = 0; $i -lt $maxRetries; $i++) {
Start-Sleep -Seconds $retryDelaySec
# Any HTTP response (even 401) means the emulator is up and accepting connections.
$null = & curl.exe -k "https://localhost:${port}/" --silent --output NUL --max-time 5
if ($LASTEXITCODE -eq 0) {
$ready = $true
} else {
Write-Host " Attempt $($i+1)/$maxRetries - not ready yet..."
}
if ($ready) {
Write-Host "Cosmos DB Emulator is ready on port $port."
break
}
}

if (-not $ready) {
Write-Host "Emulator did not become ready. Container logs:"
docker logs $containerName
docker stop $containerName 2>$null
docker rm -f $containerName 2>$null
exit 1
}

exit 0
51 changes: 51 additions & 0 deletions eng/testing/run-cosmos-container.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
# Starts the Azure Cosmos DB Linux (vNext) Emulator in a Docker container and waits for it to be ready.
# Tests run on the host machine connecting to the emulator via https://localhost:8081.
# The --protocol https flag is required because the .NET SDK does not support HTTP mode.
# Usage: ./run-cosmos-container.sh

set -e

image='mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview'
container_name='cosmos-emulator'
port=8081
max_retries=30
retry_delay=2

echo "Pulling image: $image"
docker pull "$image"

if docker ps -a --format '{{.Names}}' | grep -Eq "^${container_name}\$"; then
echo "Removing existing Cosmos DB Emulator container: $container_name"
docker rm -f "$container_name"
fi

echo "Starting Cosmos DB Emulator container on port $port with HTTPS..."
docker run -d \
--name "$container_name" \
--publish "${port}:8081" \
"$image" \
--protocol https \
--enable-explorer false

echo "Waiting for emulator to be ready (up to ~$((max_retries * retry_delay))s)..."
ready=false
for i in $(seq 1 "$max_retries"); do
sleep "$retry_delay"
if curl -ks --connect-timeout "$retry_delay" --max-time "$retry_delay" "https://localhost:${port}/" -o /dev/null; then
ready=true
echo "Cosmos DB Emulator is ready."
break
fi
echo " Attempt $i/$max_retries - not ready yet..."
done

if [ "$ready" != true ]; then
echo "Emulator did not become ready. Container logs:"
docker logs "$container_name"
docker stop "$container_name" 2>/dev/null || true
docker rm -f "$container_name" 2>/dev/null || true
exit 1
fi

exit 0
3 changes: 1 addition & 2 deletions src/EFCore/Query/Internal/QueryCompiler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,7 @@ public virtual Expression ExtractParameters(
Expression query,
Dictionary<string, object?> parameters,
IDiagnosticsLogger<DbLoggerCategory.Query> logger,
bool compiledQuery = false,
bool generateContextAccessors = false)
bool compiledQuery = false)
=> new ExpressionTreeFuncletizer(_model, _evaluatableExpressionFilter, _contextType, generateContextAccessors: false, logger)
.ExtractParameters(query, parameters, parameterize: !compiledQuery, clearParameterizedValues: true);
}
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
#region SettingDefaultFullTextSearchLanguage

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task Set_unsupported_full_text_search_default_language()
{
var exception = (await Assert.ThrowsAsync<CosmosException>(() => InitializeNonSharedTest<ContextSettingDefaultFullTextSearchLanguage>()));
Expand Down Expand Up @@ -224,6 +225,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
#region DefaultFullTextSearchLanguageNoMismatchWhenNotSpecified

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task
Explicitly_setting_default_full_text_language_doesnt_clash_with_not_setting_it_on_other_entity_for_the_same_container()
{
Expand Down Expand Up @@ -298,6 +300,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
#region DefaultFullTextSearchLanguageUsedWhenPropertyDoesntSpecifyOneExplicitly

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task Default_full_text_language_is_used_for_full_text_properties_if_they_dont_specify_language_themselves()
{
var exception = (await Assert.ThrowsAsync<CosmosException>(()
Expand Down Expand Up @@ -337,6 +340,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
#region ExplicitFullTextLanguageOverridesTheDefault

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task Explicitly_setting_full_text_language_overrides_default()
{
var exception =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ public async Task Should_throw_if_specified_region_is_wrong()
}

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
[PlatformSkipCondition(
TestUtilities.Xunit.TestPlatform.Mac,
SkipReason = "Test is very environment-dependent; when running the Cosmos emulator in a VM on Mac, ConnectionMode.Direct causes severe issues")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

namespace Microsoft.EntityFrameworkCore;

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public class CosmosSessionTokensTest(CosmosSessionTokensTest.CosmosFixture fixture) : IClassFixture<CosmosSessionTokensTest.CosmosFixture>
{
private const string DatabaseName = nameof(CosmosSessionTokensTest);
Expand Down Expand Up @@ -601,6 +602,7 @@ protected Test2Context()
}
}

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public class CosmosNonSharedSessionTokenTests(NonSharedFixture fixture) : NonSharedModelTestBase(fixture), IClassFixture<NonSharedFixture>
{
protected override ITestStoreFactory NonSharedTestStoreFactory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

namespace Microsoft.EntityFrameworkCore;

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public class CosmosTransactionalBatchTest(CosmosTransactionalBatchTest.CosmosFixture fixture) : IClassFixture<CosmosTransactionalBatchTest.CosmosFixture>, IAsyncLifetime
{
private const string DatabaseName = nameof(CosmosTransactionalBatchTest);
Expand Down
1 change: 1 addition & 0 deletions test/EFCore.Cosmos.FunctionalTests/CosmosTriggersTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ protected override ITestStoreFactory NonSharedTestStoreFactory
=> CosmosTestStoreFactory.Instance;

[ConditionalFact]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task Triggers_are_executed_on_SaveChanges()
{
var contextFactory = await InitializeNonSharedTest<TriggersContext>(shouldLogCategory: _ => true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public virtual async Task Can_attach_owner_with_dependents()
}

[ConditionalTheory, InlineData(false), InlineData(true)]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public virtual async Task Can_manipulate_embedded_collections(bool useIds)
{
var options = await Fixture.CreateOptions(seed: false);
Expand Down
1 change: 1 addition & 0 deletions test/EFCore.Cosmos.FunctionalTests/EndToEndCosmosTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ namespace Microsoft.EntityFrameworkCore;
public class EndToEndCosmosTest(NonSharedFixture fixture) : NonSharedModelTestBase(fixture), IClassFixture<NonSharedFixture>
{
[ConditionalTheory, InlineData(false), InlineData(true)]
[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public async Task Can_add_update_delete_end_to_end(bool transactionalBatch)
{
var contextFactory = await InitializeNonSharedTest<DbContext>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ FROM root c
""");
}

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public override async Task Where()
{
await base.Where();
Expand Down Expand Up @@ -87,6 +88,7 @@ FROM root c
""");
}

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public override async Task OrderBy_ElementAt()
{
// 'ORDER BY' is not supported in subqueries.
Expand Down Expand Up @@ -158,6 +160,7 @@ FROM root c
""");
}

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public override async Task Index_column()
{
// The specified query includes 'member indexer' which is currently not supported
Expand Down Expand Up @@ -205,6 +208,7 @@ public override Task GroupBy()

#endregion GroupBy

[CosmosCondition(CosmosCondition.IsNotLinuxEmulator)]
public override async Task Select_within_Select_within_Select_with_aggregates()
{
await base.Select_within_Select_within_Select_with_aggregates();
Expand Down
Loading
Loading