Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions test/SeederApi.IntegrationTest/SeederApiApplicationFactory.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using Bit.Core.Services;
using Bit.IntegrationTestCommon;
using Bit.IntegrationTestCommon.Factories;
using Microsoft.AspNetCore.TestHost;

namespace Bit.SeederApi.IntegrationTest;

Expand All @@ -15,4 +16,16 @@ public SeederApiApplicationFactory()
serviceCollection.AddHttpContextAccessor();
});
}

protected override void ConfigureWebHost(IWebHostBuilder builder)
{
base.ConfigureWebHost(builder);

builder.ConfigureTestServices(services =>
{
// Remove scheduled background jobs to prevent errors in parallel test execution
var jobService = services.First(sd => sd.ServiceType == typeof(IHostedService) && sd.ImplementationType == typeof(Jobs.JobsHostedService));
services.Remove(jobService);
});
}
}
13 changes: 9 additions & 4 deletions util/SeederApi/Controllers/SeedController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,18 @@ public async Task<IActionResult> DeleteAsync([FromRoute] string playId)
}
}


[HttpDelete]
public async Task<IActionResult> DeleteAllAsync()
public async Task<IActionResult> DeleteAllAsync([FromBody] DateTime? olderThanRequest)
{
logger.LogInformation("Deleting all seeded data");
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}

var olderThan = olderThanRequest?.ToUniversalTime() ?? DateTime.UtcNow.AddDays(-1);
logger.LogInformation("Deleting all seeded data older than {OlderThan} UTC", olderThan);

var playIds = getAllPlayIdsQuery.GetAllPlayIds();
var playIds = getAllPlayIdsQuery.GetAllPlayIds(olderThan: olderThan);

try
{
Expand Down
32 changes: 32 additions & 0 deletions util/SeederApi/Jobs/DeleteOldPlayDataJob.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using Bit.Core;
using Bit.Core.Jobs;
using Bit.SeederApi.Commands.Interfaces;
using Bit.SeederApi.Queries.Interfaces;
using Quartz;

namespace Bit.SeederApi.Jobs;

public class DeleteOldPlayDataJob : BaseJob
{
private readonly IGetAllPlayIdsQuery _getAllPlayIdsQuery;
private readonly IDestroyBatchScenesCommand _destroyBatchScenesCommand;

public DeleteOldPlayDataJob(
IGetAllPlayIdsQuery getAllPlayIdsQuery,
IDestroyBatchScenesCommand destroyBatchScenesCommand,
ILogger<DeleteOldPlayDataJob> logger)
: base(logger)
{
_getAllPlayIdsQuery = getAllPlayIdsQuery;
_destroyBatchScenesCommand = destroyBatchScenesCommand;
}

protected async override Task ExecuteJobAsync(IJobExecutionContext context)
{
_logger.LogInformation(Constants.BypassFiltersEventId, "Execute job task: DeleteOldPlayDataJob");
var olderThan = DateTime.UtcNow.AddDays(-1);
var playIds = _getAllPlayIdsQuery.GetAllPlayIds(olderThan);
await _destroyBatchScenesCommand.DestroyAsync(playIds);
_logger.LogInformation(Constants.BypassFiltersEventId, "Finished job task: DeleteOldPlayDataJob. Deleted {PlayIdCount} root items", playIds.Count);
}
}
38 changes: 38 additions & 0 deletions util/SeederApi/Jobs/JobsHostedService.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using Bit.Core.Jobs;
using Bit.Core.Settings;
using Quartz;

namespace Bit.SeederApi.Jobs;

public class JobsHostedService : BaseJobsHostedService
{
public JobsHostedService(
GlobalSettings globalSettings,
IServiceProvider serviceProvider,
ILogger<JobsHostedService> logger,
ILogger<JobListener> listenerLogger)
: base(globalSettings, serviceProvider, logger, listenerLogger) { }

public override async Task StartAsync(CancellationToken cancellationToken)
{
var everyFifteenMinutesTrigger = TriggerBuilder.Create()
.WithIdentity("everyFifteenMinutesTrigger")
.StartNow()
.WithCronSchedule("0 */15 * ? * *")
.Build();


var jobs = new List<Tuple<Type, ITrigger>>
{
new Tuple<Type, ITrigger>(typeof(DeleteOldPlayDataJob), everyFifteenMinutesTrigger),
};

Jobs = jobs;
await base.StartAsync(cancellationToken);
}

public static void AddJobsServices(IServiceCollection services)
{
services.AddTransient<DeleteOldPlayDataJob>();
}
}
9 changes: 9 additions & 0 deletions util/SeederApi/Queries/GetAllPlayIdsQuery.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,13 @@ public List<string> GetAllPlayIds()
.Distinct()
.ToList();
}

public List<string> GetAllPlayIds(DateTime olderThan)
{
return databaseContext.PlayItem
.Where(pd => pd.CreationDate < olderThan)
.Select(pd => pd.PlayId)
.Distinct()
.ToList();
}
}
6 changes: 6 additions & 0 deletions util/SeederApi/Queries/Interfaces/IGetAllPlayIdsQuery.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,10 @@ public interface IGetAllPlayIdsQuery
/// </summary>
/// <returns>A list of play IDs representing active seeded data that can be destroyed.</returns>
List<string> GetAllPlayIds();
/// <summary>
/// Retrieves all play IDs for currently tracked seeded data that were created prior to the given DateTime
/// </summary>
/// <param name="olderThan">The cutoff point for PlayId creation date</param>
/// <returns></returns>
List<string> GetAllPlayIds(DateTime olderThan);
}
1 change: 1 addition & 0 deletions util/SeederApi/SeederApi.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\Core\Core.csproj" />
<ProjectReference Include="..\..\src\SharedWeb\SharedWeb.csproj" />
<ProjectReference Include="..\Seeder\Seeder.csproj" />
</ItemGroup>
Expand Down
3 changes: 3 additions & 0 deletions util/SeederApi/Startup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ public void ConfigureServices(IServiceCollection services)
services.AddQueries();

services.AddControllers();

Jobs.JobsHostedService.AddJobsServices(services);
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 Deleting data with a hosted service always give me a little heartburn. Have we implemented opt-in techniques using environment settings in other places, by chance? I have done that in the past where my teams implemented background services that mutated data and gave us a better feel of control (especially when developing) knowing that one had to explicitly set the setting to ON to mutate/delete data?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. Are you concerned with unintentional data loss? To some extent the goal of this job is to enforce that play data is ephemeral.

That can be a server setting if we want, I suppose, but maybe a better tweak would be a lifetime?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the ephemeral environments, I am not too concerned because I would not expect an ephemeral environment to live long enough to see the job run. Since the intention is really deleting this ephemeral data from long lived environments then I'm cool with it.

Only real issue is if you're developing tests locally or on one of the QA team VMs and you're expecting data to be there over a couple days (like a weekend) then you'd better be ready for the Seeder to clean itself up.

Did we add a README.md? (Sorry on the GH app so hard to go back n forth).

services.AddHostedService<Jobs.JobsHostedService>();
}

public void Configure(
Expand Down
Loading