From 71b8cae4ef6e99c65616c7bc3d9c903024799d92 Mon Sep 17 00:00:00 2001 From: jasonford Date: Wed, 11 Mar 2026 11:46:36 -0500 Subject: [PATCH 1/6] adding comment --- apps/worker/tasks/delete_owner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/worker/tasks/delete_owner.py b/apps/worker/tasks/delete_owner.py index 54b76c3ed..68e3df1df 100644 --- a/apps/worker/tasks/delete_owner.py +++ b/apps/worker/tasks/delete_owner.py @@ -16,7 +16,7 @@ class DeleteOwnerTask(BaseCodecovTask, name=delete_owner_task_name): def run_impl(self, _db_session, ownerid: int) -> CleanupSummary: # Purposefully not catching errors since this task runs on a cron return cleanup_owner(ownerid) - +#removed retry logic because it was causing lock contention RegisteredDeleteOwnerTask = celery_app.register_task(DeleteOwnerTask()) delete_owner_task = celery_app.tasks[DeleteOwnerTask.name] From 3d2ed6b0c3e650552348c3ae6ba411d55dead680 Mon Sep 17 00:00:00 2001 From: jasonford Date: Wed, 11 Mar 2026 12:45:33 -0500 Subject: [PATCH 2/6] updated comment --- apps/worker/tasks/delete_owner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/worker/tasks/delete_owner.py b/apps/worker/tasks/delete_owner.py index 68e3df1df..47f443eac 100644 --- a/apps/worker/tasks/delete_owner.py +++ b/apps/worker/tasks/delete_owner.py @@ -13,10 +13,11 @@ class DeleteOwnerTask(BaseCodecovTask, name=delete_owner_task_name): acks_late = True # retry the task when the worker dies for whatever reason max_retries = None # aka, no limit on retries +# removed retry logic because it was causing lock contention in run_impl + def run_impl(self, _db_session, ownerid: int) -> CleanupSummary: # Purposefully not catching errors since this task runs on a cron return cleanup_owner(ownerid) -#removed retry logic because it was causing lock contention RegisteredDeleteOwnerTask = celery_app.register_task(DeleteOwnerTask()) delete_owner_task = celery_app.tasks[DeleteOwnerTask.name] From b04fd1a65a19240f79ba3f9293f0fb0eb2f71eca Mon Sep 17 00:00:00 2001 From: jasonford Date: Wed, 11 Mar 2026 12:49:24 -0500 Subject: [PATCH 3/6] fixing comment --- apps/worker/tasks/delete_owner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/worker/tasks/delete_owner.py b/apps/worker/tasks/delete_owner.py index 47f443eac..f7aed74f5 100644 --- a/apps/worker/tasks/delete_owner.py +++ b/apps/worker/tasks/delete_owner.py @@ -13,7 +13,6 @@ class DeleteOwnerTask(BaseCodecovTask, name=delete_owner_task_name): acks_late = True # retry the task when the worker dies for whatever reason max_retries = None # aka, no limit on retries -# removed retry logic because it was causing lock contention in run_impl def run_impl(self, _db_session, ownerid: int) -> CleanupSummary: # Purposefully not catching errors since this task runs on a cron From 611508180d2e380cfd4b2e9b2d7f7c39deb627fa Mon Sep 17 00:00:00 2001 From: jasonford Date: Wed, 11 Mar 2026 13:01:01 -0500 Subject: [PATCH 4/6] fixing lint --- apps/worker/tasks/delete_owner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/worker/tasks/delete_owner.py b/apps/worker/tasks/delete_owner.py index f7aed74f5..54b76c3ed 100644 --- a/apps/worker/tasks/delete_owner.py +++ b/apps/worker/tasks/delete_owner.py @@ -13,10 +13,10 @@ class DeleteOwnerTask(BaseCodecovTask, name=delete_owner_task_name): acks_late = True # retry the task when the worker dies for whatever reason max_retries = None # aka, no limit on retries - def run_impl(self, _db_session, ownerid: int) -> CleanupSummary: # Purposefully not catching errors since this task runs on a cron return cleanup_owner(ownerid) + RegisteredDeleteOwnerTask = celery_app.register_task(DeleteOwnerTask()) delete_owner_task = celery_app.tasks[DeleteOwnerTask.name] From 4a3abdf901918da761b22ea96197f91e12b9178c Mon Sep 17 00:00:00 2001 From: jasonford Date: Thu, 12 Mar 2026 10:55:03 -0500 Subject: [PATCH 5/6] adding logging --- apps/worker/services/cleanup/cleanup.py | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/worker/services/cleanup/cleanup.py b/apps/worker/services/cleanup/cleanup.py index 504796bb5..925e63633 100644 --- a/apps/worker/services/cleanup/cleanup.py +++ b/apps/worker/services/cleanup/cleanup.py @@ -1,5 +1,7 @@ import logging +from celery.exceptions import SoftTimeLimitExceeded +from django.db import OperationalError from django.db.models.query import QuerySet from services.cleanup.models import MANUAL_CLEANUP @@ -31,5 +33,30 @@ def cleanup_queryset(query: QuerySet, context: CleanupContext): if manual_cleanup is not None: manual_cleanup(context, query) else: - cleaned_models = query._raw_delete(query.db) + log.info( + "cleanup_queryset: deleting %s with query %.500s", + model.__name__, + query.query, + ) + try: + cleaned_models = query._raw_delete(query.db) + except SoftTimeLimitExceeded: + log.warning( + "cleanup_queryset: soft time limit hit, rolling back delete of %s", + model.__name__, + exc_info=True, + ) + raise + except OperationalError: + log.warning( + "cleanup_queryset: db connection dropped, rolling back delete of %s", + model.__name__, + exc_info=True, + ) + raise + log.info( + "cleanup_queryset: deleted %d rows from %s", + cleaned_models, + model.__name__, + ) context.add_progress(cleaned_models) From 8a8170b66f8c3b61a16509da21a1c5190d364b13 Mon Sep 17 00:00:00 2001 From: jasonford Date: Thu, 12 Mar 2026 11:05:26 -0500 Subject: [PATCH 6/6] adjusting django results in logs --- apps/worker/services/cleanup/cleanup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/worker/services/cleanup/cleanup.py b/apps/worker/services/cleanup/cleanup.py index 925e63633..8491f9dde 100644 --- a/apps/worker/services/cleanup/cleanup.py +++ b/apps/worker/services/cleanup/cleanup.py @@ -1,6 +1,7 @@ import logging from celery.exceptions import SoftTimeLimitExceeded +from django.core.exceptions import EmptyResultSet from django.db import OperationalError from django.db.models.query import QuerySet @@ -33,10 +34,14 @@ def cleanup_queryset(query: QuerySet, context: CleanupContext): if manual_cleanup is not None: manual_cleanup(context, query) else: + try: + query_str = str(query.query)[:500] + except EmptyResultSet: + query_str = "(empty)" log.info( - "cleanup_queryset: deleting %s with query %.500s", + "cleanup_queryset: deleting %s with query %s", model.__name__, - query.query, + query_str, ) try: cleaned_models = query._raw_delete(query.db)