Builds: delete old build objects

readthedocs/builds/models.py

@@ -2,9 +2,11 @@
 
 import datetime
 import fnmatch
+import json
 import os.path
 import re
 from functools import partial
+from io import BytesIO
 
 import regex
 import structlog
@@ -18,6 +20,7 @@
 from polymorphic.models import PolymorphicModel
 
 import readthedocs.builds.automation_actions as actions
+from readthedocs.api.v2.serializers import BuildCommandSerializer
 from readthedocs.builds.constants import BRANCH
 from readthedocs.builds.constants import BUILD_FINAL_STATES
 from readthedocs.builds.constants import BUILD_STATE
@@ -29,6 +32,7 @@
 from readthedocs.builds.constants import EXTERNAL_VERSION_STATES
 from readthedocs.builds.constants import INTERNAL
 from readthedocs.builds.constants import LATEST
+from readthedocs.builds.constants import MAX_BUILD_COMMAND_SIZE
 from readthedocs.builds.constants import PREDEFINED_MATCH_ARGS
 from readthedocs.builds.constants import PREDEFINED_MATCH_ARGS_VALUES
 from readthedocs.builds.constants import STABLE
@@ -73,6 +77,7 @@
 from readthedocs.projects.ordering import ProjectItemPositionManager
 from readthedocs.projects.validators import validate_build_config_file
 from readthedocs.projects.version_handling import determine_stable_version
+from readthedocs.storage import build_commands_storage
 
 
 log = structlog.get_logger(__name__)
@@ -869,6 +874,55 @@ def save(self, *args, **kwargs):  # noqa
         self._readthedocs_yaml_config = None
         self._readthedocs_yaml_config_changed = False
 
+    def delete(self, *args, **kwargs):
+        # Delete from storage if the build steps are stored outside the database.
+        if self.cold_storage:
+            try:
+                build_commands_storage.delete(self._storage_path)
+            except IOError:
+                log.exception("Cold Storage delete failure")
+        return super().delete(*args, **kwargs)
+
+    def move_to_storage(self):
+        if self.cold_storage:
+            return
+
+        commands = BuildCommandSerializer(self.commands, many=True).data
+        if commands:
+            for cmd in commands:
+                if len(cmd["output"]) > MAX_BUILD_COMMAND_SIZE:
+                    cmd["output"] = cmd["output"][-MAX_BUILD_COMMAND_SIZE:]
+                    cmd["output"] = (
+                        "\n\n"
+                        "... (truncated) ..."
+                        "\n\n"
+                        "Command output too long. Truncated to last 1MB."
+                        "\n\n" + cmd["output"]
+                    )
+                    log.debug("Truncating build command for build.", build_id=self.id)
+            output = BytesIO(json.dumps(commands).encode("utf8"))
+            try:
+                build_commands_storage.save(name=self._storage_path, content=output)
+                self.commands.all().delete()
+            except IOError:
+                log.exception("Cold Storage save failure")
+                return
+
+        self.cold_storage = True
+        self.save()
+
+    @property
+    def _storage_path(self):
+        """
+        Storage path where the build commands will be stored.
+
+        The path is in the format: <date>/<build_id>.json
+
+        Example: 2024-01-01/1111.json
+        """
+        date = self.date.date()
+        return f"{date}/{self.id}.json"
+
     def get_absolute_url(self):
         return reverse("builds_detail", args=[self.project.slug, self.pk])
 

readthedocs/builds/tasks.py

@@ -1,29 +1,28 @@
 import json
-from io import BytesIO
 
 import requests
 import structlog
 from django.conf import settings
+from django.db.models import Count
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from oauthlib.oauth2.rfc6749.errors import InvalidGrantError
 from oauthlib.oauth2.rfc6749.errors import TokenExpiredError
 
 from readthedocs import __version__
-from readthedocs.api.v2.serializers import BuildCommandSerializer
 from readthedocs.api.v2.utils import delete_versions_from_db
 from readthedocs.api.v2.utils import get_deleted_active_versions
 from readthedocs.api.v2.utils import run_version_automation_rules
 from readthedocs.api.v2.utils import sync_versions_to_db
 from readthedocs.builds.constants import BRANCH
+from readthedocs.builds.constants import BUILD_FINAL_STATES
 from readthedocs.builds.constants import BUILD_STATUS_FAILURE
 from readthedocs.builds.constants import BUILD_STATUS_PENDING
 from readthedocs.builds.constants import BUILD_STATUS_SUCCESS
 from readthedocs.builds.constants import EXTERNAL
 from readthedocs.builds.constants import EXTERNAL_VERSION_STATE_CLOSED
 from readthedocs.builds.constants import LOCK_EXPIRE
-from readthedocs.builds.constants import MAX_BUILD_COMMAND_SIZE
 from readthedocs.builds.constants import TAG
 from readthedocs.builds.models import Build
 from readthedocs.builds.models import BuildConfig
@@ -32,20 +31,20 @@
 from readthedocs.builds.utils import memcache_lock
 from readthedocs.core.utils import send_email
 from readthedocs.core.utils import trigger_build
+from readthedocs.core.utils.db import delete_in_batches
 from readthedocs.integrations.models import HttpExchange
 from readthedocs.notifications.models import Notification
 from readthedocs.oauth.notifications import MESSAGE_OAUTH_BUILD_STATUS_FAILURE
 from readthedocs.projects.models import Project
 from readthedocs.projects.models import WebHookEvent
-from readthedocs.storage import build_commands_storage
 from readthedocs.worker import app
 
 
 log = structlog.get_logger(__name__)
 
 
 @app.task(queue="web", bind=True)
-def archive_builds_task(self, days=14, limit=200, delete=False):
+def archive_builds_task(self, days=14, limit=200):
     """
     Task to archive old builds to cold storage.
 
@@ -71,33 +70,8 @@ def archive_builds_task(self, days=14, limit=200, delete=False):
             .prefetch_related("commands")
             .only("date", "cold_storage")[:limit]
         )
-
-        for build in queryset:
-            commands = BuildCommandSerializer(build.commands, many=True).data
-            if commands:
-                for cmd in commands:
-                    if len(cmd["output"]) > MAX_BUILD_COMMAND_SIZE:
-                        cmd["output"] = cmd["output"][-MAX_BUILD_COMMAND_SIZE:]
-                        cmd["output"] = (
-                            "\n\n"
-                            "... (truncated) ..."
-                            "\n\n"
-                            "Command output too long. Truncated to last 1MB."
-                            "\n\n" + cmd["output"]
-                        )  # noqa
-                        log.debug("Truncating build command for build.", build_id=build.id)
-                output = BytesIO(json.dumps(commands).encode("utf8"))
-                filename = "{date}/{id}.json".format(date=str(build.date.date()), id=build.id)
-                try:
-                    build_commands_storage.save(name=filename, content=output)
-                    if delete:
-                        build.commands.all().delete()
-                except IOError:
-                    log.exception("Cold Storage save failure")
-                    continue
-
-            build.cold_storage = True
-            build.save()
+        for build in queryset.iterator():
+            build.move_to_storage()
 
 
 @app.task(queue="web")
@@ -606,3 +580,42 @@ def remove_orphan_build_config():
     count = orphan_buildconfigs.count()
     orphan_buildconfigs.delete()
     log.info("Removed orphan BuildConfig objects.", count=count)
+
+
+@app.task(queue="web")
+def delete_old_build_objects(days=360 * 3, keep_recent=5_000, limit=10_000):
+    """
+    Delete old Build objects that are not needed anymore.
+
+    We keep the most recent `keep_recent` builds per project,
+    and delete builds that are older than `days` days.
+
+    :param limit: Maximum number of builds to delete in one execution.
+     Keep our DB from being overwhelmed by deleting too many builds at once.
+    """
+    cutoff_date = timezone.now() - timezone.timedelta(days=days)
+    projects = (
+        Project.objects.all()
+        .annotate(build_count=Count("builds"))
+        .filter(build_count__gt=keep_recent)
+    )
+    for project in projects.iterator():
+        builds_to_delete = project.builds.filter(
+            state__in=BUILD_FINAL_STATES,
+            date__lt=cutoff_date,
+        ).order_by("-date")[keep_recent:]
+
+        # Builds that are not in cold storage can be deleted in bulk,
+        # as they don't need to remove any files from storage.
+        _, deleted = delete_in_batches(builds_to_delete.filter(cold_storage=False), limit=limit)
+        limit -= deleted["builds.Build"]
+        if limit <= 0:
+            break
+
+        # Builds in cold storage need to be deleted one by one,
+        # so their custom delete method is called to remove the files from storage.
+        for build in builds_to_delete.filter(cold_storage=True)[:limit]:
+            build.delete()
+            limit -= 1
+            if limit <= 0:
+                break

readthedocs/projects/models.py

@@ -727,6 +727,10 @@ def delete(self, *args, **kwargs):
         qs = self.search_queries.all()
         qs._raw_delete(qs.db)
 
+        # Remove builds on cold storage one by one, so they are properly deleted from storage.
+        for build in self.builds.filter(cold_storage=True):
+            build.delete()
+
         # Remove extra resources
         clean_project_resources(self)
 

readthedocs/settings/base.py

@@ -724,7 +724,6 @@ def BUILD_MEMORY_LIMIT(self):
             "kwargs": {
                 "days": 1,
                 "limit": 500,
-                "delete": True,
             },
         },
         "every-30m-delete-inactive-external-versions": {
@@ -763,6 +762,19 @@ def BUILD_MEMORY_LIMIT(self):
             "options": {"queue": "web"},
             "kwargs": {"limit": 10_000},
         },
+        "every-day-delete-old-build-objects": {
+            "task": "readthedocs.builds.tasks.delete_old_build_objects",
+            # NOTE: we are running this task every hour for now,
+            # since we have lots of objects to delete, and we are limiting
+            # the number of deletions per task run.
+            # TODO: go back to once a day (unlimited) after we delete the backlog of objects,
+            # or keep less build objects (keep_recent=1000, days=360).
+            # It should take around 6 days to delete all the old objects on community,
+            # and 1 day on commercial.
+            "schedule": crontab(minute=0, hour="*"),
+            "options": {"queue": "web"},
+            "kwargs": {"days": 360 * 3, "keep_recent": 5_000, "limit": 10_000},
+        },
     }
 
     # Sentry