API v3: add build parity fields and cold-storage commands

readthedocs/api/v2/utils.py

@@ -1,6 +1,7 @@
 """Utility functions that are used by both views and celery tasks."""
 
 import itertools
+import json
 import re
 
 import structlog
@@ -18,6 +19,7 @@
 from readthedocs.builds.models import RegexAutomationRule
 from readthedocs.builds.models import Version
 from readthedocs.core.utils.db import delete_in_batches
+from readthedocs.storage import build_commands_storage
 
 
 log = structlog.get_logger(__name__)
@@ -285,6 +287,44 @@ def normalize_build_command(command, project_slug, version_slug):
     return command
 
 
+def get_build_commands_from_storage(build):
+    """
+    Return build commands from storage for ``cold_storage`` builds.
+
+    Returns ``None`` when commands can't be loaded from storage and callers
+    should keep using serializer output.
+    """
+    if not settings.RTD_SAVE_BUILD_COMMANDS_TO_STORAGE:
+        return None
+
+    if not build.cold_storage:
+        return None
+
+    storage_path = "{date}/{id}.json".format(
+        date=str(build.date.date()),
+        id=build.id,
+    )
+    if not build_commands_storage.exists(storage_path):
+        return None
+
+    try:
+        json_resp = build_commands_storage.open(storage_path).read()
+        commands = json.loads(json_resp)
+        for buildcommand in commands:
+            buildcommand["command"] = normalize_build_command(
+                buildcommand["command"],
+                build.project.slug,
+                build.get_version_slug(),
+            )
+        return commands
+    except Exception:
+        log.exception(
+            "Failed to read build data from storage.",
+            path=storage_path,
+        )
+        return None
+
+
 class RemoteOrganizationPagination(PageNumberPagination):
     page_size = 25
 

readthedocs/api/v2/views/model_views.py

@@ -1,6 +1,5 @@
 """Endpoints for listing Projects, Versions, Builds, etc."""
 
-import json
 from dataclasses import asdict
 
 import structlog
@@ -28,7 +27,7 @@
 from readthedocs.api.v2.permissions import HasBuildAPIKey
 from readthedocs.api.v2.permissions import IsOwner
 from readthedocs.api.v2.permissions import ReadOnlyPermission
-from readthedocs.api.v2.utils import normalize_build_command
+from readthedocs.api.v2.utils import get_build_commands_from_storage
 from readthedocs.aws.security_token_service import AWSTemporaryCredentialsError
 from readthedocs.aws.security_token_service import get_s3_build_media_scoped_credentials
 from readthedocs.aws.security_token_service import get_s3_build_tools_scoped_credentials
@@ -43,7 +42,6 @@
 from readthedocs.oauth.services import registry
 from readthedocs.projects.models import Domain
 from readthedocs.projects.models import Project
-from readthedocs.storage import build_commands_storage
 
 from ..serializers import BuildAdminReadOnlySerializer
 from ..serializers import BuildAdminSerializer
@@ -336,35 +334,13 @@ def retrieve(self, *args, **kwargs):
         This uses files from storage to get the JSON,
         and replaces the ``commands`` part of the response data.
         """
-        if not settings.RTD_SAVE_BUILD_COMMANDS_TO_STORAGE:
-            return super().retrieve(*args, **kwargs)
-
         instance = self.get_object()
         serializer = self.get_serializer(instance)
         data = serializer.data
-        if instance.cold_storage:
-            storage_path = "{date}/{id}.json".format(
-                date=str(instance.date.date()),
-                id=instance.id,
-            )
-            if build_commands_storage.exists(storage_path):
-                try:
-                    json_resp = build_commands_storage.open(storage_path).read()
-                    data["commands"] = json.loads(json_resp)
-
-                    # Normalize commands in the same way than when returning
-                    # them using the serializer
-                    for buildcommand in data["commands"]:
-                        buildcommand["command"] = normalize_build_command(
-                            buildcommand["command"],
-                            instance.project.slug,
-                            instance.get_version_slug(),
-                        )
-                except Exception:
-                    log.exception(
-                        "Failed to read build data from storage.",
-                        path=storage_path,
-                    )
+        commands = get_build_commands_from_storage(instance)
+        if commands is not None:
+            data["commands"] = commands
+
         return Response(data)
 
     @decorators.action(

readthedocs/api/v3/serializers.py

@@ -12,9 +12,11 @@
 from taggit.serializers import TaggitSerializer
 from taggit.serializers import TagListSerializerField
 
+from readthedocs.api.v2.utils import normalize_build_command
 from readthedocs.builds.constants import LATEST
 from readthedocs.builds.constants import STABLE
 from readthedocs.builds.models import Build
+from readthedocs.builds.models import BuildCommandResult
 from readthedocs.builds.models import Version
 from readthedocs.core.permissions import AdminPermission
 from readthedocs.core.resolver import Resolver
@@ -143,6 +145,32 @@ def get_version(self, obj):
         return None
 
 
+class BuildCommandSerializer(serializers.ModelSerializer):
+    run_time = serializers.ReadOnlyField()
+    command = serializers.SerializerMethodField()
+
+    class Meta:
+        model = BuildCommandResult
+        fields = [
+            "id",
+            "build",
+            "command",
+            "description",
+            "output",
+            "exit_code",
+            "start_time",
+            "end_time",
+            "run_time",
+        ]
+
+    def get_command(self, obj):
+        return normalize_build_command(
+            obj.command,
+            obj.build.project.slug,
+            obj.build.get_version_slug(),
+        )
+
+
 class BuildConfigSerializer(FlexFieldsSerializerMixin, serializers.Serializer):
     """
     Render ``Build.config`` property without modifying it.
@@ -177,6 +205,10 @@ class BuildSerializer(FlexFieldsModelSerializer):
     state = BuildStateSerializer(source="*")
     _links = BuildLinksSerializer(source="*")
     urls = BuildURLsSerializer(source="*")
+    builder = serializers.CharField(read_only=True)
+    docs_url = serializers.SerializerMethodField()
+    commit_url = serializers.SerializerMethodField()
+    commands = BuildCommandSerializer(many=True, read_only=True)
 
     class Meta:
         model = Build
@@ -191,6 +223,10 @@ class Meta:
             "success",
             "error",
             "commit",
+            "builder",
+            "docs_url",
+            "commit_url",
+            "commands",
             "_links",
             "urls",
         ]
@@ -212,6 +248,18 @@ def get_success(self, obj):
 
         return None
 
+    def get_docs_url(self, obj):
+        if obj.version:
+            return obj.version.get_absolute_url()
+
+        return None
+
+    def get_commit_url(self, obj):
+        if obj.commit:
+            return obj.get_commit_url()
+
+        return None
+
 
 class NotificationMessageSerializer(serializers.Serializer):
     id = serializers.SlugField()

readthedocs/api/v3/tests/mixins.py

@@ -12,8 +12,10 @@
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
 
-from readthedocs.builds.constants import LATEST, TAG
-from readthedocs.builds.models import Build, Version
+from readthedocs.builds.constants import LATEST
+from readthedocs.builds.constants import TAG
+from readthedocs.builds.models import Build
+from readthedocs.builds.models import Version
 from readthedocs.core.notifications import MESSAGE_EMAIL_VALIDATION_PENDING
 from readthedocs.doc_builder.exceptions import BuildCancelled
 from readthedocs.notifications.models import Notification
@@ -104,13 +106,14 @@ def setUp(self):
             state="finished",
             error="",
             success=True,
-            _config={"property": "test value"},
             version=self.version,
             project=self.project,
             builder="builder01",
             commit="a1b2c3",
             length=60,
         )
+        self.build.config = {"property": "test value"}
+        self.build.save()
 
         self.other = fixture.get(User, projects=[])
         self.others_token = fixture.get(Token, key="other", user=self.other)
@@ -134,13 +137,14 @@ def setUp(self):
             state="finished",
             error="",
             success=True,
-            _config={"property": "test value"},
             version=self.others_version,
             project=self.others_project,
             builder="builder01",
             commit="a1b2c3",
             length=60,
         )
+        self.others_build.config = {"property": "test value"}
+        self.others_build.save()
 
         # Make all non-html true so responses are complete
         self.project.versions.update(
@@ -245,7 +249,11 @@ def _get_response_dict(self, view_name, filepath=None):
         filename = Path(filepath).absolute().parent / "responses" / f"{view_name}.json"
         return json.load(open(filename))
 
-    def assertDictEqual(self, d1, d2):
+    # Keep unittest.TestCase signature compatibility.
+    # assertEqual() dispatches dict comparisons to assertDictEqual and passes
+    # `msg` as a keyword argument:
+    # https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertEqual
+    def assertDictEqual(self, d1, d2, msg=None):
         """
         Show the differences between the dicts in a human readable way.
 
@@ -258,4 +266,10 @@ def assertDictEqual(self, d1, d2):
             message = datadiff.diff(d1, d2)
         except ImportError:
             pass
+
+        if msg and message:
+            message = f"{message}\n{msg}"
+        elif msg:
+            message = msg
+
         return super().assertDictEqual(d1, d2, message)

readthedocs/api/v3/tests/responses/projects-builds-detail.json

@@ -5,6 +5,10 @@
     "error": "",
     "finished": "2019-04-29T10:01:00Z",
     "id": 1,
+    "builder": "builder01",
+    "docs_url": "http://project.readthedocs.io/en/v1.0/",
+    "commit_url": "https://github.com/rtfd/project/commit/a1b2c3",
+    "commands": [],
     "_links": {
         "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/",
         "project": "https://readthedocs.org/api/v3/projects/project/",

readthedocs/api/v3/tests/responses/projects-builds-list.json

@@ -10,6 +10,10 @@
       "error": "",
       "finished": "2019-04-29T10:01:00Z",
       "id": 1,
+      "builder": "builder01",
+      "docs_url": "http://project.readthedocs.io/en/v1.0/",
+      "commit_url": "https://github.com/rtfd/project/commit/a1b2c3",
+      "commands": [],
       "_links": {
         "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/",
         "project": "https://readthedocs.org/api/v3/projects/project/",

readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json

@@ -6,6 +6,10 @@
         "error": "",
         "finished": null,
         "id": 3,
+        "builder": null,
+        "docs_url": "http://project.readthedocs.io/en/v1.0/",
+        "commit_url": null,
+        "commands": [],
         "_links": {
             "_self": "https://readthedocs.org/api/v3/projects/project/builds/3/",
             "notifications": "https://readthedocs.org/api/v3/projects/project/builds/3/notifications/",