Add by pass to not include packages that were skipped between 7.74.0 to 7.78.0

.ddev/config.toml

@@ -221,6 +221,17 @@ exclude = [
 # Dependencies for the downloader that are security-related and should be updated separately from the others
 security_deps = ['in-toto', 'tuf', 'securesystemslib']
 
+# Historical corrections for Agent release documentation generated from
+# requirements-agent-release.txt. Ranges are inclusive.
+[overrides.release.agent.integrations.exclude_by_agent_version]
+"7.74.0..7.78.0" = [
+  "datadog-control-m",
+  "datadog-krakend",
+  "datadog-lustre",
+  "datadog-n8n",
+  "datadog-prefect",
+]
+
 
 # Explicitly mark folders as non-integrations. By default, all folders are assumed to be
 # integrations. Previously, only folders with manifest.json were considered integrations,

AGENT_INTEGRATIONS.md

@@ -312,7 +312,6 @@
 * datadog-cockroachdb: 6.4.0
 * datadog-confluent-platform: 3.3.0
 * datadog-consul: 5.4.0
-* datadog-control-m: 1.1.0
 * datadog-coredns: 6.4.0
 * datadog-couch: 9.4.0
 * datadog-couchbase: 6.5.0
@@ -386,7 +385,6 @@
 * datadog-keda: 2.4.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.4.0
-* datadog-krakend: 1.4.0
 * datadog-kube-apiserver-metrics: 7.5.0
 * datadog-kube-controller-manager: 8.4.0
 * datadog-kube-dns: 7.4.0
@@ -408,7 +406,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.3.0
 * datadog-litellm: 2.4.0
-* datadog-lustre: 1.5.0
 * datadog-mac-audit-logs: 1.4.0
 * datadog-mapr: 3.4.0
 * datadog-mapreduce: 7.4.0
@@ -422,7 +419,6 @@
 * datadog-milvus: 2.5.0
 * datadog-mongo: 10.9.0
 * datadog-mysql: 15.15.1
-* datadog-n8n: 1.1.0
 * datadog-nagios: 3.4.0
 * datadog-network: 5.7.0
 * datadog-nfsstat: 3.5.0
@@ -447,7 +443,6 @@
 * datadog-postfix: 3.4.0
 * datadog-postgres: 23.6.0
 * datadog-powerdns-recursor: 5.4.0
-* datadog-prefect: 1.0.0
 * datadog-presto: 3.4.0
 * datadog-process: 5.5.0
 * datadog-prometheus: 6.3.0
@@ -645,7 +640,6 @@
 * datadog-keda: 2.3.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.3.0
-* datadog-krakend: 1.3.0
 * datadog-kube-apiserver-metrics: 7.4.0
 * datadog-kube-controller-manager: 8.3.0
 * datadog-kube-dns: 7.3.0
@@ -667,7 +661,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.3.0
 * datadog-litellm: 2.3.0
-* datadog-lustre: 1.4.0
 * datadog-mac-audit-logs: 1.3.0
 * datadog-mapr: 3.3.0
 * datadog-mapreduce: 7.3.0
@@ -902,7 +895,6 @@
 * datadog-keda: 2.3.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.3.0
-* datadog-krakend: 1.3.0
 * datadog-kube-apiserver-metrics: 7.4.0
 * datadog-kube-controller-manager: 8.3.0
 * datadog-kube-dns: 7.3.0
@@ -924,7 +916,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.3.0
 * datadog-litellm: 2.3.0
-* datadog-lustre: 1.4.0
 * datadog-mac-audit-logs: 1.3.0
 * datadog-mapr: 3.3.0
 * datadog-mapreduce: 7.3.0
@@ -1159,7 +1150,6 @@
 * datadog-keda: 2.3.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.3.0
-* datadog-krakend: 1.3.0
 * datadog-kube-apiserver-metrics: 7.4.0
 * datadog-kube-controller-manager: 8.3.0
 * datadog-kube-dns: 7.3.0
@@ -1181,7 +1171,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.3.0
 * datadog-litellm: 2.3.0
-* datadog-lustre: 1.4.0
 * datadog-mac-audit-logs: 1.3.0
 * datadog-mapr: 3.3.0
 * datadog-mapreduce: 7.3.0
@@ -1416,7 +1405,6 @@
 * datadog-keda: 2.3.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.3.0
-* datadog-krakend: 1.3.0
 * datadog-kube-apiserver-metrics: 7.4.0
 * datadog-kube-controller-manager: 8.3.0
 * datadog-kube-dns: 7.3.0
@@ -1438,7 +1426,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.3.0
 * datadog-litellm: 2.3.0
-* datadog-lustre: 1.4.0
 * datadog-mac-audit-logs: 1.3.0
 * datadog-mapr: 3.3.0
 * datadog-mapreduce: 7.3.0
@@ -1670,7 +1657,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -1692,7 +1678,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -1924,7 +1909,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -1946,7 +1930,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -2178,7 +2161,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -2200,7 +2182,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -2432,7 +2413,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -2454,7 +2434,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -2686,7 +2665,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -2708,7 +2686,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -2939,7 +2916,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -2961,7 +2937,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -3192,7 +3167,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -3214,7 +3188,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -3445,7 +3418,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -3467,7 +3439,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -3698,7 +3669,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -3720,7 +3690,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0
@@ -3950,7 +3919,6 @@
 * datadog-keda: 2.2.0
 * datadog-keycloak: 1.2.0
 * datadog-kong: 6.2.0
-* datadog-krakend: 1.2.0
 * datadog-kube-apiserver-metrics: 7.3.0
 * datadog-kube-controller-manager: 8.2.0
 * datadog-kube-dns: 7.2.0
@@ -3972,7 +3940,6 @@
 * datadog-linux-audit-logs: 1.2.0
 * datadog-linux-proc-extras: 4.2.0
 * datadog-litellm: 2.2.0
-* datadog-lustre: 1.3.0
 * datadog-mac-audit-logs: 1.2.0
 * datadog-mapr: 3.2.0
 * datadog-mapreduce: 7.2.0

ddev/src/ddev/cli/release/agent/common.py

@@ -11,6 +11,7 @@
     AgentChangelog = dict[str, dict[str, tuple[str, bool, bool]]]
 
 DATADOG_PACKAGE_PREFIX = 'datadog-'
+AGENT_INTEGRATION_EXCLUSIONS_CONFIG = '/overrides/release/agent/integrations/exclude_by_agent_version'
 
 
 def get_agent_tags(repo: Repository, since: str, to: str) -> list[str]:
@@ -64,14 +65,9 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel
     changes_per_agent: AgentChangelog = {}
     # to keep indexing easy, we run the loop off-by-one
     for i in range(1, len(agent_tags)):
-        req_file_name = repo.agent_release_requirements.name
         current_tag = agent_tags[i - 1]
-        # Requirements for current tag
-        file_contents = repo.git.show_file(req_file_name, current_tag)
-        catalog_now = parse_agent_req_file(file_contents)
-        # Requirements for previous tag
-        file_contents = repo.git.show_file(req_file_name, agent_tags[i])
-        catalog_prev = parse_agent_req_file(file_contents)
+        catalog_now = get_agent_release_catalog(repo, current_tag)
+        catalog_prev = get_agent_release_catalog(repo, agent_tags[i])
 
         # at some point in the git history, the requirements file erroneously
         # contained the folder name instead of the package name for each check,
@@ -94,6 +90,43 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel
     return changes_per_agent
 
 
+def get_agent_release_catalog(repo: Repository, agent_version: str) -> dict[str, str]:
+    """Return the requirements catalog for an Agent tag with configured historical corrections applied."""
+    file_contents = repo.git.show_file(repo.agent_release_requirements.name, agent_version)
+    catalog = parse_agent_req_file(file_contents)
+    excluded_integrations = _get_excluded_agent_integrations(repo, agent_version)
+
+    if not excluded_integrations:
+        return catalog
+
+    return {
+        name: version for name, version in catalog.items() if normalize_package_name(name) not in excluded_integrations
+    }
+
+
+def _get_excluded_agent_integrations(repo: Repository, agent_version: str) -> set[str]:
+    from packaging.version import Version
+
+    exclusions_by_range = repo.config.get(AGENT_INTEGRATION_EXCLUSIONS_CONFIG, {})
+    if not exclusions_by_range:
+        return set()
+
+    version = Version(agent_version)
+    excluded_integrations = set()
+    for version_range, integrations in exclusions_by_range.items():
+        start, separator, end = version_range.partition('..')
+        if not separator:
+            raise ValueError(
+                f'Invalid Agent integration exclusion range {version_range!r}. '
+                'Expected format: "<START_VERSION>..<END_VERSION>".'
+            )
+
+        if Version(start) <= version <= Version(end):
+            excluded_integrations.update(normalize_package_name(integration) for integration in integrations)
+
+    return excluded_integrations
+
+
 def normalize_catalog(catalog: dict[str, str]) -> dict[str, str]:
     return {normalize_package_name(k): v for k, v in catalog.items()}
 

ddev/src/ddev/cli/release/agent/integrations.py

@@ -29,18 +29,14 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool
     tool will generate the list for every Agent since version 6.3.0
     (before that point we don't have enough information to build the log).
     """
-    from ddev.cli.release.agent.common import get_agent_tags, parse_agent_req_file
+    from ddev.cli.release.agent.common import get_agent_release_catalog, get_agent_tags
 
     agent_tags = get_agent_tags(app.repo, since, to)
-    # get the list of integrations shipped with the agent from the requirements file
-    req_file_name = app.repo.agent_release_requirements.name
 
     integrations_contents = StringIO()
     for tag in agent_tags:
         integrations_contents.write(f'## Datadog Agent version {tag}\n\n')
-        # Requirements for current tag
-        file_contents = app.repo.git.show_file(req_file_name, tag)
-        for name, ver in parse_agent_req_file(file_contents).items():
+        for name, ver in get_agent_release_catalog(app.repo, tag).items():
             integrations_contents.write(f'* {name}: {ver}\n')
         integrations_contents.write('\n')