From 73a1fbbef314952e4316be944b1b7711eadeb2e2 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Wed, 29 Apr 2026 15:47:06 -0400 Subject: [PATCH] add by pass to not include packages that were skipped between 7.74.0 and 7.78.0 --- .ddev/config.toml | 11 +++++ AGENT_INTEGRATIONS.md | 33 ------------- ddev/src/ddev/cli/release/agent/common.py | 47 ++++++++++++++++--- .../ddev/cli/release/agent/integrations.py | 8 +--- ddev/tests/cli/release/agent/conftest.py | 46 ++++++++++++++++++ .../tests/cli/release/agent/test_changelog.py | 25 ++++++++++ .../cli/release/agent/test_integrations.py | 25 ++++++++++ 7 files changed, 149 insertions(+), 46 deletions(-) diff --git a/.ddev/config.toml b/.ddev/config.toml index a767c274967e6..19afee3c62fe5 100644 --- a/.ddev/config.toml +++ b/.ddev/config.toml @@ -221,6 +221,17 @@ exclude = [ # Dependencies for the downloader that are security-related and should be updated separately from the others security_deps = ['in-toto', 'tuf', 'securesystemslib'] +# Historical corrections for Agent release documentation generated from +# requirements-agent-release.txt. Ranges are inclusive. +[overrides.release.agent.integrations.exclude_by_agent_version] +"7.74.0..7.78.0" = [ + "datadog-control-m", + "datadog-krakend", + "datadog-lustre", + "datadog-n8n", + "datadog-prefect", +] + # Explicitly mark folders as non-integrations. By default, all folders are assumed to be # integrations. Previously, only folders with manifest.json were considered integrations, diff --git a/AGENT_INTEGRATIONS.md b/AGENT_INTEGRATIONS.md index 3b323fc129ee3..943dcfa47bc7e 100644 --- a/AGENT_INTEGRATIONS.md +++ b/AGENT_INTEGRATIONS.md @@ -312,7 +312,6 @@ * datadog-cockroachdb: 6.4.0 * datadog-confluent-platform: 3.3.0 * datadog-consul: 5.4.0 -* datadog-control-m: 1.1.0 * datadog-coredns: 6.4.0 * datadog-couch: 9.4.0 * datadog-couchbase: 6.5.0 @@ -386,7 +385,6 @@ * datadog-keda: 2.4.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.4.0 -* datadog-krakend: 1.4.0 * datadog-kube-apiserver-metrics: 7.5.0 * datadog-kube-controller-manager: 8.4.0 * datadog-kube-dns: 7.4.0 @@ -408,7 +406,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.3.0 * datadog-litellm: 2.4.0 -* datadog-lustre: 1.5.0 * datadog-mac-audit-logs: 1.4.0 * datadog-mapr: 3.4.0 * datadog-mapreduce: 7.4.0 @@ -422,7 +419,6 @@ * datadog-milvus: 2.5.0 * datadog-mongo: 10.9.0 * datadog-mysql: 15.15.1 -* datadog-n8n: 1.1.0 * datadog-nagios: 3.4.0 * datadog-network: 5.7.0 * datadog-nfsstat: 3.5.0 @@ -447,7 +443,6 @@ * datadog-postfix: 3.4.0 * datadog-postgres: 23.6.0 * datadog-powerdns-recursor: 5.4.0 -* datadog-prefect: 1.0.0 * datadog-presto: 3.4.0 * datadog-process: 5.5.0 * datadog-prometheus: 6.3.0 @@ -645,7 +640,6 @@ * datadog-keda: 2.3.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.3.0 -* datadog-krakend: 1.3.0 * datadog-kube-apiserver-metrics: 7.4.0 * datadog-kube-controller-manager: 8.3.0 * datadog-kube-dns: 7.3.0 @@ -667,7 +661,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.3.0 * datadog-litellm: 2.3.0 -* datadog-lustre: 1.4.0 * datadog-mac-audit-logs: 1.3.0 * datadog-mapr: 3.3.0 * datadog-mapreduce: 7.3.0 @@ -902,7 +895,6 @@ * datadog-keda: 2.3.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.3.0 -* datadog-krakend: 1.3.0 * datadog-kube-apiserver-metrics: 7.4.0 * datadog-kube-controller-manager: 8.3.0 * datadog-kube-dns: 7.3.0 @@ -924,7 +916,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.3.0 * datadog-litellm: 2.3.0 -* datadog-lustre: 1.4.0 * datadog-mac-audit-logs: 1.3.0 * datadog-mapr: 3.3.0 * datadog-mapreduce: 7.3.0 @@ -1159,7 +1150,6 @@ * datadog-keda: 2.3.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.3.0 -* datadog-krakend: 1.3.0 * datadog-kube-apiserver-metrics: 7.4.0 * datadog-kube-controller-manager: 8.3.0 * datadog-kube-dns: 7.3.0 @@ -1181,7 +1171,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.3.0 * datadog-litellm: 2.3.0 -* datadog-lustre: 1.4.0 * datadog-mac-audit-logs: 1.3.0 * datadog-mapr: 3.3.0 * datadog-mapreduce: 7.3.0 @@ -1416,7 +1405,6 @@ * datadog-keda: 2.3.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.3.0 -* datadog-krakend: 1.3.0 * datadog-kube-apiserver-metrics: 7.4.0 * datadog-kube-controller-manager: 8.3.0 * datadog-kube-dns: 7.3.0 @@ -1438,7 +1426,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.3.0 * datadog-litellm: 2.3.0 -* datadog-lustre: 1.4.0 * datadog-mac-audit-logs: 1.3.0 * datadog-mapr: 3.3.0 * datadog-mapreduce: 7.3.0 @@ -1670,7 +1657,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -1692,7 +1678,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -1924,7 +1909,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -1946,7 +1930,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -2178,7 +2161,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -2200,7 +2182,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -2432,7 +2413,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -2454,7 +2434,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -2686,7 +2665,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -2708,7 +2686,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -2939,7 +2916,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -2961,7 +2937,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -3192,7 +3167,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -3214,7 +3188,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -3445,7 +3418,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -3467,7 +3439,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -3698,7 +3669,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -3720,7 +3690,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 @@ -3950,7 +3919,6 @@ * datadog-keda: 2.2.0 * datadog-keycloak: 1.2.0 * datadog-kong: 6.2.0 -* datadog-krakend: 1.2.0 * datadog-kube-apiserver-metrics: 7.3.0 * datadog-kube-controller-manager: 8.2.0 * datadog-kube-dns: 7.2.0 @@ -3972,7 +3940,6 @@ * datadog-linux-audit-logs: 1.2.0 * datadog-linux-proc-extras: 4.2.0 * datadog-litellm: 2.2.0 -* datadog-lustre: 1.3.0 * datadog-mac-audit-logs: 1.2.0 * datadog-mapr: 3.2.0 * datadog-mapreduce: 7.2.0 diff --git a/ddev/src/ddev/cli/release/agent/common.py b/ddev/src/ddev/cli/release/agent/common.py index a1c8673d812fa..75c9199736630 100644 --- a/ddev/src/ddev/cli/release/agent/common.py +++ b/ddev/src/ddev/cli/release/agent/common.py @@ -11,6 +11,7 @@ AgentChangelog = dict[str, dict[str, tuple[str, bool, bool]]] DATADOG_PACKAGE_PREFIX = 'datadog-' +AGENT_INTEGRATION_EXCLUSIONS_CONFIG = '/overrides/release/agent/integrations/exclude_by_agent_version' def get_agent_tags(repo: Repository, since: str, to: str) -> list[str]: @@ -64,14 +65,9 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel changes_per_agent: AgentChangelog = {} # to keep indexing easy, we run the loop off-by-one for i in range(1, len(agent_tags)): - req_file_name = repo.agent_release_requirements.name current_tag = agent_tags[i - 1] - # Requirements for current tag - file_contents = repo.git.show_file(req_file_name, current_tag) - catalog_now = parse_agent_req_file(file_contents) - # Requirements for previous tag - file_contents = repo.git.show_file(req_file_name, agent_tags[i]) - catalog_prev = parse_agent_req_file(file_contents) + catalog_now = get_agent_release_catalog(repo, current_tag) + catalog_prev = get_agent_release_catalog(repo, agent_tags[i]) # at some point in the git history, the requirements file erroneously # contained the folder name instead of the package name for each check, @@ -94,6 +90,43 @@ def get_changes_per_agent(repo: Repository, since: str, to: str) -> AgentChangel return changes_per_agent +def get_agent_release_catalog(repo: Repository, agent_version: str) -> dict[str, str]: + """Return the requirements catalog for an Agent tag with configured historical corrections applied.""" + file_contents = repo.git.show_file(repo.agent_release_requirements.name, agent_version) + catalog = parse_agent_req_file(file_contents) + excluded_integrations = _get_excluded_agent_integrations(repo, agent_version) + + if not excluded_integrations: + return catalog + + return { + name: version for name, version in catalog.items() if normalize_package_name(name) not in excluded_integrations + } + + +def _get_excluded_agent_integrations(repo: Repository, agent_version: str) -> set[str]: + from packaging.version import Version + + exclusions_by_range = repo.config.get(AGENT_INTEGRATION_EXCLUSIONS_CONFIG, {}) + if not exclusions_by_range: + return set() + + version = Version(agent_version) + excluded_integrations = set() + for version_range, integrations in exclusions_by_range.items(): + start, separator, end = version_range.partition('..') + if not separator: + raise ValueError( + f'Invalid Agent integration exclusion range {version_range!r}. ' + 'Expected format: "..".' + ) + + if Version(start) <= version <= Version(end): + excluded_integrations.update(normalize_package_name(integration) for integration in integrations) + + return excluded_integrations + + def normalize_catalog(catalog: dict[str, str]) -> dict[str, str]: return {normalize_package_name(k): v for k, v in catalog.items()} diff --git a/ddev/src/ddev/cli/release/agent/integrations.py b/ddev/src/ddev/cli/release/agent/integrations.py index 9c8281baebe4c..782f22e41d9ec 100644 --- a/ddev/src/ddev/cli/release/agent/integrations.py +++ b/ddev/src/ddev/cli/release/agent/integrations.py @@ -29,18 +29,14 @@ def integrations(app: Application, since: str, to: str, write: bool, force: bool tool will generate the list for every Agent since version 6.3.0 (before that point we don't have enough information to build the log). """ - from ddev.cli.release.agent.common import get_agent_tags, parse_agent_req_file + from ddev.cli.release.agent.common import get_agent_release_catalog, get_agent_tags agent_tags = get_agent_tags(app.repo, since, to) - # get the list of integrations shipped with the agent from the requirements file - req_file_name = app.repo.agent_release_requirements.name integrations_contents = StringIO() for tag in agent_tags: integrations_contents.write(f'## Datadog Agent version {tag}\n\n') - # Requirements for current tag - file_contents = app.repo.git.show_file(req_file_name, tag) - for name, ver in parse_agent_req_file(file_contents).items(): + for name, ver in get_agent_release_catalog(app.repo, tag).items(): integrations_contents.write(f'* {name}: {ver}\n') integrations_contents.write('\n') diff --git a/ddev/tests/cli/release/agent/conftest.py b/ddev/tests/cli/release/agent/conftest.py index 35172b2b57118..854c0f3c8c441 100644 --- a/ddev/tests/cli/release/agent/conftest.py +++ b/ddev/tests/cli/release/agent/conftest.py @@ -95,6 +95,52 @@ def write_dummy_pyproject(repo_path, integration): ) +@pytest.fixture +def repo_with_agent_release_exclusion_range(tmp_path_factory, config_file): + repo_path = tmp_path_factory.mktemp('integrations-core') + repo = Repository('integrations-core', str(repo_path)) + + def commit(msg): + repo.git.run('commit', '--no-verify', '-a', '-m', msg) + + repo.git.run('init') + repo.git.run('config', 'user.email', 'you@example.com') + repo.git.run('config', 'user.name', 'Your Name') + repo.git.run('config', 'commit.gpgsign', 'false') + repo.git.run('config', 'tag.gpgsign', 'false') + + write_agent_requirements(repo.path, ['datadog-existingcheck==2.0.0']) + repo.git.run('add', '.') + commit('7.73.0 release') + repo.git.run('tag', '7.73.0') + + write_agent_requirements(repo.path, ['datadog-existingcheck==2.0.0', 'datadog-temporary==1.0.0']) + commit('7.74.0 release') + repo.git.run('tag', '7.74.0') + + write_agent_requirements(repo.path, ['datadog-existingcheck==2.0.0', 'datadog-temporary==1.0.1']) + commit('7.78.0 release') + repo.git.run('tag', '7.78.0') + + repo.git.run('tag', '7.78.1') + + (repo.path / '.ddev').mkdir() + (repo.path / '.ddev' / 'config.toml').write_text( + ''' +[overrides.release.agent.integrations.exclude_by_agent_version] +"7.74.0..7.78.0" = ["datadog-temporary"] +''' + ) + + write_dummy_manifest(repo.path, 'existingcheck') + write_dummy_manifest(repo.path, 'temporary') + + config_file.model.repos['core'] = str(repo.path) + config_file.save() + + yield repo + + @pytest.fixture def repo_with_new_integration_patched(tmp_path_factory, config_file): """ diff --git a/ddev/tests/cli/release/agent/test_changelog.py b/ddev/tests/cli/release/agent/test_changelog.py index d4b1396895e3f..0f04b407c60fb 100644 --- a/ddev/tests/cli/release/agent/test_changelog.py +++ b/ddev/tests/cli/release/agent/test_changelog.py @@ -90,6 +90,31 @@ def test_new_integration_with_non_initial_version(repo_with_new_integration_patc assert mock_fetch_tags.call_count == 1 +def test_changelog_given_version_scoped_exclusion_returns_new_integration_after_range( + repo_with_agent_release_exclusion_range, ddev, mocker +): + mock_fetch_tags = mocker.patch('ddev.utils.git.GitRepository.fetch_tags') + + result = ddev('release', 'agent', 'changelog', '--since', '7.73.0', '--to', '7.78.1') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version [7.78.1](https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#7781) + +### New Integrations +* temporary [1.0.1](https://github.com/DataDog/integrations-core/blob/master/temporary/CHANGELOG.md) + +## Datadog Agent version [7.78.0](https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#7780) + +* There were no integration updates for this version of the Agent. + +## Datadog Agent version [7.74.0](https://github.com/DataDog/datadog-agent/blob/master/CHANGELOG.rst#7740) + +* There were no integration updates for this version of the Agent. +""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + assert mock_fetch_tags.call_count == 1 + + @pytest.fixture def repo_with_fake_changelog(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path) diff --git a/ddev/tests/cli/release/agent/test_integrations.py b/ddev/tests/cli/release/agent/test_integrations.py index 10589324d25d3..935a67749300f 100644 --- a/ddev/tests/cli/release/agent/test_integrations.py +++ b/ddev/tests/cli/release/agent/test_integrations.py @@ -53,6 +53,31 @@ def test_integrations_since_to(fake_integrations, ddev): assert result.output.rstrip('\n') == expected_output.strip('\n') +def test_integrations_given_version_scoped_exclusion_returns_filtered_catalog_only_inside_range( + repo_with_agent_release_exclusion_range, ddev +): + result = ddev('release', 'agent', 'integrations', '--since', '7.73.0', '--to', '7.78.1') + assert result.exit_code == 0 + + expected_output = """## Datadog Agent version 7.78.1 + +* datadog-existingcheck: 2.0.0 +* datadog-temporary: 1.0.1 + +## Datadog Agent version 7.78.0 + +* datadog-existingcheck: 2.0.0 + +## Datadog Agent version 7.74.0 + +* datadog-existingcheck: 2.0.0 + +## Datadog Agent version 7.73.0 + +* datadog-existingcheck: 2.0.0""" + assert result.output.rstrip('\n') == expected_output.strip('\n') + + @pytest.fixture def repo_with_fake_integrations(repo_with_history, config_file): config_file.model.repos['core'] = str(repo_with_history.path)