Feature branch for httpx migration

airflow/datadog_checks/airflow/airflow.py

@@ -7,6 +7,7 @@
 import requests
 
 from datadog_checks.base import AgentCheck, ConfigurationError
+from datadog_checks.base.utils.http_exceptions import HTTPTimeoutError
 from datadog_checks.base.utils.time import get_timestamp
 
 AIRFLOW_STATUS_OK = "OK"
@@ -141,6 +142,6 @@ def _get_json(self, url):
             self.warning(
                 "Couldn't connect to URL: %s with exception: %s. Please verify the address is reachable", url, e
             )
-        except requests.exceptions.Timeout as e:
+        except (requests.exceptions.Timeout, HTTPTimeoutError) as e:
             self.warning("Connection timeout when connecting to %s: %s", url, e)
         return None

airflow/tests/test_unit.py

@@ -26,18 +26,16 @@ def test_service_checks_cannot_connect(aggregator):
     'json_resp, expected_healthy_status, expected_healthy_value',
     [({'status': 'OK'}, AgentCheck.OK, 1), ({'status': 'KO'}, AgentCheck.CRITICAL, 0), ({}, AgentCheck.CRITICAL, 0)],
 )
-def test_service_checks_healthy_exp(aggregator, json_resp, expected_healthy_status, expected_healthy_value):
+def test_service_checks_healthy_exp(aggregator, mock_http, json_resp, expected_healthy_status, expected_healthy_value):
     instance = common.FULL_CONFIG['instances'][0]
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
-    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value=None):
-        mock_session = mock.MagicMock()
-        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session):
-            mock_resp = mock.MagicMock(status_code=200)
-            mock_resp.json.side_effect = [json_resp]
-            mock_session.get.return_value = mock_resp
+    mock_resp = mock.MagicMock(status_code=200)
+    mock_resp.json.side_effect = [json_resp]
+    mock_http.get.return_value = mock_resp
 
-            check.check(None)
+    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value=None):
+        check.check(None)
 
     tags = ['key:my-tag', 'url:http://localhost:8080']
 
@@ -54,65 +52,60 @@ def test_service_checks_healthy_exp(aggregator, json_resp, expected_healthy_stat
     ],
 )
 def test_service_checks_healthy_stable(
-    aggregator, metadb_status, scheduler_status, expected_healthy_status, expected_healthy_value
+    aggregator, mock_http, metadb_status, scheduler_status, expected_healthy_status, expected_healthy_value
 ):  # Stable is only defined in the context of Airflow 2
     instance = common.FULL_CONFIG['instances'][0]
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
-    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        mock_session = mock.MagicMock()
-        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session):
-            mock_resp = mock.MagicMock(status_code=200)
-            mock_resp.json.side_effect = [
-                {'metadatabase': {'status': metadb_status}, 'scheduler': {'status': scheduler_status}},
-                {'status': 'OK'},
-            ]
-            mock_session.get.return_value = mock_resp
+    mock_resp = mock.MagicMock(status_code=200)
+    mock_resp.json.side_effect = [
+        {'metadatabase': {'status': metadb_status}, 'scheduler': {'status': scheduler_status}},
+        {'status': 'OK'},
+    ]
+    mock_http.get.return_value = mock_resp
 
-            check.check(None)
+    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
+        check.check(None)
 
     tags = ['key:my-tag', 'url:http://localhost:8080']
 
     aggregator.assert_service_check('airflow.healthy', expected_healthy_status, tags=tags, count=1)
     aggregator.assert_metric('airflow.healthy', expected_healthy_value, tags=tags, count=1)
 
 
-def test_dag_total_tasks(aggregator, task_instance):
+def test_dag_total_tasks(aggregator, mock_http, task_instance):
     instance = common.FULL_CONFIG['instances'][0]
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
-    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        req = mock.MagicMock()
-        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
-            mock_resp = mock.MagicMock(status_code=200)
-            mock_resp.json.side_effect = [
-                {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
-                task_instance,
-            ]
-            req.get.return_value = mock_resp
+    mock_resp = mock.MagicMock(status_code=200)
+    mock_resp.json.side_effect = [
+        {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
+        task_instance,
+    ]
+    mock_http.get.return_value = mock_resp
 
-            check.check(None)
+    with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
+        check.check(None)
 
     aggregator.assert_metric('airflow.dag.task.total_running', value=1, count=1)
 
 
-def test_dag_task_ongoing_duration(aggregator, task_instance):
+def test_dag_task_ongoing_duration(aggregator, mock_http, task_instance):
     instance = common.FULL_CONFIG['instances'][0]
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
+    mock_resp = mock.MagicMock(status_code=200)
+    mock_resp.json.side_effect = [
+        {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
+    ]
+    mock_http.get.return_value = mock_resp
+
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        req = mock.MagicMock()
-        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
-            mock_resp = mock.MagicMock(status_code=200)
-            mock_resp.json.side_effect = [
-                {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
-            ]
-            req.get.return_value = mock_resp
-            with mock.patch(
-                'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances',
-                return_value=task_instance.get('task_instances'),
-            ):
-                check.check(None)
+        with mock.patch(
+            'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances',
+            return_value=task_instance.get('task_instances'),
+        ):
+            check.check(None)
 
     aggregator.assert_metric(
         'airflow.dag.task.ongoing_duration',
@@ -141,23 +134,21 @@ def test_dag_task_ongoing_duration(aggregator, task_instance):
         ),
     ],
 )
-def test_config_collect_ongoing_duration(collect_ongoing_duration, should_call_method):
+def test_config_collect_ongoing_duration(mock_http, collect_ongoing_duration, should_call_method):
     instance = {**common.FULL_CONFIG['instances'][0], 'collect_ongoing_duration': collect_ongoing_duration}
     check = AirflowCheck('airflow', common.FULL_CONFIG, [instance])
 
+    mock_resp = mock.MagicMock(status_code=200)
+    mock_resp.json.side_effect = [
+        {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
+    ]
+    mock_http.get.return_value = mock_resp
+
     with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'):
-        req = mock.MagicMock()
-        with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req):
-            mock_resp = mock.MagicMock(status_code=200)
-            mock_resp.json.side_effect = [
-                {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}},
-            ]
-            req.get.return_value = mock_resp
-
-            with mock.patch(
-                'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances'
-            ) as mock_get_all_task_instances:
-                check.check(None)
-
-                # Assert method calls
-                mock_get_all_task_instances.assert_has_calls(should_call_method, any_order=False)
+        with mock.patch(
+            'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances'
+        ) as mock_get_all_task_instances:
+            check.check(None)
+
+            # Assert method calls
+            mock_get_all_task_instances.assert_has_calls(should_call_method, any_order=False)

appgate_sdp/tests/test_unit.py

@@ -31,7 +31,7 @@ def test_check_appgate_sdp(dd_run_check, aggregator, instance, mock_http_respons
 def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance, mock_http_response):
     mock_http_response(status_code=404)
     check = AppgateSDPCheck('appgate_sdp', {}, [instance])
-    with pytest.raises(Exception, match='requests.exceptions.HTTPError'):
+    with pytest.raises(Exception, match='HTTPStatusError'):
         dd_run_check(check)
     aggregator.assert_service_check('appgate_sdp.openmetrics.health', AppgateSDPCheck.CRITICAL)
 

arangodb/tests/test_arangodb.py

@@ -9,7 +9,7 @@
 from requests import HTTPError
 
 from datadog_checks.arangodb import ArangodbCheck
-from datadog_checks.dev.http import MockResponse
+from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev.utils import get_metadata_metrics
 
 from .common import METRICS
@@ -54,7 +54,7 @@ def test_check(instance, dd_run_check, aggregator, tag_condition, base_tags):
 
     def mock_requests_get(session, url, *args, **kwargs):
         fixture = url.rsplit('/', 1)[-1]
-        return MockResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', tag_condition, fixture))
+        return MockHTTPResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', tag_condition, fixture))
 
     with mock.patch('requests.Session.get', side_effect=mock_requests_get, autospec=True):
         dd_run_check(check)

argo_workflows/tests/test_unit.py

@@ -135,6 +135,6 @@ def test_check_with_fixtures(dd_run_check, aggregator, instance, mock_http_respo
 def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance, mock_http_response):
     mock_http_response(status_code=404)
     check = ArgoWorkflowsCheck('argo_workflows', {}, [instance])
-    with pytest.raises(Exception, match='requests.exceptions.HTTPError'):
+    with pytest.raises(Exception, match='HTTPStatusError'):
         dd_run_check(check)
     aggregator.assert_service_check('argo_workflows.openmetrics.health', ArgoWorkflowsCheck.CRITICAL)

avi_vantage/tests/conftest.py

@@ -4,14 +4,14 @@
 import json
 import os
 from typing import Any, AnyStr
+from unittest.mock import MagicMock
 from urllib.parse import urlparse
 
-import mock
 import pytest
 
+from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run, get_docker_hostname, get_here
 from datadog_checks.dev.conditions import CheckDockerLogs
-from datadog_checks.dev.http import MockResponse
 
 HERE = get_here()
 
@@ -57,7 +57,7 @@ def _get_metrics(metrics_folder=NO_TENANT_METRICS_FOLDER, endpoint=None):
 
 
 @pytest.fixture
-def mock_client():
+def mock_client(mock_http):
     def mock_get(url: AnyStr, *__: Any, **___: Any):
         parsed = urlparse(url)
         resource = [part for part in parsed.path.split('/') if len(part) > 0][-1]
@@ -69,20 +69,21 @@ def mock_get(url: AnyStr, *__: Any, **___: Any):
         path['tenant=admin%2Ctenant_a%2Ctenant_b'] = MULTIPLE_TENANTS_METRICS_FOLDER
 
         if query_params:
-            return MockResponse(
+            return MockHTTPResponse(
                 file_path=os.path.join(HERE, 'compose', 'fixtures', path[query_params], f'{resource}_metrics')
             )
 
-        return MockResponse(
+        return MockHTTPResponse(
             file_path=os.path.join(HERE, 'compose', 'fixtures', NO_TENANT_METRICS_FOLDER, f'{resource}_metrics')
         )
 
     def mock_post(url: AnyStr, *__: Any, **___: Any):
-        return mock.MagicMock(status_code=200, content=b'{"results": []}')
+        return MockHTTPResponse(json_data={"results": []})
 
-    with mock.patch('datadog_checks.base.utils.http.RequestsWrapper.get', side_effect=mock_get):
-        with mock.patch('datadog_checks.base.utils.http.RequestsWrapper.post', new=mock_post):
-            yield
+    mock_http.session = MagicMock(cookies={})
+    mock_http.get.side_effect = mock_get
+    mock_http.post.side_effect = mock_post
+    yield
 
 
 @pytest.fixture(scope='session')

celery/tests/test_unit.py

@@ -44,7 +44,7 @@ def test_emits_critical_openemtrics_service_check_when_service_is_down(
     """
     mock_http_response(status_code=404)
     check = CeleryCheck("celery", {}, [instance])
-    with pytest.raises(Exception, match="requests.exceptions.HTTPError"):
+    with pytest.raises(Exception, match="HTTPStatusError"):
         dd_run_check(check)
 
     aggregator.assert_all_metrics_covered()

cert_manager/tests/test_cert_manager.py

@@ -6,8 +6,8 @@
 import mock
 import pytest
 
+from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.cert_manager import CertManagerCheck
-from datadog_checks.dev.http import MockResponse
 
 from .common import ACME_METRICS, CERT_METRICS, CONTROLLER_METRICS, MOCK_INSTANCE
 
@@ -32,7 +32,7 @@ def test_check(aggregator, dd_run_check):
     check = CertManagerCheck('cert_manager', {}, [MOCK_INSTANCE])
 
     def mock_requests_get(url, *args, **kwargs):
-        return MockResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'cert_manager.txt'))
+        return MockHTTPResponse(file_path=os.path.join(os.path.dirname(__file__), 'fixtures', 'cert_manager.txt'))
 
     with mock.patch('requests.Session.get', side_effect=mock_requests_get, autospec=True):
         dd_run_check(check)

cilium/tests/conftest.py

@@ -3,10 +3,10 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 
-import mock
 import pytest
 
 from datadog_checks.base.utils.common import get_docker_hostname
+from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.cilium import CiliumCheck
 from datadog_checks.dev import run_command
 from datadog_checks.dev.kind import kind_run
@@ -198,28 +198,18 @@ def operator_instance_use_openmetrics():
 
 
 @pytest.fixture()
-def mock_agent_data():
+def mock_agent_data(mock_openmetrics_http):
     f_name = os.path.join(os.path.dirname(__file__), "fixtures", "agent_metrics.txt")
     with open(f_name, "r") as f:
         text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
-        ),
-    ):
-        yield
+    mock_openmetrics_http.get.return_value = MockHTTPResponse(content=text_data, headers={"Content-Type": "text/plain"})
+    yield
 
 
 @pytest.fixture()
-def mock_operator_data():
+def mock_operator_data(mock_openmetrics_http):
     f_name = os.path.join(os.path.dirname(__file__), "fixtures", "operator_metrics.txt")
     with open(f_name, "r") as f:
         text_data = f.read()
-    with mock.patch(
-        'requests.Session.get',
-        return_value=mock.MagicMock(
-            status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={"Content-Type": "text/plain"}
-        ),
-    ):
-        yield
+    mock_openmetrics_http.get.return_value = MockHTTPResponse(content=text_data, headers={"Content-Type": "text/plain"})
+    yield

citrix_hypervisor/datadog_checks/citrix_hypervisor/check.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from __future__ import annotations
 
+from json import JSONDecodeError as StdJSONDecodeError
 from typing import TYPE_CHECKING, Any, Dict, List  # noqa: F401
 from xmlrpc.client import ServerProxy
 
@@ -25,7 +26,7 @@ def _safely_process_metrics_response(data: ResponseWrapper) -> dict[str, dict]:
     # See https://github.com/yaml/pyyaml/issues/443
     try:
         return data.json()
-    except requests.exceptions.JSONDecodeError:
+    except (requests.exceptions.JSONDecodeError, StdJSONDecodeError):
         return yaml.load(data.content, Loader=yaml.SafeLoader)
 
 

citrix_hypervisor/tests/conftest.py

@@ -3,11 +3,10 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 
-import mock
 import pytest
 
+from datadog_checks.base.utils.http_testing import MockHTTPResponse
 from datadog_checks.dev import docker_run
-from datadog_checks.dev.http import MockResponse
 
 from . import common
 
@@ -38,17 +37,17 @@ def mock_requests_get(url, *args, **kwargs):
     print(url_parts)
 
     if url_parts[0] == 'wrong':
-        return MockResponse(status_code=404)
+        return MockHTTPResponse(status_code=404)
 
     json_file = f"rrd_updates_{url_parts[0]}.json" if url_parts[1] == "rrd_updates" else f"{url_parts[1]}.json"
     path = os.path.join(common.HERE, 'fixtures', 'standalone', json_file)
     if not os.path.exists(path):
-        return MockResponse(status_code=404)
+        return MockHTTPResponse(status_code=404)
 
-    return MockResponse(file_path=path)
+    return MockHTTPResponse(file_path=path)
 
 
 @pytest.fixture
-def mock_responses():
-    with mock.patch('requests.Session.get', side_effect=mock_requests_get):
-        yield
+def mock_responses(mock_http):
+    mock_http.get.side_effect = mock_requests_get
+    yield