From 7c4d77c37b29b5bdb5dced2fddfbbb5f4d5f223f Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:01:30 +0530 Subject: [PATCH 1/8] feat(bindings/python): Add HttpClientLayer for custom HTTP client configuration This adds support for HttpClientLayer in Python bindings, enabling users to customize the HTTP client used for all operations. This is particularly useful for: - Testing with self-signed SSL certificates via danger_accept_invalid_certs - Setting custom request timeouts - Future advanced HTTP client configurations The implementation includes: - HttpClient class with danger_accept_invalid_certs and timeout parameters - HttpClientLayer to apply custom HTTP client to operators - Comprehensive documentation and examples - Unit tests for both sync and async usage Closes #TODO --- bindings/python/README.md | 23 +++ bindings/python/docs/api/layers.md | 15 ++ .../python/examples/http_client_custom.py | 157 ++++++++++++++++++ bindings/python/src/layers.rs | 132 +++++++++++++++ bindings/python/src/lib.rs | 9 +- .../python/tests/test_http_client_layer.py | 87 ++++++++++ bindings/python/upgrade.md | 49 ++++++ 7 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 bindings/python/examples/http_client_custom.py create mode 100644 bindings/python/tests/test_http_client_layer.py diff --git a/bindings/python/README.md b/bindings/python/README.md index 22aea23f90f4..961b60faf0ee 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -101,6 +101,29 @@ async def main(): asyncio.run(main()) ``` +### Advanced: Custom HTTP Client Configuration + +For testing with self-signed certificates or custom HTTP settings: + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create HTTP client that accepts invalid certificates (testing only!) +client = opendal.HttpClient(danger_accept_invalid_certs=True, timeout=30.0) + +# Apply to operator +op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000" +).layer(HttpClientLayer(client)) + +op.write("test.txt", b"Hello World") +``` + +**⚠️ Security Warning**: `danger_accept_invalid_certs=True` disables SSL verification. Only use in testing! + --- ## Development diff --git a/bindings/python/docs/api/layers.md b/bindings/python/docs/api/layers.md index 9d89e75ffb97..67778f5358a8 100644 --- a/bindings/python/docs/api/layers.md +++ b/bindings/python/docs/api/layers.md @@ -2,6 +2,14 @@ This page documents all layers in OpenDAL. +## HttpClient + +::: opendal.HttpClient + options: + heading: "opendal.HttpClient" + heading_level: 2 + show_source: false + ## Layer ::: opendal.layers.Layer options: @@ -30,3 +38,10 @@ This page documents all layers in OpenDAL. heading: "opendal.layers.MimeGuessLayer" heading_level: 2 show_source: false + +## HttpClientLayer +::: opendal.layers.HttpClientLayer + options: + heading: "opendal.layers.HttpClientLayer" + heading_level: 2 + show_source: false diff --git a/bindings/python/examples/http_client_custom.py b/bindings/python/examples/http_client_custom.py new file mode 100644 index 000000000000..8104cc091a90 --- /dev/null +++ b/bindings/python/examples/http_client_custom.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Example: Using Custom HTTP Client with Self-Signed Certificates + +This example demonstrates how to configure OpenDAL to work with S3-compatible +services that use self-signed SSL/TLS certificates, such as a local MinIO +instance with HTTPS enabled. + +WARNING: danger_accept_invalid_certs=True disables certificate verification +and should ONLY be used in testing/development environments. Never use this +in production! +""" + +import opendal +from opendal.layers import HttpClientLayer + + +def example_with_self_signed_cert(): + """ + Example: Connect to MinIO with self-signed certificate + """ + # Create a custom HTTP client that accepts invalid certificates + # WARNING: Only use this for testing/development! + client = opendal.HttpClient(danger_accept_invalid_certs=True) + + # Create the HTTP client layer + http_layer = HttpClientLayer(client) + + # Create an S3 operator for a local MinIO instance with HTTPS + op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000", + access_key_id="minioadmin", + secret_access_key="minioadmin", + region="us-east-1" + ) + + # Apply the custom HTTP client layer + op = op.layer(http_layer) + + # Now you can use the operator normally + # Write data + op.write("test.txt", b"Hello, OpenDAL with custom HTTP client!") + + # Read data + data = op.read("test.txt") + print(f"Read data: {data}") + + # Delete data + op.delete("test.txt") + print("Data deleted successfully") + + +def example_with_timeout(): + """ + Example: Configure HTTP client with custom timeout + """ + # Create HTTP client with 30 second timeout + client = opendal.HttpClient(timeout=30.0) + + # Create the layer + http_layer = HttpClientLayer(client) + + # Apply to operator + op = opendal.Operator("s3", bucket="my-bucket") + op = op.layer(http_layer) + + return op + + +def example_with_both_options(): + """ + Example: Custom client with both invalid certs and timeout + """ + # Create HTTP client with both options + # This is useful for development/testing against local services + client = opendal.HttpClient( + danger_accept_invalid_certs=True, + timeout=60.0 + ) + + http_layer = HttpClientLayer(client) + + op = opendal.Operator( + "s3", + bucket="test-bucket", + endpoint="https://localhost:9000", + access_key_id="testuser", + secret_access_key="testpass", + region="us-east-1" + ) + + op = op.layer(http_layer) + + return op + + +async def example_async_with_custom_client(): + """ + Example: Using custom HTTP client with async operator + """ + # Custom client also works with AsyncOperator + client = opendal.HttpClient(danger_accept_invalid_certs=True) + http_layer = HttpClientLayer(client) + + op = opendal.AsyncOperator("memory") + op = op.layer(http_layer) + + # Use async operations + await op.write("async_test.txt", b"Async write with custom client") + data = await op.read("async_test.txt") + print(f"Async read data: {data}") + + await op.delete("async_test.txt") + + +if __name__ == "__main__": + print("Example 1: Using custom HTTP client with self-signed certificates") + print("=" * 70) + print("This example requires a running MinIO instance with HTTPS.") + print("To run this example, uncomment the line below:") + print("# example_with_self_signed_cert()") + print() + + print("Example 2: Using custom timeout") + print("=" * 70) + op = example_with_timeout() + print(f"Created operator with 30s timeout: {op}") + print() + + print("Example 3: Using both options") + print("=" * 70) + op = example_with_both_options() + print(f"Created operator with invalid certs + timeout: {op}") + print() + + print("For async example, run:") + print(">>> import asyncio") + print(">>> asyncio.run(example_async_with_custom_client())") diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs index 84365f88e530..b269a42e51a0 100644 --- a/bindings/python/src/layers.rs +++ b/bindings/python/src/layers.rs @@ -24,6 +24,75 @@ pub trait PythonLayer: Send + Sync { fn layer(&self, op: Operator) -> Operator; } +/// A custom HTTP client for OpenDAL operations. +/// +/// This class allows you to create a custom HTTP client with specific +/// configurations such as accepting invalid certificates for testing +/// purposes. +/// +/// Examples +/// -------- +/// >>> import opendal +/// >>> # Create a client that accepts invalid certificates +/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True) +/// >>> # Use it with a layer +/// >>> layer = opendal.layers.HttpClientLayer(client) +/// >>> op = opendal.Operator("s3", bucket="my-bucket").layer(layer) +#[gen_stub_pyclass] +#[pyclass(module = "opendal")] +pub struct HttpClient { + client: ocore::raw::HttpClient, +} + +#[gen_stub_pymethods] +#[pymethods] +impl HttpClient { + /// Create a new HTTP client. + /// + /// Parameters + /// ---------- + /// danger_accept_invalid_certs : bool, optional + /// If set to True, the client will accept invalid SSL/TLS certificates. + /// This is useful for testing with self-signed certificates. + /// **WARNING**: This is dangerous and should only be used in testing + /// environments. Never use this in production. + /// timeout : float, optional + /// Request timeout in seconds. If not specified, no timeout is set. + /// + /// Returns + /// ------- + /// HttpClient + /// A new HTTP client instance. + #[gen_stub(override_return_type(type_repr = "HttpClient"))] + #[new] + #[pyo3(signature = ( + danger_accept_invalid_certs = false, + timeout = None + ))] + fn new(danger_accept_invalid_certs: bool, timeout: Option) -> PyResult { + let mut builder = reqwest::Client::builder(); + + if danger_accept_invalid_certs { + builder = builder.danger_accept_invalid_certs(true); + } + + if let Some(timeout) = timeout { + builder = builder.timeout(Duration::from_micros((timeout * 1_000_000.0) as u64)); + } + + let client = builder.build().map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build HTTP client: {}", + err + )) + })?; + + Ok(Self { + client: ocore::raw::HttpClient::with(client), + }) + } +} + /// Layers are used to intercept the operations on the underlying storage. #[gen_stub_pyclass] #[pyclass(module = "opendal.layers", subclass)] @@ -199,3 +268,66 @@ impl MimeGuessLayer { Ok(class) } } + +/// A layer that replaces the default HTTP client with a custom one. +/// +/// This layer allows you to customize HTTP behavior, such as accepting +/// invalid SSL/TLS certificates for testing purposes, setting custom +/// timeouts, or adding custom HTTP headers. +/// +/// Examples +/// -------- +/// >>> import opendal +/// >>> from opendal.layers import HttpClientLayer +/// >>> # Create a client that accepts invalid certificates (for testing only) +/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True) +/// >>> # Apply it to an operator +/// >>> op = opendal.Operator("s3", bucket="my-bucket", endpoint="https://localhost:9000") +/// >>> op = op.layer(HttpClientLayer(client)) +/// +/// Notes +/// ----- +/// The custom HTTP client will be used for all HTTP requests made by +/// the operator. This includes authentication requests, metadata requests, +/// and data transfer operations. +/// +/// **Security Warning**: Using ``danger_accept_invalid_certs=True`` disables +/// SSL/TLS certificate verification and should only be used in testing +/// environments. Never use this in production. +#[gen_stub_pyclass] +#[pyclass(module = "opendal.layers", extends = Layer)] +#[derive(Clone)] +pub struct HttpClientLayer { + client: HttpClient, +} + +impl PythonLayer for HttpClientLayer { + fn layer(&self, op: Operator) -> Operator { + op.layer(ocore::layers::HttpClientLayer::new( + self.client.client.clone(), + )) + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl HttpClientLayer { + /// Create a new HttpClientLayer with a custom HTTP client. + /// + /// Parameters + /// ---------- + /// client : HttpClient + /// The custom HTTP client to use for all HTTP requests. + /// + /// Returns + /// ------- + /// HttpClientLayer + #[gen_stub(override_return_type(type_repr = "HttpClientLayer"))] + #[new] + fn new(client: HttpClient) -> PyResult> { + let http_client_layer = Self { client }; + let class = PyClassInitializer::from(Layer(Box::new(http_client_layer.clone()))) + .add_subclass(http_client_layer); + Ok(class) + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 18143a794dd9..90918278ae9b 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -64,7 +64,13 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { py, m, "layers", - [Layer, RetryLayer, ConcurrentLimitLayer, MimeGuessLayer] + [ + Layer, + RetryLayer, + ConcurrentLimitLayer, + MimeGuessLayer, + HttpClientLayer + ] )?; // Types module @@ -75,6 +81,7 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { [Entry, EntryMode, Metadata, PresignedRequest] )?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/bindings/python/tests/test_http_client_layer.py b/bindings/python/tests/test_http_client_layer.py new file mode 100644 index 000000000000..8ec87d8ddda7 --- /dev/null +++ b/bindings/python/tests/test_http_client_layer.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +import opendal +from opendal.layers import HttpClientLayer + + +def test_http_client_creation(): + """Test that HttpClient can be created with default settings.""" + client = opendal.HttpClient() + assert client is not None + + +def test_http_client_with_invalid_certs(): + """Test that HttpClient can be created with danger_accept_invalid_certs.""" + client = opendal.HttpClient(danger_accept_invalid_certs=True) + assert client is not None + + +def test_http_client_with_timeout(): + """Test that HttpClient can be created with a custom timeout.""" + client = opendal.HttpClient(timeout=30.0) + assert client is not None + + +def test_http_client_with_all_options(): + """Test that HttpClient can be created with all options.""" + client = opendal.HttpClient(danger_accept_invalid_certs=True, timeout=60.0) + assert client is not None + + +def test_http_client_layer_creation(): + """Test that HttpClientLayer can be created.""" + client = opendal.HttpClient() + layer = HttpClientLayer(client) + assert layer is not None + + +def test_http_client_layer_with_operator(): + """Test that HttpClientLayer can be applied to an operator.""" + # Create a custom HTTP client that accepts invalid certificates + client = opendal.HttpClient(danger_accept_invalid_certs=True) + layer = HttpClientLayer(client) + + # Create an operator and apply the layer + # Using memory service for testing since it doesn't require external setup + op = opendal.Operator("memory") + op = op.layer(layer) + + # Verify the operator still works + assert op is not None + # Basic functionality test + op.write("test_file", b"test content") + assert op.read("test_file") == b"test content" + + +@pytest.mark.asyncio +async def test_http_client_layer_with_async_operator(): + """Test that HttpClientLayer can be applied to an async operator.""" + # Create a custom HTTP client with timeout + client = opendal.HttpClient(timeout=30.0) + layer = HttpClientLayer(client) + + # Create an async operator and apply the layer + op = opendal.AsyncOperator("memory") + op = op.layer(layer) + + # Verify the operator still works + assert op is not None + # Basic functionality test + await op.write("test_file_async", b"test content async") + assert await op.read("test_file_async") == b"test content async" diff --git a/bindings/python/upgrade.md b/bindings/python/upgrade.md index d365283c4e0a..50f5766ef959 100644 --- a/bindings/python/upgrade.md +++ b/bindings/python/upgrade.md @@ -1,5 +1,54 @@ # Upgrade to v0.47 +## New feature: HttpClientLayer for custom HTTP client configuration + +OpenDAL Python bindings now support `HttpClientLayer`, allowing you to customize the HTTP client used for all operations. This is particularly useful for: + +- **Testing with self-signed certificates**: Use `danger_accept_invalid_certs=True` when connecting to local/development services with self-signed SSL certificates (e.g., local MinIO, S3-compatible services) +- **Custom timeouts**: Set request-specific timeout values +- **Advanced HTTP configurations**: More options may be added in future versions + +### Example: Accept invalid SSL certificates (testing only) + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create a custom HTTP client that accepts invalid certificates +# WARNING: Only use this in testing/development environments! +client = opendal.HttpClient(danger_accept_invalid_certs=True) + +# Create the layer +http_layer = HttpClientLayer(client) + +# Apply to your operator +op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000", + access_key_id="minioadmin", + secret_access_key="minioadmin" +).layer(http_layer) + +# Now you can use the operator normally +op.write("test.txt", b"Hello World") +``` + +### Example: Custom timeout + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create HTTP client with 30 second timeout +client = opendal.HttpClient(timeout=30.0) +http_layer = HttpClientLayer(client) + +op = opendal.Operator("s3", bucket="my-bucket").layer(http_layer) +``` + +**Security Warning**: `danger_accept_invalid_certs=True` disables SSL/TLS certificate verification. Never use this in production environments. + ## Breaking change: Module exports are explicit `opendal.__init__` now only re-exports the `capability`, `exceptions`, `file`, `layers`, `services`, `types`, `Operator`, and `AsyncOperator` symbols. Imports such as: From 1d4d88390b9d466f3ac76601b42dc0a1686b3036 Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:24:43 +0530 Subject: [PATCH 2/8] fix(bindings/python): Add reqwest dependency and Clone trait for HttpClient --- bindings/python/Cargo.toml | 1 + bindings/python/src/layers.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 8054ef5e70ac..8ac2fd47c76b 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -208,6 +208,7 @@ opendal = { version = ">=0", path = "../../core", features = [ "layers-mime-guess", ] } pyo3 = { version = "0.27.2", features = ["generate-import-lib", "jiff-02"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] } pyo3-stub-gen = { version = "0.17" } tokio = "1" diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs index b269a42e51a0..b8bd79a6fe5d 100644 --- a/bindings/python/src/layers.rs +++ b/bindings/python/src/layers.rs @@ -40,6 +40,7 @@ pub trait PythonLayer: Send + Sync { /// >>> op = opendal.Operator("s3", bucket="my-bucket").layer(layer) #[gen_stub_pyclass] #[pyclass(module = "opendal")] +#[derive(Clone)] pub struct HttpClient { client: ocore::raw::HttpClient, } From b3b2b67945f151d978209bab6dd6236e124c3241 Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:27:54 +0530 Subject: [PATCH 3/8] fix(bindings/python): Export HttpClient in package __init__ --- bindings/python/python/opendal/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bindings/python/python/opendal/__init__.py b/bindings/python/python/opendal/__init__.py index 77f0687c632b..6ca5c6ec6c6b 100644 --- a/bindings/python/python/opendal/__init__.py +++ b/bindings/python/python/opendal/__init__.py @@ -23,8 +23,10 @@ if TYPE_CHECKING: __version__: str from opendal import capability, exceptions, file, layers, services, types + from opendal._opendal import HttpClient else: from opendal._opendal import ( + HttpClient, # noqa: F401 __version__, # noqa: F401 capability, exceptions, @@ -44,5 +46,6 @@ "services", "types", "AsyncOperator", + "HttpClient", "Operator", ] From 8e8e6bea97c6d613b6f6d9250ad0bde3be459007 Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:29:30 +0530 Subject: [PATCH 4/8] fix(bindings/python): Fix Cargo.toml formatting --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 8ac2fd47c76b..d4bf070b16b3 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -208,9 +208,9 @@ opendal = { version = ">=0", path = "../../core", features = [ "layers-mime-guess", ] } pyo3 = { version = "0.27.2", features = ["generate-import-lib", "jiff-02"] } -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] } pyo3-stub-gen = { version = "0.17" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } tokio = "1" [profile.release] From fba4e557a97adf756a9b7405b1222de738683567 Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:31:43 +0530 Subject: [PATCH 5/8] docs(bindings/python): Remove HttpClient from layers API docs HttpClient is a top-level class (opendal.HttpClient), not in the layers submodule, so it shouldn't be documented in layers.md. Its usage is shown in the examples and docstrings. --- bindings/python/docs/api/layers.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bindings/python/docs/api/layers.md b/bindings/python/docs/api/layers.md index 67778f5358a8..a2918cba7c7d 100644 --- a/bindings/python/docs/api/layers.md +++ b/bindings/python/docs/api/layers.md @@ -2,14 +2,6 @@ This page documents all layers in OpenDAL. -## HttpClient - -::: opendal.HttpClient - options: - heading: "opendal.HttpClient" - heading_level: 2 - show_source: false - ## Layer ::: opendal.layers.Layer options: From c86a84024363cbca4fe9cc82ef2da6ec7c6f437b Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:43:25 +0530 Subject: [PATCH 6/8] fix(bindings/python): Fix linting issues - Remove trailing whitespace from test file - Format reqwest dependency in Cargo.toml (multiline array) - Remove HttpClientLayer from API docs (will be auto-generated) --- bindings/python/Cargo.toml | 4 +++- bindings/python/docs/api/layers.md | 7 ------- bindings/python/tests/test_http_client_layer.py | 8 ++++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index d4bf070b16b3..bcea7db00718 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -210,7 +210,9 @@ opendal = { version = ">=0", path = "../../core", features = [ pyo3 = { version = "0.27.2", features = ["generate-import-lib", "jiff-02"] } pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] } pyo3-stub-gen = { version = "0.17" } -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = [ + "rustls-tls", +] } tokio = "1" [profile.release] diff --git a/bindings/python/docs/api/layers.md b/bindings/python/docs/api/layers.md index a2918cba7c7d..9d89e75ffb97 100644 --- a/bindings/python/docs/api/layers.md +++ b/bindings/python/docs/api/layers.md @@ -30,10 +30,3 @@ This page documents all layers in OpenDAL. heading: "opendal.layers.MimeGuessLayer" heading_level: 2 show_source: false - -## HttpClientLayer -::: opendal.layers.HttpClientLayer - options: - heading: "opendal.layers.HttpClientLayer" - heading_level: 2 - show_source: false diff --git a/bindings/python/tests/test_http_client_layer.py b/bindings/python/tests/test_http_client_layer.py index 8ec87d8ddda7..a4c37edeaad5 100644 --- a/bindings/python/tests/test_http_client_layer.py +++ b/bindings/python/tests/test_http_client_layer.py @@ -56,12 +56,12 @@ def test_http_client_layer_with_operator(): # Create a custom HTTP client that accepts invalid certificates client = opendal.HttpClient(danger_accept_invalid_certs=True) layer = HttpClientLayer(client) - + # Create an operator and apply the layer # Using memory service for testing since it doesn't require external setup op = opendal.Operator("memory") op = op.layer(layer) - + # Verify the operator still works assert op is not None # Basic functionality test @@ -75,11 +75,11 @@ async def test_http_client_layer_with_async_operator(): # Create a custom HTTP client with timeout client = opendal.HttpClient(timeout=30.0) layer = HttpClientLayer(client) - + # Create an async operator and apply the layer op = opendal.AsyncOperator("memory") op = op.layer(layer) - + # Verify the operator still works assert op is not None # Basic functionality test From 3e7fa6ad5debefc89072f7d58abf0de3bf85debd Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:57:08 +0530 Subject: [PATCH 7/8] fix(bindings/python): Fix Python linting issues - Add return type annotations to all functions - Fix docstring formatting (one-line docstrings, periods, imperative mood) - Remove trailing whitespace from all files - Sort imports in test file --- .../python/examples/http_client_custom.py | 65 ++++++++----------- .../python/tests/test_http_client_layer.py | 3 +- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/bindings/python/examples/http_client_custom.py b/bindings/python/examples/http_client_custom.py index 8104cc091a90..ab734298c9a8 100644 --- a/bindings/python/examples/http_client_custom.py +++ b/bindings/python/examples/http_client_custom.py @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. -""" -Example: Using Custom HTTP Client with Self-Signed Certificates +"""Example: Using Custom HTTP Client with Self-Signed Certificates. This example demonstrates how to configure OpenDAL to work with S3-compatible services that use self-signed SSL/TLS certificates, such as a local MinIO @@ -32,17 +31,15 @@ from opendal.layers import HttpClientLayer -def example_with_self_signed_cert(): - """ - Example: Connect to MinIO with self-signed certificate - """ +def example_with_self_signed_cert() -> None: + """Connect to MinIO with self-signed certificate.""" # Create a custom HTTP client that accepts invalid certificates # WARNING: Only use this for testing/development! client = opendal.HttpClient(danger_accept_invalid_certs=True) - + # Create the HTTP client layer http_layer = HttpClientLayer(client) - + # Create an S3 operator for a local MinIO instance with HTTPS op = opendal.Operator( "s3", @@ -52,53 +49,49 @@ def example_with_self_signed_cert(): secret_access_key="minioadmin", region="us-east-1" ) - + # Apply the custom HTTP client layer op = op.layer(http_layer) - + # Now you can use the operator normally # Write data op.write("test.txt", b"Hello, OpenDAL with custom HTTP client!") - + # Read data data = op.read("test.txt") print(f"Read data: {data}") - + # Delete data op.delete("test.txt") print("Data deleted successfully") -def example_with_timeout(): - """ - Example: Configure HTTP client with custom timeout - """ +def example_with_timeout() -> opendal.Operator: + """Configure HTTP client with custom timeout.""" # Create HTTP client with 30 second timeout client = opendal.HttpClient(timeout=30.0) - + # Create the layer http_layer = HttpClientLayer(client) - + # Apply to operator op = opendal.Operator("s3", bucket="my-bucket") op = op.layer(http_layer) - + return op -def example_with_both_options(): - """ - Example: Custom client with both invalid certs and timeout - """ +def example_with_both_options() -> opendal.Operator: + """Create custom client with both invalid certs and timeout.""" # Create HTTP client with both options # This is useful for development/testing against local services client = opendal.HttpClient( danger_accept_invalid_certs=True, timeout=60.0 ) - + http_layer = HttpClientLayer(client) - + op = opendal.Operator( "s3", bucket="test-bucket", @@ -107,28 +100,26 @@ def example_with_both_options(): secret_access_key="testpass", region="us-east-1" ) - + op = op.layer(http_layer) - + return op -async def example_async_with_custom_client(): - """ - Example: Using custom HTTP client with async operator - """ +async def example_async_with_custom_client() -> None: + """Use custom HTTP client with async operator.""" # Custom client also works with AsyncOperator client = opendal.HttpClient(danger_accept_invalid_certs=True) http_layer = HttpClientLayer(client) - + op = opendal.AsyncOperator("memory") op = op.layer(http_layer) - + # Use async operations await op.write("async_test.txt", b"Async write with custom client") data = await op.read("async_test.txt") print(f"Async read data: {data}") - + await op.delete("async_test.txt") @@ -139,19 +130,19 @@ async def example_async_with_custom_client(): print("To run this example, uncomment the line below:") print("# example_with_self_signed_cert()") print() - + print("Example 2: Using custom timeout") print("=" * 70) op = example_with_timeout() print(f"Created operator with 30s timeout: {op}") print() - + print("Example 3: Using both options") print("=" * 70) op = example_with_both_options() print(f"Created operator with invalid certs + timeout: {op}") print() - + print("For async example, run:") print(">>> import asyncio") print(">>> asyncio.run(example_async_with_custom_client())") diff --git a/bindings/python/tests/test_http_client_layer.py b/bindings/python/tests/test_http_client_layer.py index a4c37edeaad5..9fa83a11753d 100644 --- a/bindings/python/tests/test_http_client_layer.py +++ b/bindings/python/tests/test_http_client_layer.py @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. -import pytest import opendal +import pytest + from opendal.layers import HttpClientLayer From 3d329e52ee9d2b219dd8da112540909828701c7c Mon Sep 17 00:00:00 2001 From: rgidda Date: Wed, 18 Mar 2026 13:59:38 +0530 Subject: [PATCH 8/8] fix(bindings/python): Fix import order in test file --- bindings/python/tests/test_http_client_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/tests/test_http_client_layer.py b/bindings/python/tests/test_http_client_layer.py index 9fa83a11753d..f6240dd36dfd 100644 --- a/bindings/python/tests/test_http_client_layer.py +++ b/bindings/python/tests/test_http_client_layer.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -import opendal import pytest +import opendal from opendal.layers import HttpClientLayer