diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 8054ef5e70ac..bcea7db00718 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -210,6 +210,9 @@ opendal = { version = ">=0", path = "../../core", features = [ pyo3 = { version = "0.27.2", features = ["generate-import-lib", "jiff-02"] } pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] } pyo3-stub-gen = { version = "0.17" } +reqwest = { version = "0.12", default-features = false, features = [ + "rustls-tls", +] } tokio = "1" [profile.release] diff --git a/bindings/python/README.md b/bindings/python/README.md index 22aea23f90f4..961b60faf0ee 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -101,6 +101,29 @@ async def main(): asyncio.run(main()) ``` +### Advanced: Custom HTTP Client Configuration + +For testing with self-signed certificates or custom HTTP settings: + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create HTTP client that accepts invalid certificates (testing only!) +client = opendal.HttpClient(danger_accept_invalid_certs=True, timeout=30.0) + +# Apply to operator +op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000" +).layer(HttpClientLayer(client)) + +op.write("test.txt", b"Hello World") +``` + +**⚠️ Security Warning**: `danger_accept_invalid_certs=True` disables SSL verification. Only use in testing! + --- ## Development diff --git a/bindings/python/examples/http_client_custom.py b/bindings/python/examples/http_client_custom.py new file mode 100644 index 000000000000..ab734298c9a8 --- /dev/null +++ b/bindings/python/examples/http_client_custom.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Example: Using Custom HTTP Client with Self-Signed Certificates. + +This example demonstrates how to configure OpenDAL to work with S3-compatible +services that use self-signed SSL/TLS certificates, such as a local MinIO +instance with HTTPS enabled. + +WARNING: danger_accept_invalid_certs=True disables certificate verification +and should ONLY be used in testing/development environments. Never use this +in production! +""" + +import opendal +from opendal.layers import HttpClientLayer + + +def example_with_self_signed_cert() -> None: + """Connect to MinIO with self-signed certificate.""" + # Create a custom HTTP client that accepts invalid certificates + # WARNING: Only use this for testing/development! + client = opendal.HttpClient(danger_accept_invalid_certs=True) + + # Create the HTTP client layer + http_layer = HttpClientLayer(client) + + # Create an S3 operator for a local MinIO instance with HTTPS + op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000", + access_key_id="minioadmin", + secret_access_key="minioadmin", + region="us-east-1" + ) + + # Apply the custom HTTP client layer + op = op.layer(http_layer) + + # Now you can use the operator normally + # Write data + op.write("test.txt", b"Hello, OpenDAL with custom HTTP client!") + + # Read data + data = op.read("test.txt") + print(f"Read data: {data}") + + # Delete data + op.delete("test.txt") + print("Data deleted successfully") + + +def example_with_timeout() -> opendal.Operator: + """Configure HTTP client with custom timeout.""" + # Create HTTP client with 30 second timeout + client = opendal.HttpClient(timeout=30.0) + + # Create the layer + http_layer = HttpClientLayer(client) + + # Apply to operator + op = opendal.Operator("s3", bucket="my-bucket") + op = op.layer(http_layer) + + return op + + +def example_with_both_options() -> opendal.Operator: + """Create custom client with both invalid certs and timeout.""" + # Create HTTP client with both options + # This is useful for development/testing against local services + client = opendal.HttpClient( + danger_accept_invalid_certs=True, + timeout=60.0 + ) + + http_layer = HttpClientLayer(client) + + op = opendal.Operator( + "s3", + bucket="test-bucket", + endpoint="https://localhost:9000", + access_key_id="testuser", + secret_access_key="testpass", + region="us-east-1" + ) + + op = op.layer(http_layer) + + return op + + +async def example_async_with_custom_client() -> None: + """Use custom HTTP client with async operator.""" + # Custom client also works with AsyncOperator + client = opendal.HttpClient(danger_accept_invalid_certs=True) + http_layer = HttpClientLayer(client) + + op = opendal.AsyncOperator("memory") + op = op.layer(http_layer) + + # Use async operations + await op.write("async_test.txt", b"Async write with custom client") + data = await op.read("async_test.txt") + print(f"Async read data: {data}") + + await op.delete("async_test.txt") + + +if __name__ == "__main__": + print("Example 1: Using custom HTTP client with self-signed certificates") + print("=" * 70) + print("This example requires a running MinIO instance with HTTPS.") + print("To run this example, uncomment the line below:") + print("# example_with_self_signed_cert()") + print() + + print("Example 2: Using custom timeout") + print("=" * 70) + op = example_with_timeout() + print(f"Created operator with 30s timeout: {op}") + print() + + print("Example 3: Using both options") + print("=" * 70) + op = example_with_both_options() + print(f"Created operator with invalid certs + timeout: {op}") + print() + + print("For async example, run:") + print(">>> import asyncio") + print(">>> asyncio.run(example_async_with_custom_client())") diff --git a/bindings/python/python/opendal/__init__.py b/bindings/python/python/opendal/__init__.py index 77f0687c632b..6ca5c6ec6c6b 100644 --- a/bindings/python/python/opendal/__init__.py +++ b/bindings/python/python/opendal/__init__.py @@ -23,8 +23,10 @@ if TYPE_CHECKING: __version__: str from opendal import capability, exceptions, file, layers, services, types + from opendal._opendal import HttpClient else: from opendal._opendal import ( + HttpClient, # noqa: F401 __version__, # noqa: F401 capability, exceptions, @@ -44,5 +46,6 @@ "services", "types", "AsyncOperator", + "HttpClient", "Operator", ] diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs index 84365f88e530..b8bd79a6fe5d 100644 --- a/bindings/python/src/layers.rs +++ b/bindings/python/src/layers.rs @@ -24,6 +24,76 @@ pub trait PythonLayer: Send + Sync { fn layer(&self, op: Operator) -> Operator; } +/// A custom HTTP client for OpenDAL operations. +/// +/// This class allows you to create a custom HTTP client with specific +/// configurations such as accepting invalid certificates for testing +/// purposes. +/// +/// Examples +/// -------- +/// >>> import opendal +/// >>> # Create a client that accepts invalid certificates +/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True) +/// >>> # Use it with a layer +/// >>> layer = opendal.layers.HttpClientLayer(client) +/// >>> op = opendal.Operator("s3", bucket="my-bucket").layer(layer) +#[gen_stub_pyclass] +#[pyclass(module = "opendal")] +#[derive(Clone)] +pub struct HttpClient { + client: ocore::raw::HttpClient, +} + +#[gen_stub_pymethods] +#[pymethods] +impl HttpClient { + /// Create a new HTTP client. + /// + /// Parameters + /// ---------- + /// danger_accept_invalid_certs : bool, optional + /// If set to True, the client will accept invalid SSL/TLS certificates. + /// This is useful for testing with self-signed certificates. + /// **WARNING**: This is dangerous and should only be used in testing + /// environments. Never use this in production. + /// timeout : float, optional + /// Request timeout in seconds. If not specified, no timeout is set. + /// + /// Returns + /// ------- + /// HttpClient + /// A new HTTP client instance. + #[gen_stub(override_return_type(type_repr = "HttpClient"))] + #[new] + #[pyo3(signature = ( + danger_accept_invalid_certs = false, + timeout = None + ))] + fn new(danger_accept_invalid_certs: bool, timeout: Option) -> PyResult { + let mut builder = reqwest::Client::builder(); + + if danger_accept_invalid_certs { + builder = builder.danger_accept_invalid_certs(true); + } + + if let Some(timeout) = timeout { + builder = builder.timeout(Duration::from_micros((timeout * 1_000_000.0) as u64)); + } + + let client = builder.build().map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build HTTP client: {}", + err + )) + })?; + + Ok(Self { + client: ocore::raw::HttpClient::with(client), + }) + } +} + /// Layers are used to intercept the operations on the underlying storage. #[gen_stub_pyclass] #[pyclass(module = "opendal.layers", subclass)] @@ -199,3 +269,66 @@ impl MimeGuessLayer { Ok(class) } } + +/// A layer that replaces the default HTTP client with a custom one. +/// +/// This layer allows you to customize HTTP behavior, such as accepting +/// invalid SSL/TLS certificates for testing purposes, setting custom +/// timeouts, or adding custom HTTP headers. +/// +/// Examples +/// -------- +/// >>> import opendal +/// >>> from opendal.layers import HttpClientLayer +/// >>> # Create a client that accepts invalid certificates (for testing only) +/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True) +/// >>> # Apply it to an operator +/// >>> op = opendal.Operator("s3", bucket="my-bucket", endpoint="https://localhost:9000") +/// >>> op = op.layer(HttpClientLayer(client)) +/// +/// Notes +/// ----- +/// The custom HTTP client will be used for all HTTP requests made by +/// the operator. This includes authentication requests, metadata requests, +/// and data transfer operations. +/// +/// **Security Warning**: Using ``danger_accept_invalid_certs=True`` disables +/// SSL/TLS certificate verification and should only be used in testing +/// environments. Never use this in production. +#[gen_stub_pyclass] +#[pyclass(module = "opendal.layers", extends = Layer)] +#[derive(Clone)] +pub struct HttpClientLayer { + client: HttpClient, +} + +impl PythonLayer for HttpClientLayer { + fn layer(&self, op: Operator) -> Operator { + op.layer(ocore::layers::HttpClientLayer::new( + self.client.client.clone(), + )) + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl HttpClientLayer { + /// Create a new HttpClientLayer with a custom HTTP client. + /// + /// Parameters + /// ---------- + /// client : HttpClient + /// The custom HTTP client to use for all HTTP requests. + /// + /// Returns + /// ------- + /// HttpClientLayer + #[gen_stub(override_return_type(type_repr = "HttpClientLayer"))] + #[new] + fn new(client: HttpClient) -> PyResult> { + let http_client_layer = Self { client }; + let class = PyClassInitializer::from(Layer(Box::new(http_client_layer.clone()))) + .add_subclass(http_client_layer); + Ok(class) + } +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 18143a794dd9..90918278ae9b 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -64,7 +64,13 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { py, m, "layers", - [Layer, RetryLayer, ConcurrentLimitLayer, MimeGuessLayer] + [ + Layer, + RetryLayer, + ConcurrentLimitLayer, + MimeGuessLayer, + HttpClientLayer + ] )?; // Types module @@ -75,6 +81,7 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { [Entry, EntryMode, Metadata, PresignedRequest] )?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/bindings/python/tests/test_http_client_layer.py b/bindings/python/tests/test_http_client_layer.py new file mode 100644 index 000000000000..f6240dd36dfd --- /dev/null +++ b/bindings/python/tests/test_http_client_layer.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import opendal +from opendal.layers import HttpClientLayer + + +def test_http_client_creation(): + """Test that HttpClient can be created with default settings.""" + client = opendal.HttpClient() + assert client is not None + + +def test_http_client_with_invalid_certs(): + """Test that HttpClient can be created with danger_accept_invalid_certs.""" + client = opendal.HttpClient(danger_accept_invalid_certs=True) + assert client is not None + + +def test_http_client_with_timeout(): + """Test that HttpClient can be created with a custom timeout.""" + client = opendal.HttpClient(timeout=30.0) + assert client is not None + + +def test_http_client_with_all_options(): + """Test that HttpClient can be created with all options.""" + client = opendal.HttpClient(danger_accept_invalid_certs=True, timeout=60.0) + assert client is not None + + +def test_http_client_layer_creation(): + """Test that HttpClientLayer can be created.""" + client = opendal.HttpClient() + layer = HttpClientLayer(client) + assert layer is not None + + +def test_http_client_layer_with_operator(): + """Test that HttpClientLayer can be applied to an operator.""" + # Create a custom HTTP client that accepts invalid certificates + client = opendal.HttpClient(danger_accept_invalid_certs=True) + layer = HttpClientLayer(client) + + # Create an operator and apply the layer + # Using memory service for testing since it doesn't require external setup + op = opendal.Operator("memory") + op = op.layer(layer) + + # Verify the operator still works + assert op is not None + # Basic functionality test + op.write("test_file", b"test content") + assert op.read("test_file") == b"test content" + + +@pytest.mark.asyncio +async def test_http_client_layer_with_async_operator(): + """Test that HttpClientLayer can be applied to an async operator.""" + # Create a custom HTTP client with timeout + client = opendal.HttpClient(timeout=30.0) + layer = HttpClientLayer(client) + + # Create an async operator and apply the layer + op = opendal.AsyncOperator("memory") + op = op.layer(layer) + + # Verify the operator still works + assert op is not None + # Basic functionality test + await op.write("test_file_async", b"test content async") + assert await op.read("test_file_async") == b"test content async" diff --git a/bindings/python/upgrade.md b/bindings/python/upgrade.md index d365283c4e0a..50f5766ef959 100644 --- a/bindings/python/upgrade.md +++ b/bindings/python/upgrade.md @@ -1,5 +1,54 @@ # Upgrade to v0.47 +## New feature: HttpClientLayer for custom HTTP client configuration + +OpenDAL Python bindings now support `HttpClientLayer`, allowing you to customize the HTTP client used for all operations. This is particularly useful for: + +- **Testing with self-signed certificates**: Use `danger_accept_invalid_certs=True` when connecting to local/development services with self-signed SSL certificates (e.g., local MinIO, S3-compatible services) +- **Custom timeouts**: Set request-specific timeout values +- **Advanced HTTP configurations**: More options may be added in future versions + +### Example: Accept invalid SSL certificates (testing only) + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create a custom HTTP client that accepts invalid certificates +# WARNING: Only use this in testing/development environments! +client = opendal.HttpClient(danger_accept_invalid_certs=True) + +# Create the layer +http_layer = HttpClientLayer(client) + +# Apply to your operator +op = opendal.Operator( + "s3", + bucket="my-bucket", + endpoint="https://localhost:9000", + access_key_id="minioadmin", + secret_access_key="minioadmin" +).layer(http_layer) + +# Now you can use the operator normally +op.write("test.txt", b"Hello World") +``` + +### Example: Custom timeout + +```python +import opendal +from opendal.layers import HttpClientLayer + +# Create HTTP client with 30 second timeout +client = opendal.HttpClient(timeout=30.0) +http_layer = HttpClientLayer(client) + +op = opendal.Operator("s3", bucket="my-bucket").layer(http_layer) +``` + +**Security Warning**: `danger_accept_invalid_certs=True` disables SSL/TLS certificate verification. Never use this in production environments. + ## Breaking change: Module exports are explicit `opendal.__init__` now only re-exports the `capability`, `exceptions`, `file`, `layers`, `services`, `types`, `Operator`, and `AsyncOperator` symbols. Imports such as: