feat(bindings/python): Add HttpClientLayer for custom HTTP client configuration

bindings/python/Cargo.toml

@@ -210,6 +210,9 @@ opendal = { version = ">=0", path = "../../core", features = [
 pyo3 = { version = "0.27.2", features = ["generate-import-lib", "jiff-02"] }
 pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] }
 pyo3-stub-gen = { version = "0.17" }
+reqwest = { version = "0.12", default-features = false, features = [
+  "rustls-tls",
+] }
 tokio = "1"
 
 [profile.release]

bindings/python/README.md

@@ -101,6 +101,29 @@ async def main():
 asyncio.run(main())
 ```
 
+### Advanced: Custom HTTP Client Configuration
+
+For testing with self-signed certificates or custom HTTP settings:
+
+```python
+import opendal
+from opendal.layers import HttpClientLayer
+
+# Create HTTP client that accepts invalid certificates (testing only!)
+client = opendal.HttpClient(danger_accept_invalid_certs=True, timeout=30.0)
+
+# Apply to operator
+op = opendal.Operator(
+    "s3",
+    bucket="my-bucket",
+    endpoint="https://localhost:9000"
+).layer(HttpClientLayer(client))
+
+op.write("test.txt", b"Hello World")
+```
+
+**⚠️ Security Warning**: `danger_accept_invalid_certs=True` disables SSL verification. Only use in testing!
+
 ---
 
 ## Development

bindings/python/examples/http_client_custom.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Example: Using Custom HTTP Client with Self-Signed Certificates.
+
+This example demonstrates how to configure OpenDAL to work with S3-compatible
+services that use self-signed SSL/TLS certificates, such as a local MinIO
+instance with HTTPS enabled.
+
+WARNING: danger_accept_invalid_certs=True disables certificate verification
+and should ONLY be used in testing/development environments. Never use this
+in production!
+"""
+
+import opendal
+from opendal.layers import HttpClientLayer
+
+
+def example_with_self_signed_cert() -> None:
+    """Connect to MinIO with self-signed certificate."""
+    # Create a custom HTTP client that accepts invalid certificates
+    # WARNING: Only use this for testing/development!
+    client = opendal.HttpClient(danger_accept_invalid_certs=True)
+
+    # Create the HTTP client layer
+    http_layer = HttpClientLayer(client)
+
+    # Create an S3 operator for a local MinIO instance with HTTPS
+    op = opendal.Operator(
+        "s3",
+        bucket="my-bucket",
+        endpoint="https://localhost:9000",
+        access_key_id="minioadmin",
+        secret_access_key="minioadmin",
+        region="us-east-1"
+    )
+
+    # Apply the custom HTTP client layer
+    op = op.layer(http_layer)
+
+    # Now you can use the operator normally
+    # Write data
+    op.write("test.txt", b"Hello, OpenDAL with custom HTTP client!")
+
+    # Read data
+    data = op.read("test.txt")
+    print(f"Read data: {data}")
+
+    # Delete data
+    op.delete("test.txt")
+    print("Data deleted successfully")
+
+
+def example_with_timeout() -> opendal.Operator:
+    """Configure HTTP client with custom timeout."""
+    # Create HTTP client with 30 second timeout
+    client = opendal.HttpClient(timeout=30.0)
+
+    # Create the layer
+    http_layer = HttpClientLayer(client)
+
+    # Apply to operator
+    op = opendal.Operator("s3", bucket="my-bucket")
+    op = op.layer(http_layer)
+
+    return op
+
+
+def example_with_both_options() -> opendal.Operator:
+    """Create custom client with both invalid certs and timeout."""
+    # Create HTTP client with both options
+    # This is useful for development/testing against local services
+    client = opendal.HttpClient(
+        danger_accept_invalid_certs=True,
+        timeout=60.0
+    )
+
+    http_layer = HttpClientLayer(client)
+
+    op = opendal.Operator(
+        "s3",
+        bucket="test-bucket",
+        endpoint="https://localhost:9000",
+        access_key_id="testuser",
+        secret_access_key="testpass",
+        region="us-east-1"
+    )
+
+    op = op.layer(http_layer)
+
+    return op
+
+
+async def example_async_with_custom_client() -> None:
+    """Use custom HTTP client with async operator."""
+    # Custom client also works with AsyncOperator
+    client = opendal.HttpClient(danger_accept_invalid_certs=True)
+    http_layer = HttpClientLayer(client)
+
+    op = opendal.AsyncOperator("memory")
+    op = op.layer(http_layer)
+
+    # Use async operations
+    await op.write("async_test.txt", b"Async write with custom client")
+    data = await op.read("async_test.txt")
+    print(f"Async read data: {data}")
+
+    await op.delete("async_test.txt")
+
+
+if __name__ == "__main__":
+    print("Example 1: Using custom HTTP client with self-signed certificates")
+    print("=" * 70)
+    print("This example requires a running MinIO instance with HTTPS.")
+    print("To run this example, uncomment the line below:")
+    print("# example_with_self_signed_cert()")
+    print()
+
+    print("Example 2: Using custom timeout")
+    print("=" * 70)
+    op = example_with_timeout()
+    print(f"Created operator with 30s timeout: {op}")
+    print()
+
+    print("Example 3: Using both options")
+    print("=" * 70)
+    op = example_with_both_options()
+    print(f"Created operator with invalid certs + timeout: {op}")
+    print()
+
+    print("For async example, run:")
+    print(">>> import asyncio")
+    print(">>> asyncio.run(example_async_with_custom_client())")

bindings/python/python/opendal/__init__.py

@@ -23,8 +23,10 @@
 if TYPE_CHECKING:
     __version__: str
     from opendal import capability, exceptions, file, layers, services, types
+    from opendal._opendal import HttpClient
 else:
     from opendal._opendal import (
+        HttpClient,  # noqa: F401
         __version__,  # noqa: F401
         capability,
         exceptions,
@@ -44,5 +46,6 @@
     "services",
     "types",
     "AsyncOperator",
+    "HttpClient",
     "Operator",
 ]

bindings/python/src/layers.rs

@@ -24,6 +24,76 @@ pub trait PythonLayer: Send + Sync {
     fn layer(&self, op: Operator) -> Operator;
 }
 
+/// A custom HTTP client for OpenDAL operations.
+///
+/// This class allows you to create a custom HTTP client with specific
+/// configurations such as accepting invalid certificates for testing
+/// purposes.
+///
+/// Examples
+/// --------
+/// >>> import opendal
+/// >>> # Create a client that accepts invalid certificates
+/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True)
+/// >>> # Use it with a layer
+/// >>> layer = opendal.layers.HttpClientLayer(client)
+/// >>> op = opendal.Operator("s3", bucket="my-bucket").layer(layer)
+#[gen_stub_pyclass]
+#[pyclass(module = "opendal")]
+#[derive(Clone)]
+pub struct HttpClient {
+    client: ocore::raw::HttpClient,
+}
+
+#[gen_stub_pymethods]
+#[pymethods]
+impl HttpClient {
+    /// Create a new HTTP client.
+    ///
+    /// Parameters
+    /// ----------
+    /// danger_accept_invalid_certs : bool, optional
+    ///     If set to True, the client will accept invalid SSL/TLS certificates.
+    ///     This is useful for testing with self-signed certificates.
+    ///     **WARNING**: This is dangerous and should only be used in testing
+    ///     environments. Never use this in production.
+    /// timeout : float, optional
+    ///     Request timeout in seconds. If not specified, no timeout is set.
+    ///
+    /// Returns
+    /// -------
+    /// HttpClient
+    ///     A new HTTP client instance.
+    #[gen_stub(override_return_type(type_repr = "HttpClient"))]
+    #[new]
+    #[pyo3(signature = (
+        danger_accept_invalid_certs = false,
+        timeout = None
+    ))]
+    fn new(danger_accept_invalid_certs: bool, timeout: Option<f64>) -> PyResult<Self> {
+        let mut builder = reqwest::Client::builder();
+
+        if danger_accept_invalid_certs {
+            builder = builder.danger_accept_invalid_certs(true);
+        }
+
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(Duration::from_micros((timeout * 1_000_000.0) as u64));
+        }
+
+        let client = builder.build().map_err(|err| {
+            pyo3::exceptions::PyRuntimeError::new_err(format!(
+                "Failed to build HTTP client: {}",
+                err
+            ))
+        })?;
+
+        Ok(Self {
+            client: ocore::raw::HttpClient::with(client),
+        })
+    }
+}
+
 /// Layers are used to intercept the operations on the underlying storage.
 #[gen_stub_pyclass]
 #[pyclass(module = "opendal.layers", subclass)]
@@ -199,3 +269,66 @@ impl MimeGuessLayer {
         Ok(class)
     }
 }
+
+/// A layer that replaces the default HTTP client with a custom one.
+///
+/// This layer allows you to customize HTTP behavior, such as accepting
+/// invalid SSL/TLS certificates for testing purposes, setting custom
+/// timeouts, or adding custom HTTP headers.
+///
+/// Examples
+/// --------
+/// >>> import opendal
+/// >>> from opendal.layers import HttpClientLayer
+/// >>> # Create a client that accepts invalid certificates (for testing only)
+/// >>> client = opendal.HttpClient(danger_accept_invalid_certs=True)
+/// >>> # Apply it to an operator
+/// >>> op = opendal.Operator("s3", bucket="my-bucket", endpoint="https://localhost:9000")
+/// >>> op = op.layer(HttpClientLayer(client))
+///
+/// Notes
+/// -----
+/// The custom HTTP client will be used for all HTTP requests made by
+/// the operator. This includes authentication requests, metadata requests,
+/// and data transfer operations.
+///
+/// **Security Warning**: Using ``danger_accept_invalid_certs=True`` disables
+/// SSL/TLS certificate verification and should only be used in testing
+/// environments. Never use this in production.
+#[gen_stub_pyclass]
+#[pyclass(module = "opendal.layers", extends = Layer)]
+#[derive(Clone)]
+pub struct HttpClientLayer {
+    client: HttpClient,
+}
+
+impl PythonLayer for HttpClientLayer {
+    fn layer(&self, op: Operator) -> Operator {
+        op.layer(ocore::layers::HttpClientLayer::new(
+            self.client.client.clone(),
+        ))
+    }
+}
+
+#[gen_stub_pymethods]
+#[pymethods]
+impl HttpClientLayer {
+    /// Create a new HttpClientLayer with a custom HTTP client.
+    ///
+    /// Parameters
+    /// ----------
+    /// client : HttpClient
+    ///     The custom HTTP client to use for all HTTP requests.
+    ///
+    /// Returns
+    /// -------
+    /// HttpClientLayer
+    #[gen_stub(override_return_type(type_repr = "HttpClientLayer"))]
+    #[new]
+    fn new(client: HttpClient) -> PyResult<PyClassInitializer<Self>> {
+        let http_client_layer = Self { client };
+        let class = PyClassInitializer::from(Layer(Box::new(http_client_layer.clone())))
+            .add_subclass(http_client_layer);
+        Ok(class)
+    }
+}

bindings/python/src/lib.rs

@@ -64,7 +64,13 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
         py,
         m,
         "layers",
-        [Layer, RetryLayer, ConcurrentLimitLayer, MimeGuessLayer]
+        [
+            Layer,
+            RetryLayer,
+            ConcurrentLimitLayer,
+            MimeGuessLayer,
+            HttpClientLayer
+        ]
     )?;
 
     // Types module
@@ -75,6 +81,7 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
         [Entry, EntryMode, Metadata, PresignedRequest]
     )?;
 
+    m.add_class::<HttpClient>()?;
     m.add_class::<WriteOptions>()?;
     m.add_class::<ReadOptions>()?;
     m.add_class::<ListOptions>()?;