From 080ab64c7f79849e24915ae5c3224daa8946b400 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 31 Mar 2026 02:27:50 +0000 Subject: [PATCH] fix: combine k8s proxy test images to eliminate duplicate interpreter layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit py_image_layer embeds the binary name in runfiles paths, so separate binaries produce different layer digests for identical Python content. mock_k8s_server and k8s_test_client each had a 112 MB interpreter layer with different hashes — loaded twice by docker, never deduplicated. Combine both scripts into a single aspect_py_binary with a dispatcher (k8s_proxy_test_tools.py) that routes via K8S_PROXY_TEST_ROLE env var. One image, one docker load, one interpreter layer. Before: 3 docker loads (99 + 145 + 144 = 388 MB), test time 45.3s After: 2 docker loads (99 + 155 = 254 MB), test time 35.8s Also set size="medium" (300s) for safe headroom — the default size="small" (60s) was causing intermittent timeouts on slower RBE workers (passing runs at 56s, timeout at 60.1s). https://claude.ai/code/session_01ANqoTWWCxF71H5Aq2DqwnT --- .../hook_daemon/session_start/BUILD.bazel | 85 ++++++------------- .../session_start/k8s_proxy_test_tools.py | 29 +++++++ .../test_k8s_proxy_integration.py | 47 +++++----- 3 files changed, 82 insertions(+), 79 deletions(-) create mode 100644 devinfra/claude/hook_daemon/session_start/k8s_proxy_test_tools.py diff --git a/devinfra/claude/hook_daemon/session_start/BUILD.bazel b/devinfra/claude/hook_daemon/session_start/BUILD.bazel index 15b856928b..f01826fbb9 100644 --- a/devinfra/claude/hook_daemon/session_start/BUILD.bazel +++ b/devinfra/claude/hook_daemon/session_start/BUILD.bazel @@ -170,80 +170,51 @@ py_library( # === Test container images === -# Mock k8s API server (stdlib only — no pip deps). +# Combined test tools image: mock k8s server + k8s test client in one image. +# Eliminates duplicate ~112 MB interpreter layers (py_image_layer embeds the +# binary name in runfiles paths, so separate binaries can't share layers). +# See devinfra/claude/docs/plans/stable-session-identity.md for analysis. aspect_py_binary( - name = "mock_k8s_server_bin", + name = "k8s_proxy_test_tools_bin", testonly = True, - srcs = ["mock_k8s_server.py"], - main = "mock_k8s_server.py", - deps = ["//devinfra/claude/testing:proxy_ca"], -) - -py_image_layer( - name = "mock_k8s_server_layers", - testonly = True, - binary = ":mock_k8s_server_bin", -) - -oci_image( - name = "mock_k8s_server_image", - testonly = True, - base = "@debian_slim_linux_amd64", - entrypoint = py_image_entrypoint("mock_k8s_server_bin"), - env = py_image_env(binary_name = "mock_k8s_server_bin"), - tars = [":mock_k8s_server_layers"], -) - -oci_load( - name = "mock_k8s_server_load", - testonly = True, - image = ":mock_k8s_server_image", - repo_tags = ["mock-k8s-server:pinned"], -) - -filegroup( - name = "mock_k8s_server_tarball", - testonly = True, - srcs = [":mock_k8s_server_load"], - output_group = "tarball", -) - -# K8s proxy test client: exercises normalize_proxy_url() through the -# kubernetes Python client in a container on the Docker bridge network. -aspect_py_binary( - name = "k8s_test_client_bin", - testonly = True, - srcs = ["k8s_proxy_test_client.py"], - main = "k8s_proxy_test_client.py", - deps = ["@pypi//kubernetes"], + srcs = [ + "k8s_proxy_test_client.py", + "k8s_proxy_test_tools.py", + "mock_k8s_server.py", + ], + main = "k8s_proxy_test_tools.py", + deps = [ + "//devinfra/claude/testing:proxy_ca", + "@pypi//kubernetes", + ], ) py_image_layer( - name = "k8s_test_client_layers", + name = "k8s_proxy_test_tools_layers", testonly = True, - binary = ":k8s_test_client_bin", + binary = ":k8s_proxy_test_tools_bin", ) oci_image( - name = "k8s_test_client_image", + name = "k8s_proxy_test_tools_image", testonly = True, base = "@debian_slim_linux_amd64", - entrypoint = py_image_entrypoint("k8s_test_client_bin"), - env = py_image_env(binary_name = "k8s_test_client_bin"), - tars = [":k8s_test_client_layers"], + entrypoint = py_image_entrypoint("k8s_proxy_test_tools_bin"), + env = py_image_env(binary_name = "k8s_proxy_test_tools_bin"), + tars = [":k8s_proxy_test_tools_layers"], ) oci_load( - name = "k8s_test_client_load", + name = "k8s_proxy_test_tools_load", testonly = True, - image = ":k8s_test_client_image", - repo_tags = ["k8s-test-client:pinned"], + image = ":k8s_proxy_test_tools_image", + repo_tags = ["k8s-proxy-test-tools:pinned"], ) filegroup( - name = "k8s_test_client_tarball", + name = "k8s_proxy_test_tools_tarball", testonly = True, - srcs = [":k8s_test_client_load"], + srcs = [":k8s_proxy_test_tools_load"], output_group = "tarball", ) @@ -251,10 +222,10 @@ filegroup( py_test( name = "test_k8s_proxy_integration", + size = "medium", srcs = ["test_k8s_proxy_integration.py"], data = [ - ":k8s_test_client_tarball", - ":mock_k8s_server_tarball", + ":k8s_proxy_test_tools_tarball", ], env_inherit = ["DOCKER_HOST"], imports = ["../../../.."], diff --git a/devinfra/claude/hook_daemon/session_start/k8s_proxy_test_tools.py b/devinfra/claude/hook_daemon/session_start/k8s_proxy_test_tools.py new file mode 100644 index 0000000000..0fba983118 --- /dev/null +++ b/devinfra/claude/hook_daemon/session_start/k8s_proxy_test_tools.py @@ -0,0 +1,29 @@ +"""Dispatcher for combined k8s proxy test container image. + +Routes to mock_k8s_server or k8s_proxy_test_client based on the +K8S_PROXY_TEST_ROLE environment variable. Exists because py_image_layer +can only have one main= per binary, but we need both scripts in one +image to avoid duplicate ~112 MB interpreter layers. +""" + +import importlib +import os +import sys + +_ROLES = { + "mock_k8s_server": "devinfra.claude.hook_daemon.session_start.mock_k8s_server", + "k8s_proxy_test_client": "devinfra.claude.hook_daemon.session_start.k8s_proxy_test_client", +} + + +def main() -> None: + role = os.environ.get("K8S_PROXY_TEST_ROLE", "mock_k8s_server") + module_path = _ROLES.get(role) + if not module_path: + print(f"Unknown role: {role!r}. Valid: {sorted(_ROLES)}", file=sys.stderr) + sys.exit(1) + importlib.import_module(module_path).main() + + +if __name__ == "__main__": + main() diff --git a/devinfra/claude/hook_daemon/session_start/test_k8s_proxy_integration.py b/devinfra/claude/hook_daemon/session_start/test_k8s_proxy_integration.py index 61ea91fdd6..b4f1455190 100644 --- a/devinfra/claude/hook_daemon/session_start/test_k8s_proxy_integration.py +++ b/devinfra/claude/hook_daemon/session_start/test_k8s_proxy_integration.py @@ -12,6 +12,9 @@ Host→container networking is unreliable on Firecracker microVMs (RBE workers) due to missing iptables support, so everything runs container-to-container. + +mock_k8s and test_client share a single combined image to avoid duplicate +interpreter layers (~112 MB each from py_image_layer). See BUILD.bazel. """ import json @@ -39,12 +42,9 @@ _MOCK_K8S_PORT = 6444 _PROXY_CREDENTIALS = "proxy_user:test_jwt_token" -_MOCK_K8S_IMAGE = "mock-k8s-server:pinned" -_MOCK_K8S_TARBALL = "_main/devinfra/claude/hook_daemon/session_start/mock_k8s_server_load/tarball.tar" -_MOCK_K8S_ALIAS = "mock-k8s" - -_CLIENT_IMAGE = "k8s-test-client:pinned" -_CLIENT_TARBALL = "_main/devinfra/claude/hook_daemon/session_start/k8s_test_client_load/tarball.tar" +# Combined test tools image: mock k8s server + k8s test client in one image. +_TOOLS_IMAGE = "k8s-proxy-test-tools:pinned" +_TOOLS_TARBALL = "_main/devinfra/claude/hook_daemon/session_start/k8s_proxy_test_tools_load/tarball.tar" def _get_container_ip(container: docker.models.containers.Container, network_name: str) -> str: @@ -69,25 +69,22 @@ class MockK8sServer: @pytest.fixture -def mock_k8s_image() -> str: - load_image(_MOCK_K8S_TARBALL) - return _MOCK_K8S_IMAGE - - -@pytest.fixture -def client_image() -> str: - load_image(_CLIENT_TARBALL) - return _CLIENT_IMAGE +def tools_image() -> str: + """Load the combined test tools OCI image into Docker.""" + load_image(_TOOLS_TARBALL) + return _TOOLS_IMAGE @pytest.fixture -def mock_k8s_server(mock_k8s_image: str, proxy_net: docker.models.networks.Network) -> Generator[MockK8sServer]: +def mock_k8s_server(tools_image: str, proxy_net: docker.models.networks.Network) -> Generator[MockK8sServer]: """Run mock k8s API as a container on proxy_net.""" docker_client = docker.from_env() secrets_json = json.dumps(_FAKE_SECRETS) + # The combined image's entrypoint is the mock_k8s_server binary. + # Pass secrets and port as arguments. container = docker_client.containers.run( - mock_k8s_image, + tools_image, command=[secrets_json, str(_MOCK_K8S_PORT)], name=f"mock-k8s-{os.getpid()}", network=proxy_net.name, @@ -95,8 +92,6 @@ def mock_k8s_server(mock_k8s_image: str, proxy_net: docker.models.networks.Netwo ) try: - # Use Docker DNS alias for the k8s server URL. mitmproxy reaches it - # via container networking on proxy_net. assert proxy_net.name container_ip = _get_container_ip(container, proxy_net.name) logger.info("mock k8s API at %s:%d", container_ip, _MOCK_K8S_PORT) @@ -111,7 +106,7 @@ def test_k8s_secrets_via_egress_proxy_uds_mode( mitmproxy_proxy: MitmproxyFixture, proxy_net: docker.models.networks.Network, mock_k8s_server: MockK8sServer, - client_image: str, + tools_image: str, ) -> None: """read_k8s_secret succeeds through the egress proxy without a TCP auth proxy.""" docker_client = docker.from_env() @@ -124,11 +119,19 @@ def test_k8s_secrets_via_egress_proxy_uds_mode( ca_path.write_bytes(mitmproxy_proxy.ca_cert_pem) container_name = f"k8s-proxy-test-client-{os.getpid()}" + # Run the client script from the combined image using python -m. + # The image's PYTHONPATH includes the runfiles _main/ directory, + # so our modules are importable. container = docker_client.containers.run( - client_image, + tools_image, name=container_name, network=proxy_net.name, - environment={"PROXY_URL": proxy_url, "K8S_SERVER": mock_k8s_server.url, "CA_FILE": "/certs/ca.pem"}, + environment={ + "K8S_PROXY_TEST_ROLE": "k8s_proxy_test_client", + "PROXY_URL": proxy_url, + "K8S_SERVER": mock_k8s_server.url, + "CA_FILE": "/certs/ca.pem", + }, volumes={str(ca_path): {"bind": "/certs/ca.pem", "mode": "ro"}}, detach=True, )