WIP: make cluster validation Bazel tests, not pre-commit bins

.pre-commit-config.yaml

@@ -275,6 +275,19 @@ repos:
   # terraform modules via :validate and :lint targets with hermetic providers.
   # See MODULE.bazel tf.download(mirror={...}) for provider configuration.
 
+  # TODO: Finish or remove enforce_bazel_tests (experimental, now in x/enforce_bazel_tests/).
+  # Needs language: system to import shared modules. See x/enforce_bazel_tests/ENFORCE_BAZEL_TESTS.md.
+  # - repo: local
+  #   hooks:
+  #     - id: enforce-bazel-tests
+  #       name: enforce bazel test cache
+  #       entry: x/enforce_bazel_tests/enforce_bazel_tests.py
+  #       language: python
+  #       additional_dependencies: [pygit2]
+  #       pass_filenames: false
+  #       always_run: true
+  #       stages: [pre-commit]
+
   # NOTE: The following custom validations are in bazel-precommit:
   # - pytest-main check (ensures test files have pytest_bazel.main() entry points)
   # - terraform-version-centralization (checks terraform modules don't define provider versions)

cluster/docs/secrets.md

@@ -98,7 +98,7 @@ Pre-commit hook validates all SealedSecrets can be decrypted with tofu keypair:
 
 ```bash
 # Validation uses kubeseal --recovery-unseal (works offline, no cluster needed)
-bazel run //cluster/scripts/validate_cluster:validate_sealed_secrets
+bazel run //cluster/validation:validate_sealed_secrets
 ```
 
 ## Adding New SealedSecrets

cluster/docs/troubleshooting.md

@@ -783,7 +783,7 @@ cd -
 **Validate all SealedSecrets offline before deployment:**
 
 ```bash
-bazel run //cluster/scripts/validate_cluster:validate_sealed_secrets
+bazel run //cluster/validation:validate_sealed_secrets
 ```
 
 This uses `kubeseal --recovery-unseal` to verify each SealedSecret in the repo can be decrypted

cluster/k8s/BUILD.bazel

@@ -0,0 +1,95 @@
+# gazelle:ignore
+
+load("@rules_python//python:defs.bzl", "py_library")
+load("//devinfra/testing:defs.bzl", "py_test")
+
+filegroup(
+    name = "all_yaml",
+    srcs = glob(["**/*.yaml"]),
+    visibility = ["//cluster:__subpackages__"],
+)
+
+filegroup(
+    name = "all_kustomize_inputs",
+    srcs = glob(
+        ["**"],
+        exclude = [
+            "**/BUILD.bazel",
+            "**/__pycache__/**",
+            "**/*.pyc",
+            # Exclude test files and conftest from the kustomize inputs
+            "conftest.py",
+            "test_*.py",
+        ],
+    ),
+    visibility = ["//cluster:__subpackages__"],
+)
+
+# Shared fixtures for integration tests (auto-discovered by pytest)
+py_library(
+    name = "conftest",
+    srcs = ["conftest.py"],
+    imports = ["../.."],
+    deps = [
+        "//cluster/validation:cluster",
+        "//util/bazel:runfiles",
+        "@pypi//pytest",
+    ],
+)
+
+# ============================================================================
+# Integration tests — validate real cluster/k8s/ config
+#
+# Pure-analysis tests use :all_yaml as data deps (sandboxed).
+# Kustomize-dependent tests consume //cluster/validation:kustomize_build_results.
+# ============================================================================
+
+py_test(
+    name = "test_cluster_validation",
+    srcs = ["test_cluster_validation.py"],
+    data = [":all_yaml"],
+    imports = ["../.."],
+    deps = [
+        ":conftest",
+        "//cluster/validation:cluster",
+        "//cluster/validation:dependencies",
+        "//cluster/validation:health_checks",
+        "@pypi//pytest",
+        "@pypi//pytest_bazel",
+    ],
+)
+
+py_test(
+    name = "test_kustomize",
+    srcs = ["test_kustomize.py"],
+    data = [
+        ":all_yaml",
+        "//cluster/validation:kustomize_build_results",
+    ],
+    imports = ["../.."],
+    deps = [
+        "//cluster/validation:crd_layering",
+        "//cluster/validation:kustomize",
+        "//util/bazel:runfiles",
+        "@pypi//pydantic",
+        "@pypi//pytest",
+        "@pypi//pytest_bazel",
+    ],
+)
+
+py_test(
+    name = "test_flux_build",
+    srcs = ["test_flux_build.py"],
+    data = [
+        ":all_yaml",
+        "//cluster/validation:flux_build_results",
+    ],
+    imports = ["../.."],
+    deps = [
+        ":conftest",
+        "//cluster/validation:k8s",
+        "@pypi//pytest",
+        "@pypi//pytest_bazel",
+        "@pypi//pyyaml",
+    ],
+)

cluster/k8s/authentik/BUILD.bazel

@@ -0,0 +1,22 @@
+# gazelle:ignore
+
+load("//devinfra/testing:defs.bzl", "py_test")
+
+filegroup(
+    name = "all_yaml",
+    srcs = glob(["**/*.yaml"]),
+    visibility = ["//cluster:__subpackages__"],
+)
+
+py_test(
+    name = "test_blueprint_completeness",
+    srcs = ["test_blueprint_completeness.py"],
+    data = [":all_yaml"],
+    imports = ["../../.."],
+    deps = [
+        "//util/bazel:runfiles",
+        "@pypi//pytest",
+        "@pypi//pytest_bazel",
+        "@pypi//pyyaml",
+    ],
+)

cluster/k8s/authentik/test_blueprint_completeness.py

@@ -0,0 +1,35 @@
+"""Test: all authentik blueprint YAML files listed in configMapGenerator."""
+
+from __future__ import annotations
+
+import pytest_bazel
+import yaml
+
+from util.bazel.runfiles import get_required_path
+
+_AUTHENTIK_KUSTOMIZATION = "_main/cluster/k8s/authentik/kustomization.yaml"
+
+
+def test_authentik_blueprint_completeness() -> None:
+    authentik_kust = get_required_path(_AUTHENTIK_KUSTOMIZATION)
+    blueprints_dir = authentik_kust.parent / "blueprints"
+
+    with authentik_kust.open() as f:
+        doc = yaml.safe_load(f)
+
+    listed_files: set[str] = set()
+    for generator in doc.get("configMapGenerator", []):
+        if generator.get("name") == "authentik-sso-blueprints":
+            listed_files = {f.split("/")[-1] for f in generator.get("files", [])}
+            break
+
+    on_disk = {p.name for p in blueprints_dir.glob("*.yaml")}
+    unlisted = sorted(on_disk - listed_files)
+
+    assert not unlisted, (
+        f"Add to authentik-sso-blueprints files list: {', '.join(f'blueprints/{name}' for name in unlisted)}"
+    )
+
+
+if __name__ == "__main__":
+    pytest_bazel.main()

cluster/k8s/conftest.py

@@ -0,0 +1,39 @@
+"""Shared fixtures for cluster validation integration tests.
+
+Pure-analysis tests resolve cluster/k8s/ from runfiles (data deps).
+Genrule-dependent tests (kustomize, flux) load pre-built results directly.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from cluster.validation.cluster import ParsedCluster, parse_cluster
+from util.bazel.runfiles import get_required_path
+
+_K8S_ROOT_KUSTOMIZATION = "_main/cluster/k8s/kustomization.yaml"
+_FLUX_BUILD_RESULTS_RLOCATION = "_main/cluster/validation/flux_build_results.yaml"
+
+
+@pytest.fixture(scope="session")
+def workspace() -> Path:
+    """Repo root within runfiles (parent of cluster/k8s)."""
+    return get_required_path(_K8S_ROOT_KUSTOMIZATION).parent.parent.parent
+
+
+@pytest.fixture(scope="session")
+def k8s_dir(workspace: Path) -> Path:
+    return workspace / "cluster" / "k8s"
+
+
+@pytest.fixture(scope="session")
+def cluster(k8s_dir: Path) -> ParsedCluster:
+    return parse_cluster(k8s_dir)
+
+
+@pytest.fixture(scope="session")
+def flux_build_output() -> str:
+    """Load pre-built flux build output from genrule."""
+    return get_required_path(_FLUX_BUILD_RESULTS_RLOCATION).read_text()

cluster/k8s/test_cluster_validation.py

@@ -0,0 +1,46 @@
+"""Integration tests: validate real cluster/k8s/ config via pure analysis.
+
+Tests that parse the cluster kustomization tree and check structural invariants
+(no orphaned files, valid dependencies, health checks on controller resources).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest_bazel
+
+from cluster.validation.cluster import ParsedCluster
+from cluster.validation.dependencies import validate_dependencies
+from cluster.validation.health_checks import check_controller_health_checks
+
+
+def test_no_dependency_errors(cluster: ParsedCluster, k8s_dir: Path) -> None:
+    errors = validate_dependencies(cluster, k8s_dir)
+    assert not errors, "\n".join(errors)
+
+
+def test_controller_resources_have_health_checks(cluster: ParsedCluster, k8s_dir: Path, workspace: Path) -> None:
+    errors = check_controller_health_checks(cluster, k8s_dir, workspace)
+    assert not errors, "\n".join(errors)
+
+
+def test_no_orphaned_files(cluster: ParsedCluster, k8s_dir: Path) -> None:
+    referenced: set[Path] = set()
+    for kust in cluster.kustomize_files.values():
+        referenced.update(kust.resources)
+        referenced.update(kust.patches)
+        for resource in kust.resources:
+            if resource.is_dir():
+                referenced.add(resource / "kustomization.yaml")
+
+    orphaned = sorted(
+        yaml_file.relative_to(k8s_dir)
+        for yaml_file in cluster.all_yaml_files
+        if yaml_file.name != "kustomization.yaml" and yaml_file not in referenced
+    )
+    assert not orphaned, "Orphaned files not referenced by any kustomization:\n" + "\n".join(f"  {f}" for f in orphaned)
+
+
+if __name__ == "__main__":
+    pytest_bazel.main()

cluster/k8s/test_flux_build.py

@@ -0,0 +1,22 @@
+"""Test: flux build kustomization produces expected resources.
+
+Build failures are caught by the genrule itself (exits non-zero).
+This test validates the content of the output.
+"""
+
+from __future__ import annotations
+
+import pytest_bazel
+import yaml
+
+from cluster.validation.k8s import parse_k8s_resources
+
+
+def test_flux_build_has_expected_kinds(flux_build_output: str) -> None:
+    kinds = {r.kind for r in parse_k8s_resources(yaml.safe_load_all(flux_build_output))}
+    assert "Kustomization" in kinds, "No Flux Kustomization resources in flux build output"
+    assert "GitRepository" in kinds, "No GitRepository resource in flux build output"
+
+
+if __name__ == "__main__":
+    pytest_bazel.main()

cluster/k8s/test_kustomize.py

@@ -0,0 +1,50 @@
+"""Tests that require kustomize build output.
+
+All kustomize-dependent checks are combined in one test target so the
+genrule runs once. Individual checks are separate test functions for
+clear failure reporting.
+
+Note: kustomize build failures are caught by the genrule itself (exits non-zero),
+so there's no need for a separate "all kustomizations build" test.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from pathlib import Path
+
+import pytest
+import pytest_bazel
+from pydantic import TypeAdapter
+
+from cluster.validation.crd_layering import check_crd_layering
+from cluster.validation.kustomize import KustomizeBuildResult
+from util.bazel.runfiles import get_required_path
+
+_RESULTS = TypeAdapter(list[KustomizeBuildResult]).validate_json(
+    get_required_path("_main/cluster/validation/kustomize_build_results.json").read_bytes()
+)
+
+
+def test_no_duplicate_helmreleases() -> None:
+    """Each HelmRelease name must appear in exactly one kustomization."""
+    locations: dict[str, list[Path]] = defaultdict(list)
+    for result in _RESULTS:
+        for resource in result.resources:
+            if resource.kind == "HelmRelease":
+                locations[resource.name].append(result.kustomization_path.parent)
+
+    duplicates = {name: paths for name, paths in locations.items() if len(paths) > 1}
+    assert not duplicates, "Duplicate HelmReleases:\n" + "\n".join(
+        f"  {name}: {', '.join(str(p) for p in paths)}" for name, paths in duplicates.items()
+    )
+
+
+@pytest.mark.parametrize("result", _RESULTS, ids=lambda r: str(r.kustomization_path.parent))
+def test_no_crd_layering_violations(result: KustomizeBuildResult) -> None:
+    """HelmReleases must not be mixed with CRD instances in one kustomization."""
+    check_crd_layering(result)
+
+
+if __name__ == "__main__":
+    pytest_bazel.main()

cluster/scripts/BUILD.bazel

@@ -1,6 +1,5 @@
 # gazelle:ignore
 # Cluster utility scripts (non-validation).
-# Validators are in //cluster/scripts/validate_cluster/.
 
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")