Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -199,12 +199,12 @@ COMPILERS = {
),
llvm.sysroot(
name = "{version}_llvm_toolchain".format(version = major_version),
label = "@x86_64_sysroot//:sysroot",
label = "@x86_64_sysroot//sysroot:sysroot",
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

huh, i thought this was strictly an short hand / alias thing

targets = ["linux-x86_64"],
),
llvm.sysroot(
name = "{version}_llvm_toolchain".format(version = major_version),
label = "@aarch64_sysroot//:sysroot",
label = "@aarch64_sysroot//sysroot:sysroot",
targets = ["linux-aarch64"],
),
use_repo(llvm, "{version}_llvm_toolchain".format(version = major_version)),
Expand Down
16 changes: 7 additions & 9 deletions MODULE.bazel.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions bazel/packaging/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@ alias(
actual = "//src/go/rpk/cmd/rpk:rpk",
)

# Sysroot libs and loader shipped alongside packaged binaries, selected per
# target arch from the sysroot repos defined in //bazel:repositories.bzl.
alias(
name = "sysroot_runtime",
actual = select({
"@platforms//cpu:x86_64": "@x86_64_sysroot//sysroot:runtime",
"@platforms//cpu:aarch64": "@aarch64_sysroot//sysroot:runtime",
}),
)

redpanda_package(
name = "redpanda_tar",
out = "redpanda.tar.gz",
Expand All @@ -16,6 +26,7 @@ redpanda_package(
redpanda_binary = "//:redpanda",
rpath_override = "$ORIGIN/../lib",
rpk_binary = ":rpk",
sysroot_runtime = ":sysroot_runtime",
)

redpanda_package(
Expand All @@ -31,6 +42,7 @@ redpanda_package(
redpanda_binary = "//:redpanda",
rpath_override = "$ORIGIN/../lib",
rpk_binary = ":rpk",
sysroot_runtime = ":sysroot_runtime",
)

# This is "bazel-dir" packaging mode used for local DT tests.
Expand All @@ -44,6 +56,7 @@ redpanda_package(
redpanda_binary = "//:redpanda",
rpath_override = "$ORIGIN/../lib",
rpk_binary = ":rpk",
sysroot_runtime = ":sysroot_runtime",
)

# This is the ID for the nonroot user in the distroless containers.
Expand Down Expand Up @@ -77,6 +90,7 @@ redpanda_deb_package(
preinst = "scripts/redpanda.preinst",
prerm = "scripts/redpanda.prerm",
redpanda_binary = "//:redpanda",
sysroot_runtime = ":sysroot_runtime",
systemd_service = "scripts/redpanda.service",
systemd_slice = "scripts/redpanda.slice",
)
Expand Down Expand Up @@ -123,6 +137,7 @@ native_package(
"//src/v/kafka/client/direct_consumer/verifier:direct_consumer_verifier",
],
install_path = "/opt/redpanda_installs/direct_consumer_verifier",
sysroot_runtime = ":sysroot_runtime",
)

script_package(
Expand All @@ -140,6 +155,7 @@ native_package(
"//src/v/kafka/client/direct_consumer/verifier:direct_consumer_verifier",
],
install_path = "/opt/redpanda_installs/direct_consumer_verifier",
sysroot_runtime = ":sysroot_runtime",
)

# NOTE: this image is currently experimental, don't use this for production.
Expand Down
62 changes: 20 additions & 42 deletions bazel/packaging/packaging.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,6 @@ A rule to create a redpanda tarball given inputs from the build system.
"""

load("@bazel_skylib//lib:collections.bzl", "collections")
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")

def _is_versioned(file, starts_with):
""" Return true if this file has a name like libfoo.so.N """
parts = file.basename.rsplit(".", 3)
if len(parts) != 3:
return False
if not parts[0].startswith(starts_with):
return False
if parts[1] != "so":
return False
for c in parts[2].elems():
if not c.isdigit():
return False
return True

def _is_versioned_so(file):
return _is_versioned(file, "lib")

def _is_dynamic_loader(file):
return _is_versioned(file, "ld")

def _patched_file(ctx, original, suffix):
"""Return a new File for patching.
Expand Down Expand Up @@ -72,15 +51,14 @@ def _prepare_package_binaries(ctx, binaries, dynamic_loader_path):

shared_libraries = []
dynamic_loader = None
cc_toolchain = find_cpp_toolchain(ctx)
if cc_toolchain.sysroot != None and ctx.attr.include_sysroot_libs:
for cc_file in cc_toolchain.all_files.to_list():
if cc_file.path.startswith(cc_toolchain.sysroot):
if _is_versioned_so(cc_file):
shared_libraries.append(cc_file)
elif _is_dynamic_loader(cc_file):
shared_libraries.append(cc_file)
dynamic_loader = cc_file
if ctx.attr.include_sysroot_libs:
if not ctx.files.sysroot_runtime:
fail("include_sysroot_libs is True but sysroot_runtime is not set on", ctx.attr.name)
loaders = [f for f in ctx.files.sysroot_runtime if f.basename.startswith("ld-linux-")]
if len(loaders) != 1:
fail("expected exactly one ld-linux loader in sysroot_runtime, got", [f.basename for f in loaders])
dynamic_loader = loaders[0]
shared_libraries.extend(ctx.files.sysroot_runtime)

ret_binaries = []
for binary in binaries:
Expand Down Expand Up @@ -343,6 +321,10 @@ redpanda_package = rule(
mandatory = True,
),
"include_sysroot_libs": attr.bool(),
"sysroot_runtime": attr.label(
allow_files = True,
doc = "Sysroot shared libraries plus the glibc dynamic loader. Shipped to install_path/lib; the loader (basename ld-linux-*) is also set as the binaries' interpreter.",
),
"rpath_override": attr.string(mandatory = False),
"install_path": attr.string(
default = "/opt/redpanda",
Expand All @@ -354,17 +336,13 @@ redpanda_package = rule(
cfg = "exec",
default = Label("//bazel/packaging:tool"),
),
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
),
"_patchelf": attr.label(
executable = True,
allow_files = True,
cfg = "exec",
default = Label("@patchelf"),
),
},
toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
)

def _prepapare_package_conent(ctx):
Expand Down Expand Up @@ -448,6 +426,10 @@ native_package = rule(
"include_sysroot_libs": attr.bool(
default = True,
),
"sysroot_runtime": attr.label(
allow_files = True,
doc = "Sysroot shared libraries plus the glibc dynamic loader. Shipped to install_path/lib; the loader (basename ld-linux-*) is also set as the binaries' interpreter.",
),
"rpath_override": attr.string(
default = "$ORIGIN/../lib",
),
Expand All @@ -462,17 +444,13 @@ native_package = rule(
cfg = "exec",
default = Label("//bazel/packaging:tool"),
),
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
),
"_patchelf": attr.label(
executable = True,
allow_files = True,
cfg = "exec",
default = Label("@patchelf"),
),
},
toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
)

def _script_package_impl(ctx):
Expand Down Expand Up @@ -737,6 +715,10 @@ redpanda_deb_package = rule(
"include_sysroot_libs": attr.bool(
default = True,
),
"sysroot_runtime": attr.label(
allow_files = True,
doc = "Sysroot shared libraries plus the glibc dynamic loader. Shipped to install_path/lib; the loader (basename ld-linux-*) is also set as the binaries' interpreter.",
),
"rpath_override": attr.string(
default = "$ORIGIN/../lib",
),
Expand Down Expand Up @@ -794,15 +776,11 @@ redpanda_deb_package = rule(
cfg = "exec",
default = Label("//bazel/packaging:tool"),
),
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
),
"_patchelf": attr.label(
executable = True,
allow_files = True,
cfg = "exec",
default = Label("@patchelf"),
),
},
toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
)
21 changes: 7 additions & 14 deletions bazel/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ This module contains the sources for all third party dependencies.
"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//bazel:sysroot.bzl", "sysroot_repository")

def data_dependency():
"""
Expand Down Expand Up @@ -197,22 +198,14 @@ def data_dependency():
url = "https://github.com/Cyan4973/xxHash/archive/bbb27a5efb85b92a0486cf361a8635715a53f6ba.tar.gz",
)

sysroot_build_file = """
filegroup(
name = "sysroot",
srcs = glob(["*/**"]),
visibility = ["//visibility:public"],
)"""
http_archive(
sysroot_repository(
name = "x86_64_sysroot",
build_file_content = sysroot_build_file,
sha256 = "282b7eb89ca45d2309217d5d2099cc087c1e7bd55f7891b9d2ddca648b6663b7",
urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-19.1.7/sysroot-ubuntu-22.04-x86_64-2025-02-24.tar.zst"],
sha256 = "0d85fc9e155e664403c1c3c40831d865796d36a91b78a2e6d8922aa6ad3f0375",
urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-22.1.0/sysroot-ubuntu-22.04-x86_64-2026-05-05.tar.zst"],
)

http_archive(
sysroot_repository(
name = "aarch64_sysroot",
build_file_content = sysroot_build_file,
sha256 = "39e3d368d57a40d36f6735dcfe3ed699c6a5962cd47c5b1f652254f077632688",
urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-19.1.7/sysroot-ubuntu-22.04-aarch64-2025-02-27.tar.zst"],
sha256 = "1afc00adf978c90ad8ffd3b729180923c27d57a7702ea23ba35c714e11d0def2",
urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-22.1.0/sysroot-ubuntu-22.04-aarch64-2026-05-05.tar.zst"],
)
47 changes: 47 additions & 0 deletions bazel/sysroot.bzl
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The error says:

system_module_map.bzl:88:18: WARNING: Sysroot @@toolchains_llvm++llvm+current_llvm_toolchain//:sysroot-components-x86_64-linux resolved to 1689 files. Consider using the `sysroot` repository rule in @toolchains_llvm//toolchain:sysroot.bzl which provides a single-file (directory) sysroot for more efficient builds.

which suggests using that existing @toolchains_llvm//toolchain:sysroot.bzl rule, but we created our own here?

That rule does exist and also seems to do that "expose a single dir" thing (for the same reason):

https://github.com/bazel-contrib/toolchains_llvm/blob/master/toolchain/sysroot.bzl

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes that's what an earlier version of this PR uses. That works fine to resolve the warning and works at build time but it's not enough for the "patching the loader" step which needs access to all the files by label again.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still looking.

Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""Repository rule that downloads a Linux sysroot tarball and exposes it as:

- `:sysroot` — single source-directory entry consumed by the cc_toolchain.
Mirrors @toolchains_llvm//toolchain:sysroot.bzl so bazel 9's merkle_cache
treats the sysroot as one input instead of ~1700 files.
- `:runtime` — versioned shared libraries plus the glibc dynamic loader.
Packaging consumes this to ship the relevant `.so`s alongside the binary
and to set the binary's INTERP.

BUILD lives in a `sysroot/` subdirectory because http_archive can't put a
build_file in a sub-path, and a top-level `srcs = ["."]` source-directory
trips Bazel's package-boundary check.
"""

_BUILD_FILE = """
filegroup(
name = "sysroot",
srcs = ["."],
visibility = ["//visibility:public"],
)

filegroup(
name = "runtime",
srcs = glob([
"lib*/ld-linux-*.so.*",
"lib/*/lib*.so.*",
"usr/lib/*/lib*.so.*",
], allow_empty = False),
visibility = ["//visibility:public"],
)
"""

def _sysroot_repository_impl(rctx):
rctx.file("sysroot/BUILD.bazel", _BUILD_FILE)
rctx.download_and_extract(
url = rctx.attr.urls,
sha256 = rctx.attr.sha256,
output = "sysroot",
)

sysroot_repository = repository_rule(
implementation = _sysroot_repository_impl,
attrs = {
"urls": attr.string_list(mandatory = True),
"sha256": attr.string(mandatory = True),
},
)
7 changes: 7 additions & 0 deletions bazel/toolchain/Dockerfile.sysroot
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ RUN mkdir -p /output/pkgs/{usr/,}lib/$(uname -m)-linux-gnu /output/pkgs/usr/incl
&& cp -r /usr/lib/gcc/*-linux-gnu/12/libgcc*.{a,so}* /output/pkgs/usr/lib/$(uname -m)-linux-gnu/ \
&& cp -r /usr/include/* /output/pkgs/usr/include/

# Rewrite absolute symlinks (e.g. libmvec.so -> /lib/<triple>/libmvec.so.1) as
# relative basename links. Bazel directory artifacts reject absolute symlinks,
# and since the cp commands above co-locate each symlink with its target file
# in the same directory, basename works.
RUN find /output/pkgs -type l -lname '/*' -exec sh -c \
'ln -sf "$(basename "$(readlink "$1")")" "$1"' _ {} \;

FROM scratch

COPY --from=pkgs /output/pkgs /
Expand Down
10 changes: 5 additions & 5 deletions bazel/toolchain/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ LLVM_VERSION="$(gh release list --repo llvm/llvm-project --exclude-drafts --excl
OUTPUT_FILE="llvm-$LLVM_VERSION-debian-11-x86_64-$(date --rfc-3339=date -u).tar.zst"
echo "Building $OUTPUT_FILE"
LLVM_VERSION=$(echo $LLVM_VERSION | cut -d. -f1)
docker build --file Dockerfile.llvm --build-arg LLVM_VERSION=$LLVM_VERSION --output type=tar,dest=- . | zstd -o "$OUTPUT_FILE"
docker build --file Dockerfile.llvm --build-arg LLVM_VERSION=$LLVM_VERSION --output type=tar,dest=- . | zstd -19 -o "$OUTPUT_FILE"
```

The compiler output will be in a tarball in the current directory, this can be uploaded to S3 or github, then bazel can pull it down as desired.
Expand All @@ -34,7 +34,7 @@ LLVM_VERSION=$(echo $LLVM_VERSION | cut -d. -f1)
docker run --privileged --rm tonistiigi/binfmt --install arm64
# verify emulator
docker run --rm --platform linux/arm64 debian:bullseye uname -a
docker buildx build --platform=linux/arm64 --build-arg LLVM_VERSION=$LLVM_VERSION --file Dockerfile.llvm --output type=tar,dest=- . | zstd -o "$OUTPUT_FILE"
docker buildx build --platform=linux/arm64 --build-arg LLVM_VERSION=$LLVM_VERSION --file Dockerfile.llvm --output type=tar,dest=- . | zstd -19 -o "$OUTPUT_FILE"
```

### Building from a specific tag
Expand All @@ -46,7 +46,7 @@ LLVM_VERSION=22
LLVM_REF="llvmorg-22.1.0"
OUTPUT_FILE="llvm-22.1.0-debian-11-x86_64-$(date --rfc-3339=date -u).tar.zst"
echo "Building $OUTPUT_FILE"
docker build --file Dockerfile.llvm --build-arg LLVM_VERSION=$LLVM_VERSION --build-arg LLVM_REF=$LLVM_REF --output type=tar,dest=- . | zstd -o "$OUTPUT_FILE"
docker build --file Dockerfile.llvm --build-arg LLVM_VERSION=$LLVM_VERSION --build-arg LLVM_REF=$LLVM_REF --output type=tar,dest=- . | zstd -19 -o "$OUTPUT_FILE"
```

`LLVM_VERSION` is still required to install the bootstrap compiler from apt.llvm.org. `LLVM_REF` can be a tag (`llvmorg-22.1.0`) or branch (`main`).
Expand All @@ -66,12 +66,12 @@ To build an `x86_64` sysroot on an `x86_64` machine the following command can be

```
OUTPUT_FILE="sysroot-ubuntu-22.04-x86_64-$(date --rfc-3339=date -u).tar.zst"
docker build --file Dockerfile.sysroot --output type=tar,dest=- . | zstd -o "$OUTPUT_FILE"
docker build --file Dockerfile.sysroot --output type=tar,dest=- . | zstd -19 -o "$OUTPUT_FILE"
```

Building for `arm64` can be done from an `x86_64` host with the following command

```
OUTPUT_FILE="sysroot-ubuntu-22.04-aarch64-$(date --rfc-3339=date -u).tar.zst"
docker buildx build --platform=linux/arm64 --file Dockerfile.sysroot --output type=tar,dest=- . | zstd -o "$OUTPUT_FILE"
docker buildx build --platform=linux/arm64 --file Dockerfile.sysroot --output type=tar,dest=- . | zstd -19 -o "$OUTPUT_FILE"
```
Loading