From 7957af4ed0175da9dec88ad130d8b42144af8277 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 10:58:33 +0100 Subject: [PATCH 1/9] image: rename 30-var-lib-build to 40-var-lib-build Move the ephemeral build partition last in the partition table, leaving room for persistent partitions before it. --- modules/image.nix | 10 +++++----- tests/integration.nix | 2 +- tests/keylime-auto-enroll.nix | 2 +- tests/keylime.nix | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/image.nix b/modules/image.nix index 4d8e3a5..b8dc652 100644 --- a/modules/image.nix +++ b/modules/image.nix @@ -42,7 +42,7 @@ }; "/var/lib/build" = { device = "/dev/mapper/var_lib_crypt"; - fsType = config.systemd.repart.partitions."30-var-lib-build".Format; + fsType = config.systemd.repart.partitions."40-var-lib-build".Format; neededForBoot = true; }; "/boot" = { @@ -177,7 +177,7 @@ Minimize = "best"; }; }; - "30-var-lib-build".repartConfig = { + "40-var-lib-build".repartConfig = { Type = "var"; Label = "var-lib-build"; # We want to start out with a very small partition in the image, and add @@ -191,8 +191,8 @@ ## Run-time configuration of systemd-repart on first boot. # Reuse settings of the repart-generated image file on first boot - systemd.repart.partitions."30-var-lib-build" = - config.image.repart.partitions."30-var-lib-build".repartConfig + systemd.repart.partitions."40-var-lib-build" = + config.image.repart.partitions."40-var-lib-build".repartConfig // { Format = "ext4"; Encrypt = "key-file"; @@ -298,7 +298,7 @@ Type = "oneshot"; RemainAfterExit = true; ExecStart = "/bin/ln -sf /dev/disk/by-partlabel/${ - config.image.repart.partitions."30-var-lib-build".repartConfig.Label + config.image.repart.partitions."40-var-lib-build".repartConfig.Label } /run/systemd/volatile-root"; }; }; diff --git a/tests/integration.nix b/tests/integration.nix index 9b6f0a1..8b8170a 100644 --- a/tests/integration.nix +++ b/tests/integration.nix @@ -9,7 +9,7 @@ nixosAndroidBuilder.unattended.enable = lib.mkForce false; # Decrease resource usage for VM tests a bit as long as we are not actually # building android as part of the test suite. - systemd.repart.partitions."30-var-lib-build".SizeMinBytes = lib.mkVMOverride "10G"; + systemd.repart.partitions."40-var-lib-build".SizeMinBytes = lib.mkVMOverride "10G"; virtualisation = lib.mkVMOverride { diskSize = 30 * 1024; memorySize = 8 * 1024; diff --git a/tests/keylime-auto-enroll.nix b/tests/keylime-auto-enroll.nix index 10e40ae..9538d9d 100644 --- a/tests/keylime-auto-enroll.nix +++ b/tests/keylime-auto-enroll.nix @@ -103,7 +103,7 @@ in memorySize = 2 * 1024; cores = 2; }; - systemd.repart.partitions."30-var-lib-build".SizeMinBytes = lib.mkVMOverride "1G"; + systemd.repart.partitions."40-var-lib-build".SizeMinBytes = lib.mkVMOverride "1G"; nixosAndroidBuilder.unattended.enable = lib.mkForce false; diff --git a/tests/keylime.nix b/tests/keylime.nix index 05071ec..85b6c4f 100644 --- a/tests/keylime.nix +++ b/tests/keylime.nix @@ -89,7 +89,7 @@ in memorySize = 2 * 1024; cores = 2; }; - systemd.repart.partitions."30-var-lib-build".SizeMinBytes = lib.mkVMOverride "1G"; + systemd.repart.partitions."40-var-lib-build".SizeMinBytes = lib.mkVMOverride "1G"; nixosAndroidBuilder.unattended.enable = lib.mkForce false; From 8b3a9b65ad47691acebf73d49d4e2f918ff19893 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 10:59:17 +0100 Subject: [PATCH 2/9] image: create all mutable partitions at boot, not build time Remove the build-time placeholder for var-lib-build. systemd-repart only formats and LUKS-encrypts during partition creation, so placeholders would be left unformatted. Let repart create all mutable partitions from scratch on first boot instead. Point link-volatile-root at the ESP (always present in the image) so repart can discover the disk. Remove dead mkfsOptions.ext4 (no ext4 partitions in image anymore). --- modules/image.nix | 49 ++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/modules/image.nix b/modules/image.nix index b8dc652..37ca471 100644 --- a/modules/image.nix +++ b/modules/image.nix @@ -129,8 +129,6 @@ "-Efragments,ztailpacking" ]; - mkfsOptions.ext4 = [ "-Eroot_owner=1000:1000" ]; - # OVMF does not work with the default repart sector size of 4096 sectorSize = 512; @@ -177,30 +175,29 @@ Minimize = "best"; }; }; - "40-var-lib-build".repartConfig = { - Type = "var"; - Label = "var-lib-build"; - # We want to start out with a very small partition in the image, and add - # the real minimum size to to systemd.repart.partitions below instead, - # in order to resize it during boot. - SizeMinBytes = "10M"; - }; + # NOTE: Mutable partitions (40-var-lib-build, etc.) are NOT defined + # here. They are created at first boot by systemd-repart (see runtime + # config below). This is intentional: repart only formats and + # LUKS-encrypts partitions during creation. Build-time placeholders + # would be left unformatted, and adjacent partitions block in-place + # growth. }; }; }; ## Run-time configuration of systemd-repart on first boot. - # Reuse settings of the repart-generated image file on first boot - systemd.repart.partitions."40-var-lib-build" = - config.image.repart.partitions."40-var-lib-build".repartConfig - // { - Format = "ext4"; - Encrypt = "key-file"; - SizeMinBytes = "250G"; - # Tell systemd-repart to re-format and re-encrypt this partition on each boot - # if run with --factory-reset, which we do by default. - FactoryReset = true; - }; + + # Ephemeral build partition — factory-reset on every boot. + systemd.repart.partitions."40-var-lib-build" = { + Type = "var"; + Label = "var-lib-build"; + Format = "ext4"; + Encrypt = "key-file"; + SizeMinBytes = "250G"; + # Tell systemd-repart to re-format and re-encrypt this partition on each boot + # if run with --factory-reset, which we do by default. + FactoryReset = true; + }; boot.initrd.luks.devices."var_lib_crypt" = { keyFile = "/etc/disk.key"; @@ -282,10 +279,10 @@ }; }; - # Link the read-only nix store to /run/systemd/volatile-root before - # systemd-repart runs. systemd-repart normally looks for the block device - # backing "/", or this path. So this enables systemd-repart to find the - # right device at boot. + # Link the ESP to /run/systemd/volatile-root before systemd-repart + # runs. Since "/" is tmpfs, repart can't discover the disk on its + # own. This symlink points it at a partition that always exists in + # the image (the ESP), so repart finds the right disk. link-volatile-root = { description = "Create volatile-root to tell systemd-repart which disk to use"; wantedBy = [ "initrd.target" ]; @@ -298,7 +295,7 @@ Type = "oneshot"; RemainAfterExit = true; ExecStart = "/bin/ln -sf /dev/disk/by-partlabel/${ - config.image.repart.partitions."40-var-lib-build".repartConfig.Label + config.image.repart.partitions."00-esp".repartConfig.Label } /run/systemd/volatile-root"; }; }; From b57bfedbb3c820bfb4d589df1b7908092175ad8a Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:14 +0100 Subject: [PATCH 3/9] image: add TPM2-bound LUKS partitions for credentials and keylime Add 31-var-lib-credentials and 32-var-lib-keylime (64M each, Encrypt=tpm2). Created by systemd-repart on first boot, persistent across reboots (no FactoryReset). Add LUKS device entries with tpm2-device=auto and filesystem mounts. Ensure systemd-repart runs before all cryptsetup units. --- modules/image.nix | 52 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/modules/image.nix b/modules/image.nix index 37ca471..69e2cf0 100644 --- a/modules/image.nix +++ b/modules/image.nix @@ -40,6 +40,16 @@ "mode=0755" ]; }; + "/var/lib/credentials" = { + device = "/dev/mapper/var_lib_credentials_crypt"; + fsType = "ext4"; + neededForBoot = true; + }; + "/var/lib/keylime" = { + device = "/dev/mapper/var_lib_keylime_crypt"; + fsType = "ext4"; + neededForBoot = true; + }; "/var/lib/build" = { device = "/dev/mapper/var_lib_crypt"; fsType = config.systemd.repart.partitions."40-var-lib-build".Format; @@ -175,18 +185,38 @@ Minimize = "best"; }; }; - # NOTE: Mutable partitions (40-var-lib-build, etc.) are NOT defined - # here. They are created at first boot by systemd-repart (see runtime - # config below). This is intentional: repart only formats and - # LUKS-encrypts partitions during creation. Build-time placeholders - # would be left unformatted, and adjacent partitions block in-place - # growth. + # NOTE: 31-var-lib-credentials, 32-var-lib-keylime and 40-var-lib-build + # are NOT defined here. They are created at first boot by + # systemd-repart (see runtime config below). This is intentional: + # repart only formats and LUKS-encrypts partitions during creation. + # Build-time placeholders would be left unformatted, and adjacent + # partitions block in-place growth. }; }; }; ## Run-time configuration of systemd-repart on first boot. + # Persistent, TPM2-bound partitions for credentials and keylime agent state. + # These don't exist in the build-time image — systemd-repart creates them + # as new partitions on first boot (which triggers formatting + TPM2 LUKS + # enrollment). On subsequent boots, repart matches them by type+label and + # leaves them untouched (no FactoryReset). + systemd.repart.partitions."31-var-lib-credentials" = { + Type = "linux-generic"; + Label = "var-lib-credentials"; + Format = "ext4"; + Encrypt = "tpm2"; + SizeMinBytes = "64M"; + }; + systemd.repart.partitions."32-var-lib-keylime" = { + Type = "linux-generic"; + Label = "var-lib-keylime"; + Format = "ext4"; + Encrypt = "tpm2"; + SizeMinBytes = "64M"; + }; + # Ephemeral build partition — factory-reset on every boot. systemd.repart.partitions."40-var-lib-build" = { Type = "var"; @@ -199,6 +229,14 @@ FactoryReset = true; }; + boot.initrd.luks.devices."var_lib_credentials_crypt" = { + device = "/dev/disk/by-partlabel/var-lib-credentials"; + crypttabExtraOpts = [ "tpm2-device=auto" ]; + }; + boot.initrd.luks.devices."var_lib_keylime_crypt" = { + device = "/dev/disk/by-partlabel/var-lib-keylime"; + crypttabExtraOpts = [ "tpm2-device=auto" ]; + }; boot.initrd.luks.devices."var_lib_crypt" = { keyFile = "/etc/disk.key"; device = "/dev/disk/by-partlabel/var-lib-build"; @@ -257,6 +295,8 @@ services = { systemd-repart = { before = [ + "systemd-cryptsetup@var_lib_credentials_crypt.service" + "systemd-cryptsetup@var_lib_keylime_crypt.service" "systemd-cryptsetup@var_lib_crypt.service" ]; after = [ "systemd-udev-settle.service" ]; From d8c2c459adebbbc14e98ec3e56ef51d80d509051 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:21 +0100 Subject: [PATCH 4/9] credential-storage: decouple from artifact-storage, use own partition Move credentials from /var/lib/artifacts/credentials to /var/lib/credentials, backed by the new TPM2-bound LUKS partition. Remove dependency on artifactStorage.enable and the bind-mount ordering on the artifacts mount. --- modules/credential-storage.nix | 25 +++++++++++-------------- tests/credential-storage.nix | 19 ++++--------------- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/modules/credential-storage.nix b/modules/credential-storage.nix index 3e9cfa7..7db5556 100644 --- a/modules/credential-storage.nix +++ b/modules/credential-storage.nix @@ -1,8 +1,8 @@ -# Store systemd credentials persistently on the artifacts disk +# Store systemd credentials on a persistent, TPM2-bound LUKS partition # -# This module creates a `credentials` subdirectory on the artifacts partition -# and bind-mounts it to /etc/credstore.encrypted/ so systemd services can -# load encrypted credentials from persistent storage. +# This module bind-mounts the credential directory to +# /run/credstore.encrypted/ so systemd services can load encrypted +# credentials from persistent storage. # # Credentials should be encrypted with `systemd-creds encrypt` using the # machine's TPM. @@ -14,11 +14,10 @@ }: let cfg = config.nixosAndroidBuilder.credentialStorage; - artifactCfg = config.nixosAndroidBuilder.artifactStorage; in { options.nixosAndroidBuilder.credentialStorage = { - enable = lib.mkEnableOption "persistent credential storage on the artifacts disk"; + enable = lib.mkEnableOption "persistent credential storage on a TPM2-bound LUKS partition"; encryptionFlags = lib.mkOption { description = "Flags to pass to systemd-creds encrypt. See man (1) systemd-creds"; @@ -31,11 +30,12 @@ in credentialDir = lib.mkOption { description = '' - Directory where credentials are stored on the artifacts disk. - This will be bind-mounted to /run/credstore.encrypted/ + Directory where credentials are stored. + This is the mount point of the dedicated credentials partition + and will be bind-mounted to /run/credstore.encrypted/ ''; type = lib.types.path; - default = "${artifactCfg.artifactDir}/credentials"; + default = "/var/lib/credentials"; }; mountPoint = lib.mkOption { @@ -48,7 +48,7 @@ in }; }; - config = lib.mkIf (cfg.enable && artifactCfg.enable) { + config = lib.mkIf cfg.enable { systemd.tmpfiles.rules = [ "d ${cfg.credentialDir} 0700 root root - -" ]; @@ -56,10 +56,7 @@ in fileSystems."${cfg.mountPoint}" = { device = cfg.credentialDir; fsType = "none"; - options = [ - "bind" - "x-systemd.requires=${artifactCfg.artifactDir}.mount" - ]; + options = [ "bind" ]; }; security.polkit.extraConfig = '' diff --git a/tests/credential-storage.nix b/tests/credential-storage.nix index 9e0fe7b..fc1cb0b 100644 --- a/tests/credential-storage.nix +++ b/tests/credential-storage.nix @@ -7,29 +7,20 @@ { imports = [ ../modules/credential-storage.nix - ../modules/artifact-storage.nix ]; - # artifact-storage references build.sourceDir; provide it without - # pulling in the full android-build-env module. - options.nixosAndroidBuilder.build.sourceDir = lib.mkOption { - type = lib.types.path; - default = "/var/lib/build/source"; - }; - config = { virtualisation.tpm.enable = true; - nixosAndroidBuilder.artifactStorage.enable = true; nixosAndroidBuilder.credentialStorage.enable = true; - # Use a tmpfs instead of a real second disk - fileSystems."/var/lib/artifacts" = lib.mkForce { + # Use a tmpfs to simulate the dedicated credentials partition + fileSystems."/var/lib/credentials" = lib.mkForce { device = "none"; fsType = "tmpfs"; options = [ "size=64m" - "mode=0755" + "mode=0700" ]; }; }; @@ -39,8 +30,6 @@ machine.start() machine.wait_for_unit("multi-user.target") - machine.succeed("mkdir -p /var/lib/artifacts/credentials") - with subtest("credential-store is available"): machine.succeed("credential-store list || true") @@ -71,7 +60,7 @@ machine.fail("echo 'bad' | credential-store add '-dash'") with subtest("encrypted file is not plaintext"): - content = machine.succeed("cat /var/lib/artifacts/credentials/file-token") + content = machine.succeed("cat /var/lib/credentials/file-token") assert "file-secret" not in content, "Credential stored in plaintext!" ''; } From 50ea1e9aa7891c9ebcf9548ad4b2b38646195a6c Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:29 +0100 Subject: [PATCH 5/9] keylime-agent: remove persist-to-boot workaround /var/lib/keylime is now a persistent LUKS partition, so the agent's AK survives reboots natively. Remove persistAgent script, keylime-persist-agent service, and restore-from-boot logic in configureAgent. --- modules/keylime-agent.nix | 49 ++------------------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/modules/keylime-agent.nix b/modules/keylime-agent.nix index fb71b03..10a78e7 100644 --- a/modules/keylime-agent.nix +++ b/modules/keylime-agent.nix @@ -32,9 +32,6 @@ let runtimeConf = "/run/keylime/agent.conf"; runtimeCaCert = "/run/keylime/ca-cert.pem"; - # Agent state (AK) persisted here across reboots. - persistDir = "/boot/keylime"; - # Defaults for the push model agent (keylime_push_model_agent). # Only settings read by the push model are included. # registrar_ip, verifier_url, and CA cert paths are placeholders — @@ -97,7 +94,6 @@ let import pwd import grp import re - import shutil import sys SRC = "/boot/attestation-server.json" @@ -154,38 +150,8 @@ let os.chmod(RUNTIME_CONF, 0o440) print(f"Configured: registrar={ip}:{port} verifier=https://{ip}:{vport}") - - # Restore persisted AK from /boot. /var/lib is ephemeral, so without - # this the agent creates a new AK on every boot and the registrar - # rejects re-registration. - PERSIST = "${persistDir}" - WORK = "/var/lib/keylime" - if os.path.isdir(PERSIST): - for name in os.listdir(PERSIST): - src = os.path.join(PERSIST, name) - dst = os.path.join(WORK, name) - if os.path.isfile(src): - shutil.copy2(src, dst) - os.chown(dst, uid, gid) - print(f"Restored agent state from {PERSIST}") - else: - print(f"No persisted state at {PERSIST}") ''; - # Persist agent AK to /boot after the file appears so it survives reboots. - # Triggered by keylime-agent-data.path (PathExists watch). - persistAgent = pkgs.writeShellScript "keylime-persist-agent" '' - set -euo pipefail - src=/var/lib/keylime/agent_data.json - dst=${persistDir} - [ -f "$src" ] || { echo "$src not found, skipping"; exit 0; } - ${pkgs.util-linux}/bin/mount -o remount,rw /boot - mkdir -p "$dst" - cp "$src" "$dst"/ - ${pkgs.util-linux}/bin/mount -o remount,ro /boot - echo "Persisted agent AK to $dst" - ''; - in { options.services.keylime-agent = { @@ -236,7 +202,8 @@ in users.groups.keylime = { }; systemd.tmpfiles.rules = [ - "d /var/lib/keylime 0750 keylime keylime -" + # /var/lib/keylime is a persistent LUKS partition; fix ownership after mount + "z /var/lib/keylime 0750 keylime keylime - -" "d /run/keylime 0750 keylime keylime -" ]; @@ -288,18 +255,6 @@ in description = "Keylime agent data available"; }; - # Persist agent AK to /boot so the UUID survives reboots. - systemd.services.keylime-persist-agent = { - description = "Persist Keylime agent AK to /boot"; - after = [ "boot.mount" ]; - wantedBy = [ "keylime-agent-data.target" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${persistAgent}"; - }; - }; - # Report TPM PCR values to the auto-enrollment server so the # daemon can enroll this agent with the full TPM policy. # Reads the agent UUID from agent_data.json (ek_hash field), From ab837ae88eeb6f3055cbfe26f9180cdb9a48c801 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:34 +0100 Subject: [PATCH 6/9] installer-vm: enable TPM for installed system The installed system needs a TPM to create TPM2-encrypted partitions at first boot. --- packages/disk-installer/installer-vm.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/disk-installer/installer-vm.nix b/packages/disk-installer/installer-vm.nix index c9c776a..ccdf680 100644 --- a/packages/disk-installer/installer-vm.nix +++ b/packages/disk-installer/installer-vm.nix @@ -42,6 +42,7 @@ in useEFIBoot = true; mountHostNixStore = false; efi.keepVariables = false; + tpm.enable = true; # NixOS overrides filesystems for VMs by default fileSystems = lib.mkForce { }; From bad57365b62f7a41e22c36f84f994b531a8d9202 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:40 +0100 Subject: [PATCH 7/9] tests: verify partition persistence across reboot Write sentinels to /var/lib/{build,keylime,credentials}, reboot, verify build is wiped while keylime and credentials survive. --- tests/integration.nix | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration.nix b/tests/integration.nix index 8b8170a..a691324 100644 --- a/tests/integration.nix +++ b/tests/integration.nix @@ -78,6 +78,33 @@ "Secure Boot: enabled (user)", stdout, "Secure Boot is NOT active") ''; + + testPersistence = '' + with subtest("Partition persistence"): + with subtest("Write sentinel files before reboot"): + machine.succeed("echo 'build-data' > /var/lib/build/sentinel") + machine.succeed("echo 'keylime-data' > /var/lib/keylime/sentinel") + machine.succeed("echo 'cred-data' > /var/lib/credentials/sentinel") + + # Verify all three are readable + machine.succeed("cat /var/lib/build/sentinel") + machine.succeed("cat /var/lib/keylime/sentinel") + machine.succeed("cat /var/lib/credentials/sentinel") + + machine.reboot() + machine.wait_for_unit("default.target") + + with subtest("/var/lib/build is ephemeral (wiped on reboot)"): + machine.fail("test -f /var/lib/build/sentinel") + + with subtest("/var/lib/keylime persists across reboot"): + output = machine.succeed("cat /var/lib/keylime/sentinel").strip() + assert output == "keylime-data", f"Expected 'keylime-data', got '{output}'" + + with subtest("/var/lib/credentials persists across reboot"): + output = machine.succeed("cat /var/lib/credentials/sentinel").strip() + assert output == "cred-data", f"Expected 'cred-data', got '{output}'" + ''; in '' import os @@ -96,6 +123,7 @@ ${testSecureBoot} ${testVerity} ${testFHSEnv} + ${testPersistence} machine.shutdown() ''; } From 589cc09c8fb67d8248aa67055d91d76c9e53e86c Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 11:00:54 +0100 Subject: [PATCH 8/9] docs: update partition layout and credential/keylime storage paths Reflect the new disk layout: build-time image contains only immutable partitions, mutable partitions are created at boot. Update credential path from /var/lib/artifacts/credentials to /var/lib/credentials. Update keylime AK persistence description. --- README.md | 4 ++-- docs/docs.md | 55 +++++++++++++++++++++++++++------------------- docs/user-guide.md | 2 +- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 3c9d0ab..6de4086 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ This repository contains a custom Linux system to build Android Open Source Project in a (mostly) ephemeral environment. Our images, based on NixOS, provide a FHS-compatible enviroment that can run upstream Androids toolchain while being flexible and relatively easy to adapt due to the NixOS module system. -We boot into memory while keeping build state that's too big for memory in an ephemeral `/var/lib/build` partition on disk. That partition will be expanded and (re-)encrypted with a ephemeral key on each boot. -While no state is persisted between boots by default, there's an option to use a second disk as "artifact storage" to store build outputs in air-gapped environments. +We boot into memory while keeping build state that's too big for memory in an ephemeral `/var/lib/build` partition on disk. That partition is encrypted with a fresh random key on each boot. +Persistent, TPM2-bound LUKS partitions store keylime agent state and systemd-encrypted credentials across reboots. A second disk can optionally be used as "artifact storage" for build outputs in air-gapped environments. See [user-guide.md](./docs/user-guide.md) for usage guidance and [docs.md](./docs/docs.md) for a more detailed description of design considerations, used components limitations, and further work. diff --git a/docs/docs.md b/docs/docs.md index 55166aa..da52da8 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -32,8 +32,8 @@ We created a modular proof‑of‑concept based on NixOS that fulfills most of t * **[`nixpkgs`](https://github.com/nixos/nixpkgs)** - the software repository that enables reproducible builds of up‑to‑date open‑source packages. * **[`qemu`](https://qemu.org)** - used to run virtual machines during interactive, as well as automated testing. Both help to decrease testing & verification cycles during development & customization. * **[`systemd`](https://systemd.io)** - orchestrates both upstream and custom components while managing credentials and persistent state. -* **[`systemd-repart`](https://www.freedesktop.org/software/systemd/man/latest/systemd-repart.html)** - prepares signable read‑only disk images for the builder and resizes and re‑encrypts the state partition at each boot. -* **[Linux Unified Key Setup (`LUKS`)](https://gitlab.com/cryptsetup/cryptsetup/blob/master/README.md)** - encrypts the state partition with an ephemerally generated key on each boot. +* **[`systemd-repart`](https://www.freedesktop.org/software/systemd/man/latest/systemd-repart.html)** - prepares signable read‑only disk images for the builder and creates encrypted partitions at boot. +* **[Linux Unified Key Setup (`LUKS`)](https://gitlab.com/cryptsetup/cryptsetup/blob/master/README.md)** - encrypts mutable partitions. The ephemeral build partition uses a random key per boot; persistent partitions for credentials and keylime state are TPM2-bound. * **[`Keylime`](https://keylime.dev/)** - TPM-based remote attestation framework. The Rust agent runs on the builder; a Python registrar and verifier run on the attestation server. * Various **build requirements** for Android, such as Python 3 and OpenJDK. The complete list is in the `packages` section of `android-build-env.nix`. @@ -54,7 +54,7 @@ This approach guarantees that the same inputs always generate the same output, m Users with `nix` installed can clone this repository, download all dependencies and build a signed disk image, ready to flash & boot on the build machine, in a few simple steps outlined in [README.md](../README.md). The resulting disk image boots on generic `x86_64` hardware with `UEFI` as well as Secure Boot, and provides an isolated build environment. -It contains scripts for secure boot enrollment, a verified filesystem, and an ephemeral, encrypted state partition that holds build artifacts that cannot fit into memory. +It contains scripts for secure boot enrollment, a verified filesystem, persistent TPM2-bound encrypted partitions for credentials and keylime agent state, and an ephemeral encrypted partition for build artifacts. [^reproducible]: *Reproducible* in functionality. The final disk images are not yet expected to be *fully* bit-by-bit reproducible. That could be done, but would require a long-tail of removing additional sources of indeterminism, such as as date & time of build. See [reproducible.nixos.org](https://reproducible.nixos.org/) @@ -68,7 +68,7 @@ Under the hood, the image itself is built by `systemd-repart`, using NixOS modul `systemd-repart` is called twice during build-time: 1. While building `system.build.intermediateImage`: - A first image is built, it contains the `store` partition, populated with our NixOS closure as well as minimal `var-lib-build` partition. + A first image is built, it contains the `store` partition, populated with our NixOS closure. `boot` and `store-verity` remain empty during this step. 2. While building `system.build.finalImage`: @@ -77,29 +77,40 @@ Under the hood, the image itself is built by `systemd-repart`, using NixOS modul 3. The image then needs to be signed with a script outside a `nix` build process (to avoid leaking keys into the world-readable `/nix/store`. No `systemd-repart` is involved in this step. Instead we use `mtools` to read the `UKI` from the image, sign it and - together with Secure Boot update bundles, write it back to `boot` inside the image. -4. Finally, `systemd-repart` is called once more during run-time, in early boot at the start of `initrd`: The minimal `var-lib-build` partition, created in the first step above, is resized and encrypted with a new random key on each boot. That -key is generated just before `systemd-repart` in our custom `generate-disk-key.service`. +4. Finally, `systemd-repart` is called once more during run-time, in early boot at the start of `initrd`: All mutable partitions are created from scratch on first boot. The ephemeral build partition is factory-reset and re-encrypted with a new random key on each boot. +The key is generated just before `systemd-repart` in our custom `generate-disk-key.service`. ### Disk Layout +The build-time image contains only immutable partitions: + | Partition | Label | Format | Mountpoint | |---------------------+----------------+------------------+------------| | **00‑esp** | `boot` | `vfat` | `/boot` | -| **10‑store‑verity** | `store-verity` | `dm-verity hash` | `n/a` | +| **10‑store‑verity** | `store-verity` | `dm-verity hash` | `n/a` | | **20‑store** | `store` | `erofs` | `/usr` | -| **30‑var‑lib-build** | `var-lib-build` | `ext4` | `/var/lib/build` | + +At first boot, `systemd-repart` creates additional mutable partitions: + +| Partition | Label | Format | Mountpoint | Lifecycle | +|--------------------------+----------------------+------------------+----------------------+-----------| +| **31‑var‑lib‑credentials** | `var-lib-credentials` | `LUKS+ext4` (TPM2) | `/var/lib/credentials` | Persistent | +| **32‑var‑lib‑keylime** | `var-lib-keylime` | `LUKS+ext4` (TPM2) | `/var/lib/keylime` | Persistent | +| **40‑var‑lib‑build** | `var-lib-build` | `LUKS+ext4` (random key) | `/var/lib/build` | Ephemeral | - **boot** – Holds the signed Unified Kernel Image (`UKI`) as an `EFI` application, as well as Secure Boot update bundles for enrollment. The partition itself is unsigned and mounted read‑only during boot. - **store-verity** – Stores the `dm‑verity` hash for the `/usr` partition. The hash is passed as `usrhash` in the kernel command line, which is signed as part of the `UKI`. -- **store** – Contains the read-only Nix store, bind‑mounted into `/nix/store` in the running system. The integrity of `/usr` is verified at runtime using `dm‑verity`. -- **var-lib-build** – A minimal, ephemeral state partition. See next section below. +- **store** – Contains the read-only Nix store, bind‑mounted into `/nix/store` in the running system. The integrity of `/usr` is verified at runtime using `dm‑verity`. +- **var-lib-credentials** – TPM2-bound LUKS partition for `systemd-creds` encrypted credentials. Created on first boot, persists across reboots. Becomes inaccessible if Secure Boot keys change (PCR 7 binding). +- **var-lib-keylime** – TPM2-bound LUKS partition for keylime agent state (Attestation Key). Created on first boot, persists across reboots. +- **var-lib-build** – Ephemeral build workspace, see next section. Notably, the root filesystem (`/`) is, along with an optional writable overlay of the Nix store, kept entirely in RAM (`tmpfs`) and therefore not present in the image. There's also no boot loader, because the `UKI` acts as an `EFI` application and is directly loaded by the hosts firmware. ### Ephemeral State Partition -The `/var/lib/build` partition is deliberately designed to be temporary and encrypted. Each time the system boots, a fresh key is generated and the partition is resized to match the current disk size. This ensures that sensitive build artifacts never persist beyond a single session, reducing the risk of leaking proprietary information or to introduce impurities between different builds. +The `/var/lib/build` partition is deliberately designed to be temporary and encrypted. Each time the system boots, a fresh key is generated and the partition is factory-reset. This ensures that sensitive build artifacts never persist beyond a single session, reducing the risk of leaking proprietary information or to introduce impurities between different builds. ### Secure Boot Support @@ -213,7 +224,7 @@ The keylime agent (`services.keylime-agent`) is enabled by default on every buil At boot, the agent reads `/boot/attestation-server.json` (written to the ESP by `configure-disk-image set-attestation-server`) to learn the registrar IP, verifier URL, and CA certificate. It then registers with the registrar and begins periodic attestation. -The agent's **Attestation Key** (AK) is persisted to `/boot/keylime/` so that re-registrations after reboot use the same key, avoiding rejection by the registrar. +The agent's **Attestation Key** (AK) is stored in `/var/lib/keylime/`, which is a persistent, TPM2-bound LUKS partition. This ensures re-registrations after reboot use the same key, avoiding rejection by the registrar. ### Server (Registrar & Verifier) @@ -291,11 +302,11 @@ Two tools are included for PCR management: ## Credential Storage {#credential-storage} -The `credential-storage.nix` module provides TPM-backed persistent storage for secrets on the target machine. It uses `systemd-creds` to encrypt credentials with the machine's TPM, bound to PCR 7 (Secure Boot policy), and stores them on the artifact storage disk. +The `credential-storage.nix` module provides TPM-backed persistent storage for secrets on the target machine. It uses `systemd-creds` to encrypt credentials with the machine's TPM, bound to PCR 7 (Secure Boot policy), and stores them on a dedicated LUKS partition that is itself TPM2-bound. A `credential-store` utility for credentials management is included. See the [user guide](user-guide.pdf) for usage. -Encrypted credentials are kept in `/var/lib/artifacts/credentials/`, which is bind-mounted to `/run/credstore.encrypted/`. This is one of the standard directories that systemd searches when a service uses `LoadCredentialEncrypted=`, so stored credentials can be consumed by systemd services without additional configuration. +Encrypted credentials are kept in `/var/lib/credentials/`, a persistent TPM2-bound LUKS partition. This directory is bind-mounted to `/run/credstore.encrypted/`, one of the standard directories that systemd searches when a service uses `LoadCredentialEncrypted=`, so stored credentials can be consumed by systemd services without additional configuration. \pagebreak @@ -354,12 +365,12 @@ Main components are: - **(3)** First run of `systemd-repart` (`system.build.intermediateImage`): - Starts from a blank disk image. - Store paths from the NixOS closure are copied into the newly `store` partition. - - `esp`, `store-verity` and `var-lib-build` are created but stay empty for the moment. + - `esp` and `store-verity` are created but stay empty for the moment. - **(4)** With a filled store partition, `dm-verity` hashes can be calculated. So we build a new `UKI`, taking kernel & initrd from the NixOS closure and add the root hash of the `dm-verity` merkle tree to the kernels command line as `usrhash`. - **(5)** Second run of `systemd-repart` (`system.build.finalImage`): - Starts from the intermediate image from step **(3)**. - - The `store` and `var-lib-build` partitions are copied as-is. + - The `store` partition is copied as-is. - `dm-verity` hashes are written to the `store-verity` partition. - The unsigned `UKI` from step **(4)** is copied into the `esp` partition. - With that being done, the image is built and contains our entire NixOS closure, including the `fhsenv`, in a `dm-verity`-checked store partition, as well as the `UKI` including `usrhash`. @@ -397,7 +408,7 @@ flowchart TB halt["Display error & halt"] generate-disk-key["(3) Generate ephemeral encryption key"] - systemd-repart["(4) Resize, Format and Encrypt state partition"] + systemd-repart["(4) Create/reset partitions (TPM2 + ephemeral)"] mount["(5) Mount read-only & state partitions"] build-android["(7) `fetch-android` & `build-android` are executed"] android-tools["Android Build Tools (`repo`, `lunch`, `ninja`, etc.)"] @@ -434,16 +445,16 @@ flowchart TB - If it is **active** and our image is booting succesfully, we trust the firmware here and continue to boot normally. - If it is in **setup** mode, we enroll certificates stored on our ESP. Setting the platform key disables setup mode automatically and reboot the machine right after. - If it is **disabled** or in any unknown mode, we halt the machine but don't power it off to keep the error message readable. -3. Before encrypting the disks, we run `generate-disk-key.service`. A simple script that reads 64 bytes from `/dev/urandom` without ever storing it on disk. All state is encrypted with - that key, so that if the host shuts down for whatever reason - including sudden power loss - the encrypted data +3. Before encrypting the disks, we run `generate-disk-key.service`. A simple script that reads 64 bytes from `/dev/urandom` without ever storing it on disk. The ephemeral build partition is encrypted with + that key, so that if the host shuts down for whatever reason - including sudden power loss - the build data ends up unusable. -4. `systemd-repart` searches for the small, empty state partition on its boot media and resizes it before using `LUKS` to - encrypt it with the ephemeral key from **(2)**. +4. `systemd-repart` creates and manages mutable partitions. On first boot, it creates all three: the TPM2-bound credentials and keylime partitions (persistent) and the ephemeral build partition (encrypted with the key from **(3)**). On subsequent boots, the persistent partitions are left untouched while the build partition is factory-reset and re-encrypted with a fresh key. 5. We proceed to mount required file systems: * A read-only `/usr` partition, containing our `/nix/store` and all software in the image, checked by `dm-verity`. * Bind-mounts for `/bin` and `/lib` to simulate a conventional, `FHS`-based Linux for the build. * An ephemeral `/` file system (`tmpfs`) - * `/var/lib/build` from the encrypted partition created in **(3)**. + * `/var/lib/build` from the ephemeral encrypted partition. + * `/var/lib/credentials` and `/var/lib/keylime` from the TPM2-bound persistent partitions. 6. With all mounts in place, we are ready to finish the boot process by switching into Stage 2 of NixOS. 7. With the system fully booted, we can start the build in various ways. In unattended mode (`nixosAndroidBuilder.unattended.enable`), a configurable sequence of steps is executed automatically. In interactive mode, the following scripts are available: * `select-branch` presents a dialog to choose from configured branches (auto-selects if only one is configured). diff --git a/docs/user-guide.md b/docs/user-guide.md index 37c56c9..ab92b98 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -481,7 +481,7 @@ $ credential-store add api-token ~/token.txt Credential names must start with a letter or digit and may only contain letters, digits, dots, hyphens, and underscores. -Stored credentials are encrypted with the machine's TPM, bound to PCR 7 (Secure Boot policy). They are persisted on the artifact storage disk at `/var/lib/artifacts/credentials/` and bind-mounted to `/run/credstore.encrypted/`, which is automatically searched by systemd when services use `LoadCredentialEncrypted=`. +Stored credentials are encrypted with the machine's TPM, bound to PCR 7 (Secure Boot policy). They are kept on a dedicated TPM2-bound LUKS partition at `/var/lib/credentials/` and bind-mounted to `/run/credstore.encrypted/`, which is automatically searched by systemd when services use `LoadCredentialEncrypted=`. The encryption parameters can be customized via `nixosAndroidBuilder.credentialStorage.encryptionFlags` - for example, to also bind to PCR 11 (the specific UKI): From 291ad5fc218f0683fa3ad867479d475e06f8f4b6 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 19 Mar 2026 12:03:51 +0100 Subject: [PATCH 9/9] image: resolve boot disk via dm-verity sysfs instead of partition label Instead of symlinking a partition label to volatile-root (which could match the wrong disk if e.g. the installer is still attached), follow the dm-verity 'usr' device to its backing store partition via sysfs, then walk up to the parent disk. The verity device is cryptographically bound to this UKI's usrhash, so it uniquely identifies the correct disk. --- modules/image.nix | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/modules/image.nix b/modules/image.nix index 69e2cf0..8f476f3 100644 --- a/modules/image.nix +++ b/modules/image.nix @@ -244,6 +244,25 @@ boot.initrd.systemd = let + # Find the disk we booted from by following the dm-verity device + # tree. The verity backing device is cryptographically bound to + # this UKI's usrhash — so it uniquely identifies our image's disk, + # even if other disks have partitions with the same labels. + findBootDisk = pkgs.writeScript "find-boot-disk" '' + #!/bin/sh + set -e + # dm-verity "usr" is set up by systemd-veritysetup-generator from + # the usrhash= kernel cmdline. Find the dm block device for "usr", + # then walk sysfs from its slave (our store partition) up to the + # parent disk. + dm_dev=$(dmsetup info -c --noheadings -o blkdevname usr) + for slave in /sys/block/"$dm_dev"/slaves/*; do + dev_name=$(basename "$slave") + disk=$(basename "$(readlink -f "/sys/class/block/$dev_name/..")") + ln -sf "/dev/$disk" /run/systemd/volatile-root + break + done + ''; waitForDisk = pkgs.writeScript "wait-for-disk" '' #!/bin/sh set -e @@ -266,6 +285,7 @@ }; # We need to list our scripts here, otherwise store paths won't be in initrd storePaths = [ + findBootDisk waitForDisk generateDiskKey ]; @@ -319,13 +339,16 @@ }; }; - # Link the ESP to /run/systemd/volatile-root before systemd-repart - # runs. Since "/" is tmpfs, repart can't discover the disk on its - # own. This symlink points it at a partition that always exists in - # the image (the ESP), so repart finds the right disk. + # Tell systemd-repart which disk to operate on. Since "/" is + # tmpfs, repart can't discover it on its own. We follow the + # dm-verity backing device (cryptographically bound to this + # image's usrhash) to find the correct disk — even if other + # disks have partitions with the same labels. link-volatile-root = { description = "Create volatile-root to tell systemd-repart which disk to use"; wantedBy = [ "initrd.target" ]; + after = [ "veritysetup.target" ]; + requires = [ "veritysetup.target" ]; before = [ "systemd-repart.service" ]; requiredBy = [ "systemd-repart.service" ]; unitConfig = { @@ -334,9 +357,7 @@ serviceConfig = { Type = "oneshot"; RemainAfterExit = true; - ExecStart = "/bin/ln -sf /dev/disk/by-partlabel/${ - config.image.repart.partitions."00-esp".repartConfig.Label - } /run/systemd/volatile-root"; + ExecStart = findBootDisk; }; }; };