test(cardano): switch epoch test fixtures to use OCI images

.github/workflows/epoch-tests.yml

@@ -6,52 +6,49 @@ on:
 jobs:
   epoch-test:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
         include:
-          # mainnet: seed 200, upstream 200-400
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 242, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 243, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 245, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 250, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 278, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 279, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 280, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 285, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 286, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 287, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 288, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 289, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 290, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 300, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 325, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 326, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 350, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 352, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 375, upstream_range: "200-400" }
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 400, upstream_range: "200-400" }
-          # mainnet: seed 200, upstream 200-500
-          - { network: mainnet, seed_epoch: 200, ground_truth_epoch: 500, upstream_range: "200-500" }
-          # preview: seed 500, upstream 500-700
-          - { network: preview, seed_epoch: 500, ground_truth_epoch: 550, upstream_range: "500-700" }
-          - { network: preview, seed_epoch: 500, ground_truth_epoch: 649, upstream_range: "500-700" }
-          - { network: preview, seed_epoch: 500, ground_truth_epoch: 700, upstream_range: "500-700" }
+          - { network: mainnet, epoch: 242 }
+          - { network: mainnet, epoch: 243 }
+          - { network: mainnet, epoch: 245 }
+          - { network: mainnet, epoch: 250 }
+          - { network: mainnet, epoch: 278 }
+          - { network: mainnet, epoch: 279 }
+          - { network: mainnet, epoch: 280 }
+          - { network: mainnet, epoch: 285 }
+          - { network: mainnet, epoch: 286 }
+          - { network: mainnet, epoch: 287 }
+          - { network: mainnet, epoch: 288 }
+          - { network: mainnet, epoch: 289 }
+          - { network: mainnet, epoch: 290 }
+          - { network: mainnet, epoch: 300 }
+          - { network: mainnet, epoch: 325 }
+          - { network: mainnet, epoch: 326 }
+          - { network: mainnet, epoch: 350 }
+          - { network: mainnet, epoch: 352 }
+          - { network: mainnet, epoch: 375 }
+          - { network: mainnet, epoch: 400 }
+          - { network: mainnet, epoch: 500 }
+          - { network: preview, epoch: 100 }
+          - { network: preview, epoch: 550 }
+          - { network: preview, epoch: 649 }
+          - { network: preview, epoch: 700 }
 
-    name: "${{ matrix.network }}-${{ matrix.ground_truth_epoch }}"
+    name: "${{ matrix.network }}-${{ matrix.epoch }}"
+
+    env:
+      DOLOS_FIXTURE_DIR: ${{ github.workspace }}/fixtures
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v6
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_REGION }}
-
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
@@ -66,32 +63,27 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-cargo-epoch-tests-
 
-      - name: Download seed
-        run: |
-          mkdir -p /tmp/fixtures/seeds
-          aws s3 cp "s3://${{ vars.S3_BUCKET }}/seeds/${{ matrix.network }}-${{ matrix.seed_epoch }}.tar" /tmp/seed.tar
-          tar xf /tmp/seed.tar -C /tmp/fixtures/seeds
-          rm /tmp/seed.tar
+      - name: Install oras
+        uses: oras-project/setup-oras@v1
+        with:
+          version: 1.2.0
 
-      - name: Download ground truth
+      - name: Log in to GHCR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          mkdir -p /tmp/fixtures/ground-truth
-          aws s3 cp "s3://${{ vars.S3_BUCKET }}/ground-truth/${{ matrix.network }}-${{ matrix.ground_truth_epoch }}.tar" /tmp/ground-truth.tar
-          tar xf /tmp/ground-truth.tar -C /tmp/fixtures/ground-truth
-          rm /tmp/ground-truth.tar
+          export DOCKER_CONFIG="$(mktemp -d)"
+          echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
+          echo "$GITHUB_TOKEN" | oras login ghcr.io -u "${{ github.actor }}" --password-stdin
 
-      - name: Download upstream
+      - name: Pull test fixture
         run: |
-          mkdir -p /tmp/fixtures/upstream
-          aws s3 cp "s3://${{ vars.S3_BUCKET }}/upstream/${{ matrix.network }}-${{ matrix.upstream_range }}.tar" /tmp/upstream.tar
-          tar xf /tmp/upstream.tar -C /tmp/fixtures/upstream
-          rm /tmp/upstream.tar
+          mkdir -p "$DOLOS_FIXTURE_DIR"
+          cargo xtask fixture pull --network ${{ matrix.network }} --epoch ${{ matrix.epoch }}
 
       - name: Run epoch test
-        env:
-          DOLOS_FIXTURE_DIR: /tmp/fixtures
-        run: cargo test --test epoch_pots -- --nocapture
+        run: cargo test --test epoch_pots --release -- --nocapture
 
       - name: Cleanup
         if: always()
-        run: rm -rf /tmp/fixtures
+        run: rm -rf "$DOLOS_FIXTURE_DIR"

crates/testing/src/harness/cardano.rs

@@ -230,8 +230,20 @@ impl LedgerHarness {
             genesis,
         };
 
-        // 5. Bootstrap (integrity check + drain pending work)
-        domain.bootstrap()?;
+        // 5. Seed WAL from state cursor so the integrity check accepts
+        //    seed-based runs. Mirrors CLI's seed_wal_from_state (#950).
+        if let Some(cursor) = domain.state.read_cursor()? {
+            if cursor.is_fully_defined() {
+                domain.wal.reset_to(&cursor)?;
+            }
+        }
+
+        // 6. Integrity check only. We skip the rest of `bootstrap()`
+        //    (catch_up_stores + drain_pending_work) because the harness
+        //    uses NoOp archive/index stores, and catch_up_indexes would
+        //    try to decode the synthetic `LogValue::origin()` block
+        //    produced by `reset_to` and fail.
+        domain.check_integrity()?;
 
         Ok(Self {
             domain,

tests/epoch_pots/README.md

@@ -68,3 +68,39 @@ cargo test --test epoch_pots -- --nocapture
 ```
 
 Set `EPOCH_POTS_KEEP_DIR=1` to preserve the temporary working directories for debugging.
+
+## Pulling Fixtures From GHCR
+
+Fixtures are published as OCI artifacts on the GitHub Container Registry, one
+tag per test: `ghcr.io/txpipe/dolos-fixtures:{network}-{subject_epoch}`. Each
+tag bundles the seed, ground-truth, and upstream-fragment layers a single test
+needs.
+
+```bash
+# one-time: install oras (see https://oras.land/docs/installation)
+# one-time: log in to GHCR
+echo "$GH_PAT" | oras login ghcr.io -u <your-user> --password-stdin
+
+# pull everything preview-100 needs
+cargo xtask fixture pull --network preview --epoch 100
+
+# run the test
+cargo test --test epoch_pots --release -- --nocapture
+```
+
+`cargo xtask fixture pull` extracts the three layers into
+`$DOLOS_FIXTURE_DIR/{seeds,ground-truth,upstream}/<key>/` (the same layout the
+test harness discovers). `DOLOS_FIXTURE_DIR` defaults to the `fixtures.local_dir`
+value in `xtask.toml` but can be overridden by exporting the env var.
+
+To publish a new fixture bundle: arrange the three source directories under
+`$DOLOS_FIXTURE_DIR` (seed under `seeds/<network>-<seed_epoch>/`, ground-truth
+under `ground-truth/<network>-<subject_epoch>/`, upstream fragment under
+`upstream/<network>-<start>-<end>/`), then:
+
+```bash
+cargo xtask fixture push \
+  --network <net> --epoch <subject_epoch> \
+  --seed-epoch <seed_epoch> \
+  --upstream-start <start> --upstream-end <end>
+```

xtask/Cargo.toml

@@ -23,3 +23,6 @@ postgres = "0.19"
 postgres-native-tls = "0.5"
 bech32 = { workspace = true }
 pallas = { workspace = true }
+tar = "0.4"
+zstd = "0.13"
+tempfile = "3.20.0"

xtask/src/config.rs

@@ -32,6 +32,8 @@ pub struct XtaskConfig {
     pub dbsync: DbSyncConfig,
     #[serde(default)]
     pub seeds: SeedConfig,
+    #[serde(default)]
+    pub fixtures: FixtureConfig,
 }
 
 impl Default for XtaskConfig {
@@ -41,6 +43,7 @@ impl Default for XtaskConfig {
             snapshots: SnapshotConfig::default(),
             dbsync: DbSyncConfig::default(),
             seeds: SeedConfig::default(),
+            fixtures: FixtureConfig::default(),
         }
     }
 }
@@ -95,6 +98,21 @@ impl SeedConfig {
     }
 }
 
+/// Fixture packaging / registry configuration.
+///
+/// `local_dir` is the base under which fixtures live on disk, with subdirs
+/// `seeds/`, `ground-truth/`, `upstream/`. This is also where fixture pulls
+/// extract to and where pushes read from — i.e. the path the test harness
+/// uses as `DOLOS_FIXTURE_DIR`.
+///
+/// `registry` is the OCI registry + repo reference (without a tag) used for
+/// pushing and pulling test-fixture artifacts.
+#[derive(Debug, Deserialize, Default)]
+pub struct FixtureConfig {
+    pub local_dir: Option<PathBuf>,
+    pub registry: Option<String>,
+}
+
 /// DBSync connection URLs per network.
 #[derive(Debug, Deserialize, Default)]
 pub struct DbSyncConfig {

xtask/src/fixture/mod.rs

@@ -0,0 +1,109 @@
+//! fixture subcommands.
+//!
+//! Push and pull test-fixture bundles to/from an OCI registry (GHCR by default).
+//! Each tag `{network}-{subject_epoch}` is an OCI artifact with three layers
+//! (seed, ground-truth, upstream fragment) stitched together — one pull per
+//! test.
+
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use clap::Subcommand;
+use xshell::Shell;
+
+use crate::config::XtaskConfig;
+use crate::util::resolve_path;
+
+pub mod oras;
+pub mod pack;
+pub mod pull;
+pub mod push;
+
+/// Artifact type for a test-fixture manifest.
+pub const ARTIFACT_TYPE: &str = "application/vnd.txpipe.dolos.test-fixture.v1";
+
+/// Layer media type for seed fixtures (fjall state dir).
+pub const SEED_MEDIA_TYPE: &str = "application/vnd.txpipe.dolos.fixture.seed.v1.tar+zstd";
+
+/// Layer media type for ground-truth fixtures (CSV bundle).
+pub const GROUND_TRUTH_MEDIA_TYPE: &str =
+    "application/vnd.txpipe.dolos.fixture.ground-truth.v1.tar+zstd";
+
+/// Layer media type for upstream fragments (Mithril immutable chunks).
+pub const UPSTREAM_MEDIA_TYPE: &str = "application/vnd.txpipe.dolos.fixture.upstream.v1.tar+zstd";
+
+/// Current fixture schema version. Bump if the on-disk layout changes.
+pub const SCHEMA_VERSION: &str = "1";
+
+// Manifest-level annotation keys.
+pub const ANN_SCHEMA_VERSION: &str = "io.txpipe.dolos.schema-version";
+pub const ANN_NETWORK: &str = "io.txpipe.dolos.network";
+pub const ANN_SUBJECT_EPOCH: &str = "io.txpipe.dolos.subject-epoch";
+pub const ANN_SEED_EPOCH: &str = "io.txpipe.dolos.seed-epoch";
+pub const ANN_UPSTREAM_START: &str = "io.txpipe.dolos.upstream-start";
+pub const ANN_UPSTREAM_END: &str = "io.txpipe.dolos.upstream-end";
+
+/// Layer filenames — also used as `org.opencontainers.image.title` annotations,
+/// which is how `oras pull` decides the output filename for each blob.
+pub const SEED_LAYER_FILE: &str = "seed.tar.zst";
+pub const GROUND_TRUTH_LAYER_FILE: &str = "ground-truth.tar.zst";
+pub const UPSTREAM_LAYER_FILE: &str = "upstream.tar.zst";
+
+#[derive(Debug, Subcommand)]
+pub enum FixtureCmd {
+    /// Pack and push a test-fixture bundle to the configured registry.
+    Push(push::PushArgs),
+
+    /// Pull a test-fixture bundle from the registry and extract it locally.
+    Pull(pull::PullArgs),
+}
+
+pub fn run(sh: &Shell, cmd: FixtureCmd) -> Result<()> {
+    match cmd {
+        FixtureCmd::Push(args) => push::run(sh, &args),
+        FixtureCmd::Pull(args) => pull::run(sh, &args),
+    }
+}
+
+/// Build the full registry reference `{registry}:{network}-{subject_epoch}`.
+pub fn build_ref(registry: &str, network: &str, subject_epoch: u64) -> String {
+    format!("{registry}:{network}-{subject_epoch}")
+}
+
+/// On-disk directory name for a seed fixture: `{network}-{epoch}`.
+pub fn seed_key(network: &str, epoch: u64) -> String {
+    format!("{network}-{epoch}")
+}
+
+/// On-disk directory name for a ground-truth fixture: `{network}-{epoch}`.
+pub fn ground_truth_key(network: &str, epoch: u64) -> String {
+    format!("{network}-{epoch}")
+}
+
+/// On-disk directory name for an upstream fragment: `{network}-{start}-{end}`.
+pub fn upstream_key(network: &str, start: u64, end: u64) -> String {
+    format!("{network}-{start}-{end}")
+}
+
+/// Resolve the base directory where fixtures are read from / extracted to.
+///
+/// Precedence: `$DOLOS_FIXTURE_DIR` env var (matches what the test reads) →
+/// `xtask.toml` `fixtures.local_dir` key. The env override is how CI points
+/// at a runner-local path without touching the repo's `xtask.toml`.
+pub fn resolve_local_dir(repo_root: &Path, config: &XtaskConfig) -> Result<PathBuf> {
+    if let Ok(v) = std::env::var("DOLOS_FIXTURE_DIR") {
+        if !v.is_empty() {
+            return Ok(PathBuf::from(v));
+        }
+    }
+
+    config
+        .fixtures
+        .local_dir
+        .as_ref()
+        .map(|p| resolve_path(repo_root, p))
+        .context(
+            "fixtures.local_dir not configured (set `[fixtures] local_dir` in xtask.toml \
+             or export DOLOS_FIXTURE_DIR)",
+        )
+}