Merge pull request #5 from hisoka-io/fix-flaky-cache-timing-assert

 flaky-test fix

Author: 0xdenjii

.config/nextest.toml

@@ -1,41 +1,14 @@
-# nextest config for the raven workspace. Loaded by `cargo nextest run`.
-# Does NOT affect `cargo test` invocations.
-#
-# Why this exists: 109+ release-mode test binaries. `cargo test` runs
-# binaries SEQUENTIALLY (parallelism only within a binary via
-# --test-threads). nextest builds all binaries first, then runs them
-# all in parallel processes, saturating cores across binaries. On this
-# 16-core box that translates to a 2-4× wall-time reduction on full
-# workspace test runs.
-#
-# Usage:
-#   cargo nextest run                                    # all default tests
-#   cargo nextest run --release                          # release-mode (matches CI)
-#   cargo nextest run -p raven-railgun-cli               # single crate
-#   cargo nextest run --features chaos-tests             # include chaos
-#   cargo nextest run --no-fail-fast                     # don't bail on first failure
-#
-# Doc tests are NOT run by nextest; use `cargo test --doc` for those.
+# nextest config for the raven workspace; does not affect `cargo test`.
 
 [profile.default]
-# Retry once on flake (network-touching tests, port-bind races); a real
-# bug surfaces after the second pass too.
+# absorb network/port-bind flake; a real bug fails the retry too
 retries = 1
-# Surface progress as tests run, not just at the end. Helps when a
-# single binary stalls a long run.
 status-level = "pass"
 final-status-level = "fail"
-# Capture both stdout and stderr; surface only on failure.
 failure-output = "immediate"
 success-output = "never"
-
-# Slow-test detection. Tests over 60s in release mode are exceptional
-# (only chaos children + production-cell #[ignore]'d ones). Surface
-# them so the slow-tail is visible.
 slow-timeout = { period = "60s", terminate-after = 5 }
 
-# CI profile: stricter — no retries (fail loudly so flakes get fixed),
-# JSON output for parsing. Activate via `cargo nextest run --profile ci`.
 [profile.ci]
 retries = 0
 status-level = "all"
@@ -44,16 +17,8 @@ failure-output = "final"
 success-output = "never"
 slow-timeout = { period = "120s", terminate-after = 3 }
 
-# Test groups: serialize tests that fight on global resources. Each
-# group is allowed ONE test at a time across the whole run.
-#
-# - chaos-subprocess: tests that fork+SIGKILL a child binary. They share
-#   the same child-binary executable, port binds, /tmp dirs, and have
-#   long teardown. Running them concurrent triggers race conditions
-#   that the production code does not have to handle.
-# - flock-tests: tests that take an flock on a shared data_dir layout.
-#   Rare, but already-once a real source of flake.
-
+# serialize tests that contend on shared global resources (child binary,
+# port binds, /tmp dirs, flock on a shared data_dir): one at a time
 [test-groups]
 chaos-subprocess = { max-threads = 1 }
 flock-tests = { max-threads = 1 }

.dockerignore

@@ -1,11 +1,9 @@
-# Docker build context filter. Build context is the repo root
-# (the railgun adapter path-depends on ../../crates/inspire).
+# Build context is the repo root (the adapter path-depends on ../../crates/inspire).
 
-# Build outputs.
 target/
 **/target/
 
-# Secrets + local-only AI/context. NEVER ship to the daemon or image.
+# secrets + local-only AI/context; never ship to the daemon or image
 run/
 no-commit/
 .claude/
@@ -16,17 +14,14 @@ CLAUDE.*.md
 MEMORY.md
 .mcp.json
 
-# VCS internals. The submodule SOURCE under crates/inspire/ is kept;
-# only the gitlink/.git metadata is dropped.
+# drop the gitlink metadata but keep the crates/inspire source
 .git/
 **/.git
 
-# Node / wasm-pack scratch.
 **/node_modules/
 **/pkg-node/
 **/pkg-bundler/
 
-# Editor / OS cruft.
 **/.DS_Store
 **/*.swp
 **/.idea/

.github/workflows/ci.yml

@@ -73,9 +73,7 @@ jobs:
   railgun-static:
     name: railgun adapter (fmt + check + clippy)
     runs-on: ubuntu-latest
-    # fmt + cargo check + clippy land in ~5-8 min on the 2-vCPU runner
-    # because they skip codegen + link. Cap at 30 min so a hung tool
-    # surfaces fast instead of blocking the matrix.
+    # cap so a hung tool surfaces instead of blocking the matrix
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
@@ -107,26 +105,8 @@ jobs:
         run: cargo clippy --manifest-path adapters/railgun/Cargo.toml --workspace --all-targets -- -D warnings
 
   railgun-tests:
-    # Test execution is the wall-time floor of CI: the 9-crate adapter +
-    # raven-inspire submodule build, under `release`, took ~48 min on a
-    # 2-vCPU runner. Four stacked optimisations bring it under ~20 min:
-    #
-    #   1. `[profile.ci-test]` in adapters/railgun/Cargo.toml
-    #      (lto=off, codegen-units=16, opt-level=2) — saturates both
-    #      cores during codegen and drops the ~120-binary thin-LTO link
-    #      pass.
-    #   2. lld linker via RUSTFLAGS — parallel symbol resolution, ~25%
-    #      off the link phase vs the default bfd ld.
-    #   3. sccache w/ GHA-cache backend — caches individual rustc
-    #      invocations across runs. Cold = no win; warm = ~70-80% hit
-    #      on the inspire dep closure (the dominant cost).
-    #   4. Matrix split — three parallel runners by package group.
-    #      Wall time = max(shard), not sum.
-    #
-    # Each shard groups packages so the per-shard test count is roughly
-    # balanced (engine=45, cli=23, rest=40 integration-test files).
-    # `fail-fast: false` lets a single shard fail without cancelling the
-    # others, which speeds debugging.
+    # split into three balanced shards (ci-test profile + lld + sccache)
+    # to keep the adapter + inspire test build under the time budget
     name: railgun tests (${{ matrix.shard.name }})
     needs: railgun-static
     runs-on: ubuntu-latest
@@ -149,24 +129,13 @@ jobs:
               -p raven-railgun-ppoi-mirror
               -p raven-railgun-core
     env:
-      # sccache: official Mozilla wrapper. GHA backend ships with the
-      # action; `SCCACHE_GHA_ENABLED=true` is consumed by the action
-      # itself. Setting RUSTC_WRAPPER here makes cargo route every rustc
-      # invocation through sccache for the rest of the job.
       RUSTC_WRAPPER: sccache
       SCCACHE_GHA_ENABLED: "true"
-      # lld linker. `.cargo/config.toml` carries the same flag for local
-      # dev; setting it via RUSTFLAGS overrides any nested config that
-      # would re-pin the default ld. -D warnings is preserved from the
-      # workflow-level env to keep build-time lint posture identical.
+      # re-declare -D warnings since RUSTFLAGS overrides the workflow-level env
       RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
     steps:
       - name: free disk space
-        # The adapter workspace's target/ tree (10 inner crates, full
-        # test build with --all-targets) exceeds the runner's default
-        # ~14 GiB free budget. Strip pre-installed tooling that the
-        # Rust build path does not touch (CodeQL, Android SDK, .NET,
-        # Haskell, Docker pre-pulls) to free ~30 GiB.
+        # adapter target/ tree overruns the runner's default free budget
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
@@ -182,33 +151,14 @@ jobs:
       - name: install lld linker
         run: sudo apt-get update && sudo apt-get install -y lld
       - name: install sccache
-        # v0.0.10 ships sccache 0.10+ which uses the GHA cache service
-        # v2 endpoint. v0.0.6 still pinned sccache 0.8.x against the
-        # decommissioned classic /artifactcache endpoint and 400'd on
-        # every run.
+        # v0.0.10 ships sccache 0.10+ for the GHA cache v2 endpoint
         uses: mozilla-actions/sccache-action@v0.0.10
       - uses: Swatinem/rust-cache@v2
         with:
-          # Shared cache across all three matrix shards so the second
-          # and third shards reuse the first shard's saved target/ tree.
-          # Without this, each shard would rebuild raven-inspire cold.
+          # shared so shards reuse the first shard's target/ instead of rebuilding inspire cold
           shared-key: "railgun-tests"
       - uses: taiki-e/install-action@nextest
       - name: cargo nextest (ci-test profile)
-        # `ci-test` profile (see adapters/railgun/Cargo.toml): inherits
-        # release semantics for runtime perf (so the InsPIRe respond
-        # path stays under nextest's 120s slow-test ceiling) but with
-        # lto=off + codegen-units=16 for compile parallelism.
-        #
-        # `--profile ci` selects the nextest profile in
-        # adapters/railgun/.config/nextest.toml: retries=0, JSON-only
-        # failure output, 120s slow-timeout × 3 strikes, test-groups
-        # serialising chaos-child / flock / arcswap stress tests.
-        #
-        # Doc-tests skipped: the adapter workspace has zero `///`
-        # Rust doctests today; `cargo test --doc` would re-build every
-        # dep from scratch for no coverage. Re-add the step if a real
-        # doctest lands later.
         run: |
           cargo nextest run \
             --manifest-path adapters/railgun/Cargo.toml \
@@ -217,10 +167,6 @@ jobs:
             --cargo-profile ci-test \
             --profile ci
       - name: sccache stats
-        # Surface cache-hit rate so we can tell whether the warm-run
-        # speedup is materialising. First run will show ~0% hit; second
-        # run on the same branch should show >50% hit on the inspire
-        # dep closure.
         run: sccache --show-stats
         if: always()
 
@@ -269,14 +215,8 @@ jobs:
         run: cargo check --manifest-path crates/binary-fuse-filter/Cargo.toml --target wasm32-unknown-unknown
 
   sdk-tests:
-    # Fresh-clone-build gate: the TypeScript SDK at adapters/railgun/sdk
-    # depends on the wasm-pack output of raven-inspire-client-wasm
-    # (gitignored). A fresh clone must:
-    #   1. Build the wasm-pack pkg-node + pkg-bundler outputs.
-    #   2. Stay under the 500 KB gzipped bundle ceiling
-    #      (adapters/railgun/scripts/check-wasm-bundle-size.sh).
-    #   3. `pnpm install` resolve cleanly against pnpm-lock.yaml.
-    #   4. `pnpm test` pass against the freshly built pkg-node.
+    # fresh-clone gate: SDK depends on the gitignored wasm-pack output,
+    # so this rebuilds it, enforces the bundle ceiling, then runs tests
     name: sdk tests (typescript + wasm-pack)
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -14,10 +14,9 @@ target/
 no-commit/
 .claude/
 .agent/
-CLAUDE.md
+CLAUDE.*
 MEMORY.md
 tmp/
 .code-review-graph/
-
-# Cargo lock is committed for binaries + workspaces
-# (keep default: commit Cargo.lock)
+.mcp.json
+run/

adapters/railgun/.config/nextest.toml

@@ -1,41 +1,14 @@
-# nextest config for the raven workspace. Loaded by `cargo nextest run`.
-# Does NOT affect `cargo test` invocations.
-#
-# Why this exists: 109+ release-mode test binaries. `cargo test` runs
-# binaries SEQUENTIALLY (parallelism only within a binary via
-# --test-threads). nextest builds all binaries first, then runs them
-# all in parallel processes, saturating cores across binaries. On this
-# 16-core box that translates to a 2-4× wall-time reduction on full
-# workspace test runs.
-#
-# Usage:
-#   cargo nextest run                                    # all default tests
-#   cargo nextest run --release                          # release-mode (matches CI)
-#   cargo nextest run -p raven-railgun-cli               # single crate
-#   cargo nextest run --features chaos-tests             # include chaos
-#   cargo nextest run --no-fail-fast                     # don't bail on first failure
-#
-# Doc tests are NOT run by nextest; use `cargo test --doc` for those.
+# nextest config for the railgun adapter workspace; does not affect `cargo test`.
 
 [profile.default]
-# Retry once on flake (network-touching tests, port-bind races); a real
-# bug surfaces after the second pass too.
+# absorb network/port-bind flake; a real bug fails the retry too
 retries = 1
-# Surface progress as tests run, not just at the end. Helps when a
-# single binary stalls a long run.
 status-level = "pass"
 final-status-level = "fail"
-# Capture both stdout and stderr; surface only on failure.
 failure-output = "immediate"
 success-output = "never"
-
-# Slow-test detection. Tests over 60s in release mode are exceptional
-# (only chaos children + production-cell #[ignore]'d ones). Surface
-# them so the slow-tail is visible.
 slow-timeout = { period = "60s", terminate-after = 5 }
 
-# CI profile: stricter — no retries (fail loudly so flakes get fixed),
-# JSON output for parsing. Activate via `cargo nextest run --profile ci`.
 [profile.ci]
 retries = 0
 status-level = "all"
@@ -44,16 +17,8 @@ failure-output = "final"
 success-output = "never"
 slow-timeout = { period = "120s", terminate-after = 3 }
 
-# Test groups: serialize tests that fight on global resources. Each
-# group is allowed ONE test at a time across the whole run.
-#
-# - chaos-subprocess: tests that fork+SIGKILL a child binary. They share
-#   the same child-binary executable, port binds, /tmp dirs, and have
-#   long teardown. Running them concurrent triggers race conditions
-#   that the production code does not have to handle.
-# - flock-tests: tests that take an flock on a shared data_dir layout.
-#   Rare, but already-once a real source of flake.
-
+# serialize tests that contend on shared global resources (child binary,
+# port binds, /tmp dirs, flock on a shared data_dir): one at a time
 [test-groups]
 chaos-subprocess = { max-threads = 1 }
 flock-tests = { max-threads = 1 }

adapters/railgun/Cargo.toml

@@ -1,12 +1,5 @@
-# Standalone workspace. Excluded from the raven root workspace because it
-# pulls heavy transitive deps (axum + tokio + reqwest, once more crates land)
-# that would feature-unify into the wasm32 build of the core crates.
-#
-# Build everything:
-#   cargo build --manifest-path adapters/railgun/Cargo.toml --release
-#
-# Run tests:
-#   cargo test --manifest-path adapters/railgun/Cargo.toml
+# Standalone workspace, detached from the raven root so its heavy native
+# deps (axum, tokio, reqwest) don't feature-unify into the core wasm32 build.
 [workspace]
 resolver = "2"
 members = [
@@ -119,27 +112,9 @@ codegen-units = 1
 strip = "symbols"
 opt-level = 3
 
-# CI test profile: inherits release semantics (so InsPIRe respond paths
-# stay fast enough to keep per-test wall under nextest's 120s ci slow-
-# timeout) but disables link-time optimization, parallelises LLVM
-# codegen across all available cores, and skips symbol stripping.
-#
-# Why the deviation from `release`:
-#   * `codegen-units = 1` forces single-unit LLVM codegen per crate.
-#     On a 2-vCPU runner that serialises raven-inspire's large crates.
-#     `codegen-units = 16` saturates both cores during codegen.
-#   * `lto = "thin"` adds a link-time optimisation pass that runs on
-#     EVERY one of the ~120 test binaries (each links the full
-#     raven-inspire dep closure). Cumulative cost dominates the build.
-#     `lto = "off"` removes the pass entirely.
-#   * `opt-level = 2` keeps respond paths within ~1.3x of opt-level=3
-#     wall (measured locally) while compiling materially faster than
-#     opt-level=3 (LLVM skips the most-expensive inlining passes).
-#   * `debug = "line-tables-only"` keeps panic backtraces resolvable
-#     without the full DWARF cost of `debug = true`.
-#
-# Use only for test execution under nextest. The `release` profile is
-# still the source of truth for benchmarks + the production binaries.
+# Test-only profile: keep release-grade runtime perf so respond paths stay
+# under nextest's slow-timeout, but trade link/codegen optimisation for
+# compile parallelism. release stays the source of truth for benches + prod.
 [profile.ci-test]
 inherits = "release"
 lto = "off"

adapters/railgun/Dockerfile

@@ -1,7 +1,5 @@
-# Operator binary image for `raven-railgun`.
-#
-# Build context MUST be the repo root: the adapter path-depends on
-# ../../crates/inspire (a self-contained workspace submodule).
+# Operator binary image for raven-railgun. Build context MUST be the repo
+# root, since the adapter path-depends on ../../crates/inspire:
 #   docker build -t raven-railgun -f adapters/railgun/Dockerfile .
 
 FROM rust:1.85-slim-bookworm AS build
@@ -11,9 +9,8 @@ RUN apt-get update \
  && rm -rf /var/lib/apt/lists/*
 WORKDIR /build
 
-# Explicit allowlist COPY. The adapter's only external path-dep is
-# crates/inspire; nothing else from root is needed. This never pulls
-# run/ (secrets) or no-commit/ into a build layer.
+# allowlist COPY: crates/inspire is the only external path-dep, so nothing
+# else from root (run/ secrets, no-commit/) lands in a build layer
 COPY rust-toolchain.toml ./
 COPY crates/inspire ./crates/inspire
 COPY adapters/railgun ./adapters/railgun
@@ -24,8 +21,7 @@ RUN cargo build --release \
 RUN strip /build/adapters/railgun/target/release/raven-railgun || true
 
 FROM debian:bookworm-slim AS runtime
-# bookworm-slim (not distroless): operators want sh + curl for the
-# healthcheck and docker exec. tini forwards SIGTERM to the binary.
+# slim, not distroless: operators want sh + curl for healthcheck/exec; tini forwards SIGTERM
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
         ca-certificates libssl3 curl tini \
@@ -41,16 +37,11 @@ COPY --from=build /build/adapters/railgun/target/release/raven-railgun \
      /usr/local/bin/raven-railgun
 RUN strip /usr/local/bin/raven-railgun || true
 
-# /srv/raven/data instance dirs, /srv/raven/snapshots export drop-zone,
-# /srv/raven/secrets operator-mounted (config + bearer + ed25519 key).
+# data: instance dirs; snapshots: export drop-zone; secrets: operator-mounted config + bearer + key
 VOLUME ["/srv/raven/data", "/srv/raven/snapshots", "/srv/raven/secrets"]
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=5s --retries=3 --start-period=60s \
     CMD curl -f http://localhost:8080/v1/health/ready || exit 1
 USER raven:raven
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/raven-railgun"]
-
-# serve-production --config <toml> selects the multi-instance path; the
-# TOML carries the bearer token, bind, RPC config, and per-instance
-# data_dirs. Override the argv or mount path via docker run as needed.
 CMD ["serve-production", "--config", "/srv/raven/secrets/mainnet.toml"]