diff --git a/.github/workflows/check-runtime-migration.yml b/.github/workflows/check-runtime-migration.yml index 43b85b1a..35017e56 100644 --- a/.github/workflows/check-runtime-migration.yml +++ b/.github/workflows/check-runtime-migration.yml @@ -185,20 +185,56 @@ jobs: key: try-runtime-snapshot-${{ matrix.runtime.name }}- fail-on-cache-miss: true + # Resolve the external commit sha and key the WASM cache on it so subsequent + # runs skip the multi-minute out-of-tree build. + - name: Resolve external runtime ref + if: ${{ matrix.runtime.external_runtime != null }} + id: external-runtime + run: | + REPO='${{ matrix.runtime.external_runtime.repo }}' + REF='${{ matrix.runtime.external_runtime.ref }}' + SHA=$(git ls-remote "$REPO" "$REF" | awk '{print $1}' | head -n1) + if [ -z "$SHA" ]; then + echo "❌ Failed to resolve $REF in $REPO" + exit 1 + fi + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + - name: Cache external runtime build (try-runtime) + if: ${{ matrix.runtime.external_runtime != null }} + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: target/external-runtimes/${{ matrix.runtime.name }}-try-runtime + key: external-runtime-try-runtime-${{ matrix.runtime.name }}-${{ steps.external-runtime.outputs.sha }} + + - name: Build external runtime (try-runtime) + if: ${{ matrix.runtime.external_runtime != null }} + working-directory: examples + run: just build-external-runtime ${{ matrix.runtime.name }} "try-runtime" > /dev/null + - name: Run ${{ matrix.runtime.name }} Runtime Checks env: PACKAGE_NAME: ${{ matrix.runtime.package }} + RUNTIME_NAME: ${{ matrix.runtime.name }} + IS_EXTERNAL: ${{ matrix.runtime.external_runtime != null }} TRY_RUNTIME_SPEC_NAME_CHECK: ${{ matrix.runtime.try_runtime.spec_name_check}} TRY_RUNTIME_EXTRA_FLAGS: ${{ matrix.runtime.try_runtime.extra_flags }} run: | try-runtime --version - FEATURES="try-runtime" - echo "Setting features: ${FEATURES}" - cargo build --release -p "$PACKAGE_NAME" --features "$FEATURES" -q --locked - RUNTIME_BLOB_NAME=$(echo "$PACKAGE_NAME" | sed 's/-/_/g').compact.compressed.wasm - RUNTIME_BLOB_PATH="./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME" + + if [ "$IS_EXTERNAL" = "true" ]; then + # External runtime was built by the previous step; locate the WASM + # in the external checkout's target directory. + RUNTIME_BLOB_PATH="./target/external-runtimes/${RUNTIME_NAME}-try-runtime/target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME" + else + FEATURES="try-runtime" + echo "Setting features: ${FEATURES}" + cargo build --release -p "$PACKAGE_NAME" --features "$FEATURES" -q --locked + RUNTIME_BLOB_PATH="./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME" + fi + export RUST_LOG=remote-ext=debug,runtime=trace (set -x; try-runtime \ diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 313f2110..55abf9b6 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -111,6 +111,33 @@ jobs: echo "subxt-cli already installed" fi + # Resolve the Fellows commit sha and key the WASM cache on it so subsequent + # runs skip the multi-minute out-of-tree build. + - name: Resolve external runtime ref + if: ${{ matrix.runtime.external_runtime != null }} + id: external-runtime + run: | + REPO='${{ matrix.runtime.external_runtime.repo }}' + REF='${{ matrix.runtime.external_runtime.ref }}' + SHA=$(git ls-remote "$REPO" "$REF" | awk '{print $1}' | head -n1) + if [ -z "$SHA" ]; then + echo "Failed to resolve $REF in $REPO" + exit 1 + fi + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + - name: Cache external runtime build + if: ${{ matrix.runtime.external_runtime != null }} + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: target/external-runtimes/${{ matrix.runtime.name }} + key: external-runtime-${{ matrix.runtime.name }}-${{ steps.external-runtime.outputs.sha }} + + - name: Build external runtime + if: ${{ matrix.runtime.external_runtime != null }} + working-directory: examples + run: just build-external-runtime ${{ matrix.runtime.name }} > /dev/null + - name: Start services working-directory: examples env: diff --git a/.gitignore b/.gitignore index 464a25b4..0ef9b8dc 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ console-ui/playwright/.cache/ # Specs for zombienet (need to be fresh) zombienet/bulletin-westend-spec.json zombienet/bulletin-paseo-spec.json +zombienet/bulletin-polkadot-spec.json diff --git a/examples/justfile b/examples/justfile index f1063535..40ae79f7 100644 --- a/examples/justfile +++ b/examples/justfile @@ -91,12 +91,19 @@ bulletin-parachain-zombienet-start test_dir runtime="bulletin-westend-runtime": ZOMBIENET_BIN="$ZOMBIENET_BIN_DIR/zombienet" # Pick spec script + zombienet TOML based on the runtime + SPEC_ENV=() if [ "{{ runtime }}" = "bulletin-westend-runtime" ]; then SPEC_SCRIPT="create_bulletin_westend_spec.sh" ZOMBIENET_TOML="zombienet/bulletin-westend-local.toml" elif [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then SPEC_SCRIPT="create_bulletin_paseo_spec.sh" ZOMBIENET_TOML="zombienet/bulletin-paseo-local.toml" + elif [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then + SPEC_SCRIPT="create_bulletin_polkadot_spec.sh" + ZOMBIENET_TOML="zombienet/bulletin-polkadot-local.toml" + # Runtime lives out-of-tree: build (or reuse cached) WASM first. + WASM_PATH="$(just build-external-runtime bulletin-polkadot)" + SPEC_ENV+=(WASM_PATH="$WASM_PATH") else echo "❌ Unhandled runtime: {{ runtime }}" exit 1 @@ -104,7 +111,7 @@ bulletin-parachain-zombienet-start test_dir runtime="bulletin-westend-runtime": echo " Creating chain spec for {{ runtime }}..." CSB_DIR="$(cd "$ROOT_DIR" && just binaries-chain-spec-builder)" - PATH="$CSB_DIR:$PATH" "$ROOT_DIR/scripts/$SPEC_SCRIPT" + env "${SPEC_ENV[@]}" PATH="$CSB_DIR:$PATH" "$ROOT_DIR/scripts/$SPEC_SCRIPT" cat <[-], +# builds the runtime package (optionally with the given comma-separated +# `features`), and prints the path to the compressed WASM on stdout. +# +# In CI, cache `target/external-runtimes/[-]` keyed by the +# resolved Fellows commit sha (use `git ls-remote ` to resolve) +# to skip the rebuild on cache hit. Different feature sets get separate +# checkouts so caches don't collide. +build-external-runtime name features="": + #!/usr/bin/env bash + set -e + ROOT_DIR="{{ justfile_directory() }}/.." + MATRIX="$ROOT_DIR/scripts/runtimes-matrix.json" + + REPO=$(jq -er --arg n "{{ name }}" '.[] | select(.name == $n) | .external_runtime.repo' "$MATRIX") + REF=$(jq -er --arg n "{{ name }}" '.[] | select(.name == $n) | .external_runtime.ref' "$MATRIX") + SUBPATH=$(jq -er --arg n "{{ name }}" '.[] | select(.name == $n) | .external_runtime.path' "$MATRIX") + PACKAGE=$(jq -er --arg n "{{ name }}" '.[] | select(.name == $n) | .package' "$MATRIX") + + FEATURES="{{ features }}" + if [ -n "$FEATURES" ]; then + VARIANT_SUFFIX="-$(echo "$FEATURES" | tr ',' '-' | tr -cd 'a-zA-Z0-9-')" + else + VARIANT_SUFFIX="" + fi + CHECKOUT_DIR="$ROOT_DIR/target/external-runtimes/{{ name }}${VARIANT_SUFFIX}" + WASM_PATH="$CHECKOUT_DIR/target/release/wbuild/$PACKAGE/${PACKAGE//-/_}.compact.compressed.wasm" + + >&2 echo "🔧 build-external-runtime: {{ name }} from $REPO @ $REF ($SUBPATH) [features=${FEATURES:-default}]" + + mkdir -p "$(dirname "$CHECKOUT_DIR")" + if [ ! -d "$CHECKOUT_DIR/.git" ]; then + >&2 echo "📥 Cloning $REPO into $CHECKOUT_DIR..." + git clone --filter=blob:none "$REPO" "$CHECKOUT_DIR" >&2 + else + git -C "$CHECKOUT_DIR" remote set-url origin "$REPO" + fi + + >&2 echo "🔀 Fetching ref: $REF..." + git -C "$CHECKOUT_DIR" fetch --depth 1 origin "$REF" >&2 + git -C "$CHECKOUT_DIR" checkout -q FETCH_HEAD + + BUILD_ARGS=(--release -p "$PACKAGE") + if [ -n "$FEATURES" ]; then + BUILD_ARGS+=(--features "$FEATURES") + fi + + if [ ! -f "$WASM_PATH" ]; then + >&2 echo "🔨 Building $PACKAGE [features=${FEATURES:-default}]..." + (cd "$CHECKOUT_DIR/$SUBPATH" && cargo build "${BUILD_ARGS[@]}" >&2) + else + >&2 echo "♻️ Reusing cached WASM at $WASM_PATH" + fi + + if [ ! -f "$WASM_PATH" ]; then + >&2 echo "❌ WASM not produced at: $WASM_PATH" + exit 1 + fi + + echo "$WASM_PATH" + # Check if Docker is available and running _check-docker: #!/usr/bin/env bash @@ -166,6 +237,10 @@ PASEO_PEER1_ID := "12D3KooWKjTeRJH8nMcFytc7qTTCQy7JrFgiZFr7iUjd1aPEBn8v" PASEO_PEER1_PORT := "10001" PASEO_PEER2_ID := "12D3KooWM8qgmWsh9ddbdX3kqR7W8tWuh62zhsdpwfs81eSnQuaH" PASEO_PEER2_PORT := "12347" +POLKADOT_PEER1_ID := "12D3KooWGgRm1jr8y5qKQYBAvjwfufdSVTLUnzeJacFdUmR8Sqvt" +POLKADOT_PEER1_PORT := "10001" +POLKADOT_PEER2_ID := "12D3KooWMbdwf9fEURv6BAAEBXM3raYDdcWCZYfiZA4eZHaFjYPd" +POLKADOT_PEER2_PORT := "12347" # Download and install Kubo IPFS binary @@ -238,6 +313,8 @@ ipfs-kubo-start test_dir runtime: ipfs-kubo-install PEERS='[{"ID":"{{ WESTEND_PEER1_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ WESTEND_PEER1_PORT }}/ws"]},{"ID":"{{ WESTEND_PEER2_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ WESTEND_PEER2_PORT }}/ws"]}]' elif [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then PEERS='[{"ID":"{{ PASEO_PEER1_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ PASEO_PEER1_PORT }}/ws"]},{"ID":"{{ PASEO_PEER2_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ PASEO_PEER2_PORT }}/ws"]}]' + elif [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then + PEERS='[{"ID":"{{ POLKADOT_PEER1_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ POLKADOT_PEER1_PORT }}/ws"]},{"ID":"{{ POLKADOT_PEER2_ID }}","Addrs":["/ip4/127.0.0.1/tcp/{{ POLKADOT_PEER2_PORT }}/ws"]}]' else echo "❌ Unhandled runtime: {{ runtime }}" exit 1 @@ -369,6 +446,8 @@ ipfs-start test_dir runtime: _check-docker PEERS="[{\"ID\":\"{{ WESTEND_PEER1_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ WESTEND_PEER1_PORT }}/ws\"]},{\"ID\":\"{{ WESTEND_PEER2_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ WESTEND_PEER2_PORT }}/ws\"]}]" elif [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then PEERS="[{\"ID\":\"{{ PASEO_PEER1_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ PASEO_PEER1_PORT }}/ws\"]},{\"ID\":\"{{ PASEO_PEER2_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ PASEO_PEER2_PORT }}/ws\"]}]" + elif [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then + PEERS="[{\"ID\":\"{{ POLKADOT_PEER1_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ POLKADOT_PEER1_PORT }}/ws\"]},{\"ID\":\"{{ POLKADOT_PEER2_ID }}\",\"Addrs\":[\"${PEER_ADDR}/tcp/{{ POLKADOT_PEER2_PORT }}/ws\"]}]" else echo "❌ Unhandled runtime: {{ runtime }}" exit 1 @@ -433,7 +512,7 @@ setup-services test_dir runtime ipfs_mode="kubo-docker": npm-install #!/usr/bin/env bash set -e - if [ "{{ runtime }}" = "bulletin-westend-runtime" ] || [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then + if [ "{{ runtime }}" = "bulletin-westend-runtime" ] || [ "{{ runtime }}" = "bulletin-paseo-runtime" ] || [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then just bulletin-parachain-zombienet-start "{{ test_dir }}" "{{ runtime }}" else echo "❌ Unhandled runtime: {{ runtime }} specified!" @@ -530,6 +609,8 @@ run-test-authorize-and-store test_dir runtime mode="ws" ws_url="ws://127.0.0.1:1 PARACHAIN_CHAINSPEC_PATH="{{ test_dir }}/bulletin-westend-collator-2/cfg/westend-local-1010.json" elif [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then PARACHAIN_CHAINSPEC_PATH="{{ test_dir }}/bulletin-paseo-collator-2/cfg/westend-local-1501.json" + elif [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then + PARACHAIN_CHAINSPEC_PATH="{{ test_dir }}/bulletin-polkadot-collator-2/cfg/westend-local-1010.json" else echo "❌ Unhandled runtime: {{ runtime }}" exit 1 @@ -739,6 +820,8 @@ run-authorize-and-store runtime mode="ws" ipfs_mode="kubo-docker": npm-install PARACHAIN_CHAINSPEC_PATH="$TEST_DIR/bulletin-westend-collator-2/cfg/westend-local-1010.json" elif [ "{{ runtime }}" = "bulletin-paseo-runtime" ]; then PARACHAIN_CHAINSPEC_PATH="$TEST_DIR/bulletin-paseo-collator-2/cfg/westend-local-1501.json" + elif [ "{{ runtime }}" = "bulletin-polkadot-runtime" ]; then + PARACHAIN_CHAINSPEC_PATH="$TEST_DIR/bulletin-polkadot-collator-2/cfg/westend-local-1010.json" else echo "❌ Unhandled runtime: {{ runtime }}" exit 1 diff --git a/scripts/create_bulletin_polkadot_spec.sh b/scripts/create_bulletin_polkadot_spec.sh new file mode 100755 index 00000000..10ccf658 --- /dev/null +++ b/scripts/create_bulletin_polkadot_spec.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Generate the zombienet chain spec for bulletin-polkadot from a pre-built +# runtime WASM. The runtime lives in the polkadot-fellows/runtimes repo and +# is built out-of-tree (see the `build-external-runtime` justfile recipe). +# +# Env vars: +# WASM_PATH - path to bulletin_polkadot_runtime.compact.compressed.wasm + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +: "${WASM_PATH:?WASM_PATH must be set (path to bulletin_polkadot_runtime.compact.compressed.wasm)}" + +if [ ! -f "$WASM_PATH" ]; then + echo "❌ WASM not found at: $WASM_PATH" + exit 1 +fi + +cd "$ROOT_DIR" + +chain-spec-builder create \ + -p 1010 \ + -c westend \ + -i bulletin-polkadot \ + -n Bulletin \ + -t local \ + -r "$WASM_PATH" \ + named-preset local_testnet + +mv chain_spec.json ./zombienet/bulletin-polkadot-spec.json +echo "✅ Wrote ./zombienet/bulletin-polkadot-spec.json" diff --git a/scripts/runtimes-matrix.json b/scripts/runtimes-matrix.json index 443519df..9668d81f 100644 --- a/scripts/runtimes-matrix.json +++ b/scripts/runtimes-matrix.json @@ -31,5 +31,21 @@ "pallet_xcm_benchmarks::generic": "templates/xcm-bench-template.hbs", "pallet_xcm_benchmarks::fungible": "templates/xcm-bench-template.hbs" } + }, + { + "name": "bulletin-polkadot", + "package": "bulletin-polkadot-runtime", + "uris": [ + "wss://bulletin-rpc.polkadot.io" + ], + "integration_tests": true, + "try_runtime": { + "extra_flags": "--blocktime 24000 --disable-spec-version-check" + }, + "external_runtime": { + "repo": "https://github.com/x3c41a/runtimes.git", + "ref": "ndk/port-pr405", + "path": "./system-parachains/bulletin/bulletin-polkadot" + } } ] diff --git a/zombienet/bulletin-polkadot-local.toml b/zombienet/bulletin-polkadot-local.toml new file mode 100644 index 00000000..c0e0073c --- /dev/null +++ b/zombienet/bulletin-polkadot-local.toml @@ -0,0 +1,67 @@ +# To run the network, execute the following command: +# +# cd +# ./scripts/create_bulletin_polkadot_spec.sh +# POLKADOT_BINARY_PATH=~/local_bridge_testing/bin/polkadot POLKADOT_PARACHAIN_BINARY_PATH=~/local_bridge_testing/bin/polkadot-parachain zombienet -p native spawn ./zombienet/bulletin-polkadot-local.toml + +[settings] +node_spawn_timeout = 240 + +[relaychain] +default_command = "{{POLKADOT_BINARY_PATH}}" +default_args = ["-lruntime=debug,xcm=trace"] +chain = "westend-local" + +[[relaychain.nodes]] +name = "alice" +validator = true +p2p_port = 30333 +rpc_port = 9942 +balance = 2000000000000 + +[[relaychain.nodes]] +name = "bob" +validator = true +p2p_port = 30433 +rpc_port = 9943 +balance = 2000000000000 + +[[parachains]] +id = 1010 +chain_spec_path = "./zombienet/bulletin-polkadot-spec.json" +cumulus_based = true + +[[parachains.collators]] +name = "bulletin-polkadot-collator-1" +command = "{{POLKADOT_PARACHAIN_BINARY_PATH}}" +validator = true +p2p_port = 10001 +rpc_port = 10000 +args = [ + "--ipfs-server", + "--pool-kbytes 65536", + "-lparachain=info,runtime=debug,xcm=trace,sub-libp2p::bitswap=trace,runtime::transaction-storage=trace", + # Embedded relay chain args (after "--"): + # Use -:--port to exclude zombienet's auto-generated --port flag, + # then provide --listen-addr to bind only on 127.0.0.1 (avoids litep2p + # IPv6 dual-stack crash on macOS and prevents 0.0.0.0 exposure). + "--", + "--listen-addr=/ip4/127.0.0.1/tcp/10003/ws", + "-:--port", +] + +[[parachains.collators]] +name = "bulletin-polkadot-collator-2" +command = "{{POLKADOT_PARACHAIN_BINARY_PATH}}" +validator = true +p2p_port = 12347 +rpc_port = 12346 +args = [ + "--ipfs-server", + "--pool-kbytes 65536", + "-lparachain=info,runtime=debug,xcm=trace,bitswap=trace,sub-libp2p::bitswap=trace,runtime::transaction-storage=trace", + # Embedded relay chain args (after "--"): + "--", + "--listen-addr=/ip4/127.0.0.1/tcp/12349/ws", + "-:--port", +]