action.yml

name: 'FerrFlow Benchmarks'
description: 'Run benchmarks and detect performance regressions'
author: 'FerrFlow-Org'
branding:
  icon: 'activity'
  color: 'blue'

inputs:
  type:
    description: 'Benchmark type: micro, full, or all'
    required: false
    default: 'all'
  skip-competitors:
    description: 'Skip competitor benchmarks in full mode'
    required: false
    default: 'false'
  alert-threshold:
    description: 'Regression alert threshold (e.g. 120%)'
    required: false
    default: '120%'
  comment-on-pr:
    description: 'Post benchmark results as PR comment'
    required: false
    default: 'true'
  full-regression-threshold:
    description: 'Relative threshold for full benchmark regressions (e.g. 125%)'
    required: false
    default: '125%'
  binary-size-threshold:
    description: 'Binary size growth threshold (e.g. 120%)'
    required: false
    default: '120%'
  warmup:
    description: 'Number of warmup runs before timing (hyperfine --warmup)'
    required: false
    # 2 warmups is the smallest value that still primes the OS page
    # cache and filesystem readahead reliably (one cold + one settled
    # iteration); going from 3 to 2 saves ~10% of the total bench
    # wall-time across the full matrix without changing the variance
    # of the timed runs measurably.
    default: '2'
  runs:
    description: 'Number of timed runs (hyperfine --runs)'
    required: false
    default: '10'
  definitions:
    description: 'Path to the directory of JSON fixture definitions for benchmark fixture generation'
    required: true
  verbose:
    description: 'Show full error output when a benchmark command fails validation'
    required: false
    default: 'false'
  ferrflow-token:
    description: 'GitHub token for PR comments and artifact access'
    required: true
  group-filter:
    description: 'Optional criterion bench group filter (substring). If set, only benches matching this are run. Use for matrix sharding.'
    required: false
    default: ''

outputs:
  regression-detected:
    description: 'Whether a regression was detected'
    value: ${{ steps.compare.outputs.regression }}
  benchmark-summary:
    description: 'Formatted benchmark summary (markdown) for release notes'
    value: ${{ steps.release-summary.outputs.summary }}

runs:
  using: 'composite'
  steps:
    - name: Configure git identity
      shell: bash
      run: |
        git config user.name "FerrFlow"
        git config user.email "contact@ferrflow.com"

    # ------------------------------------------------------------------
    # Micro benchmarks (criterion)
    # ------------------------------------------------------------------
    - name: Run micro benchmarks
      if: inputs.type == 'micro' || inputs.type == 'all'
      shell: bash
      # cargo bench sometimes exits non-zero after the last benchmark prints
      # (criterion teardown noise, swallowed by the stderr redirect) and its
      # bencher-format output occasionally emits orphan `bench: XYZ ns/iter`
      # lines missing the `test <name> ...` prefix. Neither is recoverable
      # for us: the benchmark-action that consumes output.txt chokes on the
      # orphan lines, and the non-zero exit was killing the job even when
      # every real benchmark completed. Tolerate both: capture stderr
      # separately, let the pipe return the bench exit code, and only fail
      # the step if NO well-formed rows got captured.
      env:
        GROUP_FILTER: ${{ inputs.group-filter }}
      run: |
        set +e
        BENCH_ARGS=(--bench ferrflow_benchmarks -- --output-format bencher)
        if [ -n "$GROUP_FILTER" ]; then
          # criterion takes the filter as a positional arg, not --filter
          BENCH_ARGS+=("$GROUP_FILTER")
        fi
        cargo bench "${BENCH_ARGS[@]}" 2> bench-stderr.log | tee raw-output.txt
        bench_exit=${PIPESTATUS[0]}
        set -e
        if [ "$bench_exit" != "0" ]; then
          echo "::warning::cargo bench exited ${bench_exit} — tolerating as long as benchmark rows were captured. Stderr tail:"
          tail -40 bench-stderr.log || true
        fi
        grep -E '^test .+ \.\.\. bench:[[:space:]]+[0-9,]+ ns/iter' raw-output.txt > output.txt || true
        echo "---"
        echo "Filtered bencher output ($(wc -l < output.txt) lines kept):"
        cat output.txt
        if [ ! -s output.txt ]; then
          echo "::error::no well-formed benchmark rows captured; cargo bench exit was ${bench_exit}"
          exit 1
        fi

    - name: Find baseline artifact from main
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'pull_request'
      id: find-baseline
      shell: bash
      run: |
        RUN_ID=$(gh api "repos/${{ github.repository }}/actions/artifacts?name=criterion-baseline&per_page=1" \
          --jq '.artifacts[0].workflow_run.id // empty')
        echo "run_id=${RUN_ID}" >> "$GITHUB_OUTPUT"
      env:
        GH_TOKEN: ${{ inputs.ferrflow-token }}
      continue-on-error: true

    - name: Download micro baseline
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'pull_request' && steps.find-baseline.outputs.run_id
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: criterion-baseline
        path: baseline/
        run-id: ${{ steps.find-baseline.outputs.run_id }}
        github-token: ${{ inputs.ferrflow-token }}
      continue-on-error: true

    - name: Create empty micro baseline if missing
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'pull_request'
      shell: bash
      run: |
        if [[ ! -f baseline/benchmark-data.json ]]; then
          mkdir -p baseline
          echo '[]' > baseline/benchmark-data.json
          echo "Created empty micro baseline (no previous data found)"
        fi

    - name: Compare and comment (PR)
      # Skip the compare+comment step entirely when the caller disables PR
      # commenting (matrix shards set comment-on-pr: false — an aggregate job
      # merges all shard outputs and runs the comparison once with proper
      # pull-requests: write permission). Keeping this step here would fail
      # on regression alerts because matrixed jobs don't have permission to
      # post PR reviews.
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'pull_request' && inputs.comment-on-pr == 'true'
      uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
      with:
        tool: cargo
        output-file-path: output.txt
        external-data-json-path: baseline/benchmark-data.json
        comment-on-alert: true
        alert-threshold: ${{ inputs.alert-threshold }}
        fail-on-alert: false
        github-token: ${{ inputs.ferrflow-token }}
        comment-always: ${{ inputs.comment-on-pr }}

    - name: Prepare micro baseline (main)
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
      with:
        tool: cargo
        output-file-path: output.txt
        external-data-json-path: baseline/benchmark-data.json
        save-data-file: true

    - name: Upload micro baseline
      if: (inputs.type == 'micro' || inputs.type == 'all') && github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: criterion-baseline
        path: baseline/benchmark-data.json
        retention-days: 90
        overwrite: true

    # ------------------------------------------------------------------
    # Full benchmarks (hyperfine)
    # ------------------------------------------------------------------
    - name: Install hyperfine
      if: inputs.type == 'full' || inputs.type == 'all'
      uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2.82.7
      with:
        tool: hyperfine

    - name: Install jq
      if: inputs.type == 'full' || inputs.type == 'all'
      shell: bash
      run: sudo apt-get install -y jq

    - name: Build project
      if: inputs.type == 'full' || inputs.type == 'all'
      shell: bash
      run: cargo build --release

    - name: Add binary to PATH
      if: inputs.type == 'full' || inputs.type == 'all'
      shell: bash
      run: echo "$PWD/target/release" >> $GITHUB_PATH

    - name: Generate benchmark fixtures
      if: inputs.type == 'full' || inputs.type == 'all'
      uses: FerrLabs/Fixtures@7c4919f7db721c94fe8a91558a86759f4b63b004 # v1.0.5
      with:
        definitions: ${{ inputs.definitions }}
        generated-dir: /tmp/bench-fixtures

    - name: Run full benchmarks
      if: inputs.type == 'full' || inputs.type == 'all'
      shell: bash
      env:
        GITHUB_TOKEN: ${{ inputs.ferrflow-token }}
        DEFINITIONS: ${{ inputs.definitions }}
        WARMUP: ${{ inputs.warmup }}
        RUNS: ${{ inputs.runs }}
        SKIP_COMPETITORS: ${{ inputs.skip-competitors }}
        VERBOSE: ${{ inputs.verbose }}
        ACTION_PATH: ${{ github.action_path }}
      run: |
        mkdir -p benchmarks/results
        ARGS=(--fixtures-dir /tmp/bench-fixtures --results-dir benchmarks/results)
        ARGS+=(--definitions-dir "$DEFINITIONS")
        ARGS+=(--warmup "$WARMUP" --runs "$RUNS")
        if [[ "$SKIP_COMPETITORS" == "true" ]]; then
          ARGS+=(--skip-competitors)
        fi
        if [[ "$VERBOSE" == "true" ]]; then
          ARGS+=(--verbose)
        fi
        bash "$ACTION_PATH/scripts/run.sh" "${ARGS[@]}" | tee benchmarks/results/summary.md

    - name: Download full baseline
      if: inputs.type == 'full' || inputs.type == 'all'
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: hyperfine-baseline
        path: benchmarks/results/
      continue-on-error: true

    - name: Check for regressions
      if: inputs.type == 'full' || inputs.type == 'all'
      id: compare
      shell: bash
      env:
        FULL_REGRESSION_THRESHOLD: ${{ inputs.full-regression-threshold }}
        BINARY_SIZE_THRESHOLD: ${{ inputs.binary-size-threshold }}
        ACTION_PATH: ${{ github.action_path }}
      run: |
        set +e
        output=$(bash "$ACTION_PATH/scripts/compare.sh" benchmarks/results/baseline.json benchmarks/results/latest.json 2>&1)
        exit_code=$?
        set -e
        echo "$output"
        if echo "$output" | grep -q "status=no-baseline"; then
          echo "regression=false" >> "$GITHUB_OUTPUT"
          echo "baseline=missing" >> "$GITHUB_OUTPUT"
        elif [[ $exit_code -eq 0 ]]; then
          echo "regression=false" >> "$GITHUB_OUTPUT"
          echo "baseline=found" >> "$GITHUB_OUTPUT"
        else
          echo "regression=true" >> "$GITHUB_OUTPUT"
          echo "baseline=found" >> "$GITHUB_OUTPUT"
        fi

    - name: Post full benchmark results as PR comment
      if: (inputs.type == 'full' || inputs.type == 'all') && github.event_name == 'pull_request' && inputs.comment-on-pr == 'true'
      uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
      with:
        github-token: ${{ inputs.ferrflow-token }}
        script: |
          const fs = require('fs');
          const summary = fs.readFileSync('benchmarks/results/summary.md', 'utf8').trim();
          if (!summary) return;

          const baselineMissing = '${{ steps.compare.outputs.baseline }}' === 'missing';
          const marker = '<!-- ferrflow-full-bench -->';
          let body = `${marker}\n## Full Benchmark Results (hyperfine)\n\n${summary}`;
          if (baselineMissing) {
            body += '\n\n> **Note:** No baseline found — regression comparison was skipped. A baseline will be saved when this is merged to main.';
          }

          const { data: comments } = await github.rest.issues.listComments({
            owner: context.repo.owner,
            repo: context.repo.repo,
            issue_number: context.issue.number,
          });

          const existing = comments.find(c => c.body.includes(marker));
          if (existing) {
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: existing.id,
              body,
            });
          } else {
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body,
            });
          }

    - name: Upload full baseline
      if: (inputs.type == 'full' || inputs.type == 'all') && github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: hyperfine-baseline
        path: benchmarks/results/latest.json
        retention-days: 90
        overwrite: true

    - name: Generate release summary
      if: (inputs.type == 'full' || inputs.type == 'all') && github.event_name == 'push' && github.ref == 'refs/heads/main'
      id: release-summary
      shell: bash
      run: |
        ARGS="benchmarks/results/latest.json"
        if [[ -f benchmarks/results/baseline.json ]]; then
          ARGS="$ARGS --with-delta benchmarks/results/baseline.json"
        fi
        bash ${{ github.action_path }}/scripts/format-release.sh $ARGS > benchmarks/results/release-summary.md
        {
          echo "summary<<BENCH_EOF"
          cat benchmarks/results/release-summary.md
          echo "BENCH_EOF"
        } >> "$GITHUB_OUTPUT"

    - name: Upload release summary
      if: (inputs.type == 'full' || inputs.type == 'all') && github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: benchmark-release-summary
        path: benchmarks/results/release-summary.md
        retention-days: 5
        overwrite: true