Run cram tests in parallel using cramp

.github/workflows/ci.yaml

@@ -119,7 +119,7 @@ jobs:
           echo "Running cram tests without coverage"
           export AUGUR="${{ github.workspace }}/bin/augur"
         fi
-        cram tests/
+        scripts/cramp tests/
     - name: Upload coverage
       if: env.COVERAGE_FILE
       uses: actions/upload-artifact@v7

dev_env.yml

@@ -25,4 +25,5 @@ dependencies:
 
   # Tools used in tests
   - conda-forge::jq
+  - conda-forge::parallel
   - conda-forge::tsv-utils

run_tests.sh

@@ -43,7 +43,7 @@ python3 -m pytest $coverage_arg $filtered_args
 # Only run functional tests if we are not running a subset of tests for pytest.
 if [ "$partial_test" = 0 ]; then
     echo "Running functional tests with cram"
-    cram tests/
+    scripts/cramp tests/
 else
     echo "Skipping functional tests when running a subset of unit tests"
 fi

scripts/cram-log-merge

@@ -0,0 +1,41 @@
+#!/usr/bin/env perl
+# Merge output from multiple invocations of Cram into one.
+use strict;
+use warnings;
+
+$|++; # unbuffer stdout
+
+my $ok   = 0;
+my $skip = 0;
+my $fail = 0;
+my $buf  = "";
+my $kept = "";
+
+while (<<>>) {
+    if (/^# Ran (\d+) tests, (\d+) skipped, (\d+) failed[.]/) {
+        # End of a Cram run; track counts and print output.
+        $ok   += $1;
+        $skip += $2;
+        $fail += $3;
+
+        chomp $buf if $buf =~ /^[.s!]+$/;
+        print $buf;
+        $buf = "";
+    }
+    elsif (/^# Kept temporary directory:/) {
+        # Hold --keep-tmpdir output till very end.
+        $kept .= $_;
+    }
+    else {
+        # Accumulate output from a Cram run
+        $buf .= $_;
+    }
+}
+
+die if $buf; # assert empty
+
+print "\n";
+print "# Ran $ok tests, $skip skipped, $fail failed.\n";
+print $kept if $kept;
+
+exit $fail;

scripts/cramp

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+    # Separate options, if any, from paths.  If options are given, they must be
+    # delimited from paths with "--".
+    local -a cram_opts
+    local job_opt="--jobs=+0"
+
+    if [[ $# -eq 0 ]]; then
+        exit-with-help
+    fi
+    if [[ "${1:-}" == -* ]]; then
+        while [[ $# -gt 0 ]]; do
+            case "$1" in
+                --help|-h)
+                    exit-with-help;;
+
+                --version|-V)
+                    exit-with-version;;
+
+                --jobs=?*|-j?*)
+                    job_opt="$1"
+                    shift;;
+
+                --jobs|-j)
+                    job_opt="--jobs=$2"
+                    shift 2;;
+
+                --interactive|-i|--yes|-y|--no|-n)
+                    echo "cram's interactivity options (e.g. $1) are not supported under cramp" >&2
+                    exit 1;;
+
+                --)
+                    shift
+                    break;;
+
+                *)
+                    cram_opts+=("$1")
+                    shift;;
+            esac
+        done
+    fi
+
+      find "$@" -type f -name '*.t' -print0 \
+    | sort --zero-terminated \
+    | parallel --null --line-buffer --keep-order "$job_opt" -- cram "${cram_opts[@]}" \
+    | "$(dirname "$0")"/cram-log-merge
+}
+
+exit-with-help() {
+    cram --help | sed -Ee '
+        /^Usage: /c Usage: cramp [<OPTIONS> --] [<PATHS>]
+
+        /^Options:/ {
+            i Cram, in parallel.
+            i
+            i If <OPTIONS> are given, they must be separated from test <PATHS> by "--".
+            i 
+            i cramp options:
+            i \ \ -jN, --jobs=N       number of tests to run in parallel
+            i \ \ -h, --help          show this help message and exit
+            i \ \ -V, --version       show version information and exit
+            i 
+            c cram options:
+        }
+        /\s--(help|version)\s/d         # takeover help and version
+        /\s--(interactive|yes|no)\s/d   # no interactivity
+    '
+    exit
+}
+
+exit-with-version() {
+    echo "Cram, parallel (version 0)"
+    echo "Copyright 2024 Thomas Sibley <tom@zulutango.org>"
+    echo
+    cram --version
+    exit
+}
+
+main "$@"

tests/functional/tree/cram/_setup.sh

@@ -1,2 +1,7 @@
 export AUGUR="${AUGUR:-$TESTDIR/../../../../bin/augur}"
 set -o pipefail
+
+# IQ-Tree writes to the input file directory, so we need to copy the data
+# to allow running tests in parallel.
+# This also avoids the data directory being polluted with output files.
+cp -r "$TESTDIR/../data" .

tests/functional/tree/cram/iqtree-compressed-input.t

@@ -5,6 +5,6 @@ Setup
 Build a tree with excluded sites using a compressed input file.
 
   $ ${AUGUR} tree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta.xz" \
-  >  --exclude-sites "$TESTDIR/../data/excluded_sites.txt" \
+  >  --alignment "data/aligned.fasta.xz" \
+  >  --exclude-sites "data/excluded_sites.txt" \
   >  --output tree_raw.nwk &> /dev/null

tests/functional/tree/cram/iqtree-conflicting-default-args.t

@@ -7,8 +7,8 @@ Expect error message.
 
   $ ${AUGUR} tree \
   >  --method iqtree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta" \
-  >  --tree-builder-args="--threads-max 1 --msa $TESTDIR/../data/aligned.fasta" \
+  >  --alignment "data/aligned.fasta" \
+  >  --tree-builder-args="--threads-max 1 --msa data/aligned.fasta" \
   >  --output "tree_raw.nwk"
   ERROR: The following tree builder arguments conflict with hardcoded defaults. Remove these arguments and try again: --threads-max, --msa
   [1]

tests/functional/tree/cram/iqtree-extend-args.t

@@ -6,6 +6,6 @@ Build a tree, augmenting existing default arguments with custom arguments.
 
   $ ${AUGUR} tree \
   >  --method iqtree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta" \
+  >  --alignment "data/aligned.fasta" \
   >  --tree-builder-args="--polytomy" \
   >  --output tree_raw.nwk > /dev/null

tests/functional/tree/cram/iqtree-model-auto.t

@@ -5,7 +5,7 @@ Setup
 Try building a tree with IQ-TREE using its ModelTest functionality, by supplying a substitution model of "auto".
 
   $ ${AUGUR} tree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta" \
+  >  --alignment "data/aligned.fasta" \
   >  --method iqtree \
   >  --substitution-model auto \
   >  --output tree_raw.nwk \

tests/functional/tree/cram/iqtree-more-threads.t

@@ -5,7 +5,7 @@ Setup
 Try building a tree with IQ-TREE with more threads (4) than there are input sequences (3).
 
   $ ${AUGUR} tree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta" \
+  >  --alignment "data/aligned.fasta" \
   >  --method iqtree \
   >  --output tree_raw.nwk \
   >  --nthreads 4 > /dev/null

tests/functional/tree/cram/iqtree-override-args.t

@@ -7,7 +7,7 @@ Since the following custom arguments are incompatible with the default IQ-TREE a
 
   $ ${AUGUR} tree \
   >  --method iqtree \
-  >  --alignment "$TESTDIR/../data/full_aligned.fasta" \
+  >  --alignment "data/full_aligned.fasta" \
   >  --tree-builder-args="--polytomy -bb 1000 -bnni" \
   >  --override-default-args \
   >  --output tree_raw.nwk > /dev/null

tests/functional/tree/cram/iqtree-preserve-fa.t

@@ -5,10 +5,10 @@ Setup
 Build a tree with an input file that doesn't end in .fasta, and ensure it's not overwritten.
 
   $ ${AUGUR} tree \
-  >  --alignment "$TESTDIR/../data/aligned.fa" \
+  >  --alignment "data/aligned.fa" \
   >  --method iqtree \
   >  --output tree_raw.nwk \
   >  --nthreads 1 > /dev/null
 
-  $ sha256sum "$TESTDIR/../data/aligned.fa" | awk '{print $1}'
+  $ sha256sum "data/aligned.fa" | awk '{print $1}'
   169a9f5f70b94e26a2c4ab2b3180d4b463112581438515557a9797adc834863d

tests/functional/tree/cram/iqtree.t

@@ -5,7 +5,7 @@ Setup
 Try building a tree with IQ-TREE.
 
   $ ${AUGUR} tree \
-  >  --alignment "$TESTDIR/../data/aligned.fasta" \
+  >  --alignment "data/aligned.fasta" \
   >  --method iqtree \
   >  --output tree_raw.nwk \
   >  --nthreads 1 > /dev/null