diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f081fe0a8..e319ff771 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,7 +119,7 @@ jobs: echo "Running cram tests without coverage" export AUGUR="${{ github.workspace }}/bin/augur" fi - cram tests/ + scripts/cramp tests/ - name: Upload coverage if: env.COVERAGE_FILE uses: actions/upload-artifact@v7 diff --git a/dev_env.yml b/dev_env.yml index 6347fb000..41e93bf33 100644 --- a/dev_env.yml +++ b/dev_env.yml @@ -25,4 +25,5 @@ dependencies: # Tools used in tests - conda-forge::jq + - conda-forge::parallel - conda-forge::tsv-utils diff --git a/run_tests.sh b/run_tests.sh index 758b5f226..dbd270d65 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -43,7 +43,7 @@ python3 -m pytest $coverage_arg $filtered_args # Only run functional tests if we are not running a subset of tests for pytest. if [ "$partial_test" = 0 ]; then echo "Running functional tests with cram" - cram tests/ + scripts/cramp tests/ else echo "Skipping functional tests when running a subset of unit tests" fi diff --git a/scripts/cram-log-merge b/scripts/cram-log-merge new file mode 100755 index 000000000..aa89fca2e --- /dev/null +++ b/scripts/cram-log-merge @@ -0,0 +1,41 @@ +#!/usr/bin/env perl +# Merge output from multiple invocations of Cram into one. +use strict; +use warnings; + +$|++; # unbuffer stdout + +my $ok = 0; +my $skip = 0; +my $fail = 0; +my $buf = ""; +my $kept = ""; + +while (<<>>) { + if (/^# Ran (\d+) tests, (\d+) skipped, (\d+) failed[.]/) { + # End of a Cram run; track counts and print output. + $ok += $1; + $skip += $2; + $fail += $3; + + chomp $buf if $buf =~ /^[.s!]+$/; + print $buf; + $buf = ""; + } + elsif (/^# Kept temporary directory:/) { + # Hold --keep-tmpdir output till very end. + $kept .= $_; + } + else { + # Accumulate output from a Cram run + $buf .= $_; + } +} + +die if $buf; # assert empty + +print "\n"; +print "# Ran $ok tests, $skip skipped, $fail failed.\n"; +print $kept if $kept; + +exit $fail; diff --git a/scripts/cramp b/scripts/cramp new file mode 100755 index 000000000..9700ab715 --- /dev/null +++ b/scripts/cramp @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + # Separate options, if any, from paths. If options are given, they must be + # delimited from paths with "--". + local -a cram_opts + local job_opt="--jobs=+0" + + if [[ $# -eq 0 ]]; then + exit-with-help + fi + if [[ "${1:-}" == -* ]]; then + while [[ $# -gt 0 ]]; do + case "$1" in + --help|-h) + exit-with-help;; + + --version|-V) + exit-with-version;; + + --jobs=?*|-j?*) + job_opt="$1" + shift;; + + --jobs|-j) + job_opt="--jobs=$2" + shift 2;; + + --interactive|-i|--yes|-y|--no|-n) + echo "cram's interactivity options (e.g. $1) are not supported under cramp" >&2 + exit 1;; + + --) + shift + break;; + + *) + cram_opts+=("$1") + shift;; + esac + done + fi + + find "$@" -type f -name '*.t' -print0 \ + | sort --zero-terminated \ + | parallel --null --line-buffer --keep-order "$job_opt" -- cram "${cram_opts[@]}" \ + | "$(dirname "$0")"/cram-log-merge +} + +exit-with-help() { + cram --help | sed -Ee ' + /^Usage: /c Usage: cramp [ --] [] + + /^Options:/ { + i Cram, in parallel. + i + i If are given, they must be separated from test by "--". + i + i cramp options: + i \ \ -jN, --jobs=N number of tests to run in parallel + i \ \ -h, --help show this help message and exit + i \ \ -V, --version show version information and exit + i + c cram options: + } + /\s--(help|version)\s/d # takeover help and version + /\s--(interactive|yes|no)\s/d # no interactivity + ' + exit +} + +exit-with-version() { + echo "Cram, parallel (version 0)" + echo "Copyright 2024 Thomas Sibley " + echo + cram --version + exit +} + +main "$@" diff --git a/tests/functional/tree/cram/_setup.sh b/tests/functional/tree/cram/_setup.sh index 9c29b1c17..848acd14b 100644 --- a/tests/functional/tree/cram/_setup.sh +++ b/tests/functional/tree/cram/_setup.sh @@ -1,2 +1,7 @@ export AUGUR="${AUGUR:-$TESTDIR/../../../../bin/augur}" set -o pipefail + +# IQ-Tree writes to the input file directory, so we need to copy the data +# to allow running tests in parallel. +# This also avoids the data directory being polluted with output files. +cp -r "$TESTDIR/../data" . diff --git a/tests/functional/tree/cram/iqtree-compressed-input.t b/tests/functional/tree/cram/iqtree-compressed-input.t index e82e16d34..e6cf30aed 100644 --- a/tests/functional/tree/cram/iqtree-compressed-input.t +++ b/tests/functional/tree/cram/iqtree-compressed-input.t @@ -5,6 +5,6 @@ Setup Build a tree with excluded sites using a compressed input file. $ ${AUGUR} tree \ - > --alignment "$TESTDIR/../data/aligned.fasta.xz" \ - > --exclude-sites "$TESTDIR/../data/excluded_sites.txt" \ + > --alignment "data/aligned.fasta.xz" \ + > --exclude-sites "data/excluded_sites.txt" \ > --output tree_raw.nwk &> /dev/null diff --git a/tests/functional/tree/cram/iqtree-conflicting-default-args.t b/tests/functional/tree/cram/iqtree-conflicting-default-args.t index 863e1e1ae..b913d5492 100644 --- a/tests/functional/tree/cram/iqtree-conflicting-default-args.t +++ b/tests/functional/tree/cram/iqtree-conflicting-default-args.t @@ -7,8 +7,8 @@ Expect error message. $ ${AUGUR} tree \ > --method iqtree \ - > --alignment "$TESTDIR/../data/aligned.fasta" \ - > --tree-builder-args="--threads-max 1 --msa $TESTDIR/../data/aligned.fasta" \ + > --alignment "data/aligned.fasta" \ + > --tree-builder-args="--threads-max 1 --msa data/aligned.fasta" \ > --output "tree_raw.nwk" ERROR: The following tree builder arguments conflict with hardcoded defaults. Remove these arguments and try again: --threads-max, --msa [1] diff --git a/tests/functional/tree/cram/iqtree-extend-args.t b/tests/functional/tree/cram/iqtree-extend-args.t index 526ad07b4..abe1bb710 100644 --- a/tests/functional/tree/cram/iqtree-extend-args.t +++ b/tests/functional/tree/cram/iqtree-extend-args.t @@ -6,6 +6,6 @@ Build a tree, augmenting existing default arguments with custom arguments. $ ${AUGUR} tree \ > --method iqtree \ - > --alignment "$TESTDIR/../data/aligned.fasta" \ + > --alignment "data/aligned.fasta" \ > --tree-builder-args="--polytomy" \ > --output tree_raw.nwk > /dev/null diff --git a/tests/functional/tree/cram/iqtree-model-auto.t b/tests/functional/tree/cram/iqtree-model-auto.t index 5930cce8e..ec12b6156 100644 --- a/tests/functional/tree/cram/iqtree-model-auto.t +++ b/tests/functional/tree/cram/iqtree-model-auto.t @@ -5,7 +5,7 @@ Setup Try building a tree with IQ-TREE using its ModelTest functionality, by supplying a substitution model of "auto". $ ${AUGUR} tree \ - > --alignment "$TESTDIR/../data/aligned.fasta" \ + > --alignment "data/aligned.fasta" \ > --method iqtree \ > --substitution-model auto \ > --output tree_raw.nwk \ diff --git a/tests/functional/tree/cram/iqtree-more-threads.t b/tests/functional/tree/cram/iqtree-more-threads.t index 3b84cf68e..da83503f3 100644 --- a/tests/functional/tree/cram/iqtree-more-threads.t +++ b/tests/functional/tree/cram/iqtree-more-threads.t @@ -5,7 +5,7 @@ Setup Try building a tree with IQ-TREE with more threads (4) than there are input sequences (3). $ ${AUGUR} tree \ - > --alignment "$TESTDIR/../data/aligned.fasta" \ + > --alignment "data/aligned.fasta" \ > --method iqtree \ > --output tree_raw.nwk \ > --nthreads 4 > /dev/null diff --git a/tests/functional/tree/cram/iqtree-override-args.t b/tests/functional/tree/cram/iqtree-override-args.t index 7871e481d..8f4bf6133 100644 --- a/tests/functional/tree/cram/iqtree-override-args.t +++ b/tests/functional/tree/cram/iqtree-override-args.t @@ -7,7 +7,7 @@ Since the following custom arguments are incompatible with the default IQ-TREE a $ ${AUGUR} tree \ > --method iqtree \ - > --alignment "$TESTDIR/../data/full_aligned.fasta" \ + > --alignment "data/full_aligned.fasta" \ > --tree-builder-args="--polytomy -bb 1000 -bnni" \ > --override-default-args \ > --output tree_raw.nwk > /dev/null diff --git a/tests/functional/tree/cram/iqtree-preserve-fa.t b/tests/functional/tree/cram/iqtree-preserve-fa.t index b1d3ba2f3..7ccabfcff 100644 --- a/tests/functional/tree/cram/iqtree-preserve-fa.t +++ b/tests/functional/tree/cram/iqtree-preserve-fa.t @@ -5,10 +5,10 @@ Setup Build a tree with an input file that doesn't end in .fasta, and ensure it's not overwritten. $ ${AUGUR} tree \ - > --alignment "$TESTDIR/../data/aligned.fa" \ + > --alignment "data/aligned.fa" \ > --method iqtree \ > --output tree_raw.nwk \ > --nthreads 1 > /dev/null - $ sha256sum "$TESTDIR/../data/aligned.fa" | awk '{print $1}' + $ sha256sum "data/aligned.fa" | awk '{print $1}' 169a9f5f70b94e26a2c4ab2b3180d4b463112581438515557a9797adc834863d diff --git a/tests/functional/tree/cram/iqtree.t b/tests/functional/tree/cram/iqtree.t index 9b300b06b..99c890d51 100644 --- a/tests/functional/tree/cram/iqtree.t +++ b/tests/functional/tree/cram/iqtree.t @@ -5,7 +5,7 @@ Setup Try building a tree with IQ-TREE. $ ${AUGUR} tree \ - > --alignment "$TESTDIR/../data/aligned.fasta" \ + > --alignment "data/aligned.fasta" \ > --method iqtree \ > --output tree_raw.nwk \ > --nthreads 1 > /dev/null