FCL-1843 | add VCR with fixtures for isolated E2E test running

.env.example

@@ -1,4 +1,4 @@
-MARKLOGIC_HOST=marklogic.caselaw
+MARKLOGIC_HOST=ml.external.staging.caselaw.nationalarchives.gov.uk
 MARKLOGIC_USER=admin
 MARKLOGIC_PASSWORD=admin
 ROLLBAR_ACCESS_TOKEN=
@@ -17,3 +17,9 @@ ASSETS_CDN_BASE_URL=https://assets.caselaw.nationalarchives.gov.uk
 
 # SHOW_WEASYPRINT_LOGS=True
 MAX_QUERY_WORDS=5
+
+# Controls whether to use recorded cassettes for HTTP requests
+VCR_ENABLED=true
+
+# VCR modes: playback (use cassettes), record (save requests)
+VCR_MODE=playback

.github/workflows/ci.yml

@@ -249,10 +249,7 @@ jobs:
           persist-credentials: false
 
       - name: Copy .env.example file
-        uses: canastro/copy-file-action@ae66602ce7d214dbd2e298c1db67a81388755a0a
-        with:
-          source: ".env.example"
-          target: ".env"
+        run: cp .env.example .env
 
       - name: Create docker network as used in dev
         run: docker network create caselaw
@@ -275,34 +272,12 @@ jobs:
       - name: Load Postgres image
         run: docker load --input /tmp/postgres-image.tar
 
-      - name: Checkout Marklogic repo
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          repository: "nationalarchives/ds-caselaw-marklogic"
-          path: "./marklogic"
-          persist-credentials: false
-
-      - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
-
       - name: Setup Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
 
       - name: Install requests
         run: "pip install requests"
 
-      - name: Run marklogic docker container
-        run: docker compose up -d --wait
-        working-directory: "./marklogic"
-
-      - name: Deploy marklogic configuration
-        run: gradle mlDeploy
-        working-directory: "./marklogic"
-
-      - name: Populate marklogic DB with test data
-        run: python ./development_scripts/populate_from_caselaw.py
-        working-directory: "./marklogic"
-
       - name: Run DB Migrations
         run: docker compose run --rm django python manage.py migrate --settings=config.settings.test
 
@@ -315,9 +290,6 @@ jobs:
       - name: Build the JS and SASS
         run: npm run build
 
-      - name: Update court dates
-        run: docker compose run --rm django python manage.py recalculate_court_dates
-
       - name: Kick off the django server itself
         run: docker compose exec -d django python manage.py runserver 0.0.0.0:3000
 
@@ -348,9 +320,5 @@ jobs:
           path: e2e_tests/snapshots
           retention-days: 7
 
-      - name: Tear down marklogic
-        run: docker compose down
-        working-directory: "./marklogic"
-
       - name: Tear down the Stack
         run: docker compose down

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: "/migrations/"
+exclude: (^|/)(migrations/|vcr_cassettes/)
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks

README.md

@@ -178,6 +178,15 @@ Pytest unit tests can be run with `fab test`.
 We also have a suite of end to end tests (in the `e2e_tests/` directory) written with [playwright-pytest](https://playwright.dev/python/docs/api/class-playwright), which can be run with `fab e2etest`.
 These will run by default against the running `django` container. You can supply a `baseURL` argument to test against staging or production.
 
+### VCR Cassettes
+
+To allow running the E2E tests in isolation, and to run the application in a state where it doesn't call out to external services, we have added VCR cassettes that can be played for external requests instead of calling out to that external service.
+
+To enable these, add the following to your .env:
+
+VCR_ENABLED=true // Controls whether to use recorded cassettes for HTTP requests
+VCR_MODE=playback // VCR modes: playback (use cassettes), record (save requests)
+
 ### memray memory tools
 
 To generate a flamegraph run `fab flamegraph`. After using the application, press Ctrl-C and the application will quit and the flamegraph will appear in your browser.
@@ -220,7 +229,7 @@ def test_my_page(page: Page):
     assert_is_accessible(page)
 ```
 
-If the page is accessible, the console won't output anything other than the usual test output. If there are accessibility issues, there will be output that explains what the issues are and also helpful links explaining how to fix them.
+    If the page is accessible, the console won't output anything other than the usual test output. If there are accessibility issues, there will be output that explains what the issues are and also helpful links explaining how to fix them.
 
 #### Test traces in CI
 

config/settings/local.py

@@ -1,5 +1,8 @@
 from .base import *  # noqa
 from .base import env
+from .base import ROOT_DIR
+
+import os
 
 # GENERAL
 # ------------------------------------------------------------------------------
@@ -48,6 +51,7 @@
     "corsheaders.middleware.CorsMiddleware",  # must be first
     "django.middleware.security.SecurityMiddleware",
     "whitenoise.middleware.WhiteNoiseMiddleware",
+    "config.vcr_middleware.VCRMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
@@ -63,6 +67,13 @@
     "waffle.middleware.WaffleMiddleware",
 ]
 
+VCR_ENABLED = os.getenv("VCR_ENABLED", "true").lower() == "true"
+VCR_MODE = os.getenv("VCR_MODE", "playback")
+VCR_CASSETTE_DIR = str(ROOT_DIR / "vcr_cassettes")
+
+if VCR_ENABLED:
+    os.makedirs(VCR_CASSETTE_DIR, exist_ok=True)
+
 
 # Only allow Storybook CORS in dev
 DEBUG = env.bool("DJANGO_DEBUG", default=True)