Add pytest runner mode

.github/workflows/main.yml

@@ -9,6 +9,7 @@ jobs:
     name: tests-python${{ matrix.python-version }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
         os: ["macOS-latest", "ubuntu-latest", "windows-latest"]
@@ -35,9 +36,43 @@ jobs:
       - name: Install and run tests macOS
         run: |
           tox -epy --notest
-          .tox/py/bin/pip install gnureadline subunit2sql
+          .tox/py/bin/pip install gnureadline
           tox -epy
         if: runner.os == 'macOS'
+  pytest:
+    name: test-pytest-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["macOS-latest", "ubuntu-latest", "windows-latest"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Pip cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-tests-${{ hashFiles('setup.py','requirements-dev.txt','constraints.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-pip-tests-
+            ${{ runner.os }}-${{ matrix.python-version }}-pip-
+            ${{ runner.os }}-${{ matrix.python-version }}
+      - name: Install Deps
+        run: python -m pip install -U 'tox<4' setuptools virtualenv wheel
+      - name: Install and Run Tests
+        run: tox -e py -- --pytest
+        if: runner.os != 'macOS'
+      - name: Install and run tests macOS
+        run: |
+          tox -epy --notest
+          .tox/py/bin/pip install gnureadline
+          tox -epy -- --pytest
+        if: runner.os == 'macOS'
+
   lint:
     name: pep8
     runs-on: ubuntu-latest

doc/source/MANUAL.rst

@@ -68,6 +68,7 @@ A full example config file is::
   test_path=./project/tests
   top_dir=./
   group_regex=([^\.]*\.)*
+  runner=pytest
 
 
 The ``group_regex`` option is used to specify is used to provide a scheduler
@@ -77,7 +78,10 @@ You can also specify the ``parallel_class=True`` instead of
 group_regex to group tests in the stestr scheduler together by
 class. Since this is a common use case this enables that without
 needing to memorize the complicated regex for ``group_regex`` to do
-this.
+this. The ``runner`` argument is used to specify the test runner to use. By
+default a runner based on Python's standard library ``unittest`` module is
+used. However, if you'd prefer to use ``pytest`` as your runner you can specify
+this as the runner argument in the config file.
 
 There is also an option to specify all the options in the config file via the
 CLI. This way you can run stestr directly without having to write a config file
@@ -137,6 +141,8 @@ providing configs in TOML format, the configuration directives
 **must** be located in a ``[tool.stestr]`` section, and the filename
 **must** have a ``.toml`` extension.
 
+
+
 Running tests
 -------------
 
@@ -166,6 +172,35 @@ Additionally you can specify a specific class or method within that file using
 will skip discovery and directly call the test runner on the test method in the
 specified test class.
 
+.. note::
+
+   If you're using ``--pytest`` or have the runner configured to pytest, then
+   the ``--no-discover``/``-n`` option passes the id field directly to
+   ``pytest`` and the id passed via the argument needs to be in a format that
+   pytest will accept.
+
+Test runners
+''''''''''''
+
+By default ``stestr`` is built to run tests leveraging the Python standard
+library ``unittest`` modules runner. stestr includes a test runner that will
+emit the subunit protocol it relies on internally to handle live results from
+parallel workers. However, there is an alternative runner available that
+leverages ``pytest`` which is a popular test runner and testing library
+alternative to the standard library's ``unittest`` module. The ``stestr``
+project bundles a ``pytest`` plugin that adds real time subunit output to
+pytest. As a test suite author the ``pytest`` plugin enables you to write your
+test suite using pytest's test library instead of ``unittest``. There are two
+ways to specify your test runner, first is the ``--pytest`` flag on
+``stestr run``. This tells stestr for this test run use ``pytest`` as the
+runner instead of ``unittest``, this is good for a/b comparisons between the
+test runners and also general investigations with using different test runners.
+The other option is to leverage your project's config file and set the
+``runner`` field to either ``pytest`` or ``unittest`` (although ``unittest`` is
+always the default so you shouldn't ever need to set it). This is the more
+natural fit because if your test suite is written using pytest it won't be
+compatible with the unittest based runner.
+
 Running with pdb
 ''''''''''''''''
 

requirements.txt

@@ -10,3 +10,4 @@ PyYAML>=3.10.0 # MIT
 voluptuous>=0.8.9 # BSD License
 tomlkit>=0.11.6 # MIT
 extras>=1.0.0
+pytest>=2.3 # MIT

setup.cfg

@@ -46,6 +46,8 @@ stestr.cm =
     history_list = stestr.commands.history:HistoryList
     history_show = stestr.commands.history:HistoryShow
     history_remove = stestr.commands.history:HistoryRemove
+pytest11 =
+    stestr_subunit = stestr.pytest_subunit
 
 [extras]
 sql =

stestr/commands/run.py

@@ -244,6 +244,12 @@ def get_parser(self, prog_name):
             help="If set, show non-text attachments. This is "
             "generally only useful for debug purposes.",
         )
+        parser.add_argument(
+            "--pytest",
+            action="store_true",
+            dest="pytest",
+            help="If set to True enable using pytest as the test runner",
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -335,6 +341,7 @@ def take_action(self, parsed_args):
             all_attachments=all_attachments,
             show_binary_attachments=args.show_binary_attachments,
             pdb=args.pdb,
+            pytest=args.pytest,
         )
 
         # Always output slowest test info if requested, regardless of other
@@ -396,6 +403,7 @@ def run_command(
     all_attachments=False,
     show_binary_attachments=True,
     pdb=False,
+    pytest=False,
 ):
     """Function to execute the run command
 
@@ -460,6 +468,8 @@ def run_command(
     :param str pdb: Takes in a single test_id to bypasses test
         discover and just execute the test specified without launching any
         additional processes. A file name may be used in place of a test name.
+    :param bool pytest: Set to true to use pytest as the test runner instead of
+        the stestr stdlib based unittest runner
 
     :return return_code: The exit code for the command. 0 for success and > 0
         for failures.
@@ -519,13 +529,15 @@ def run_command(
         stdout.write(msg)
         return 2
 
+    conf = config_file.TestrConf.load_from_file(config)
     if no_discover:
         ids = no_discover
-        if "::" in ids:
-            ids = ids.replace("::", ".")
-        if ids.find("/") != -1:
-            root = ids.replace(".py", "")
-            ids = root.replace("/", ".")
+        if not pytest and conf.runner != "pytest":
+            if "::" in ids:
+                ids = ids.replace("::", ".")
+            if ids.find("/") != -1:
+                root = ids.replace(".py", "")
+                ids = root.replace("/", ".")
         stestr_python = sys.executable
         if os.environ.get("PYTHON"):
             python_bin = os.environ.get("PYTHON")
@@ -535,7 +547,10 @@ def run_command(
             raise RuntimeError(
                 "The Python interpreter was not found and " "PYTHON is not set"
             )
-        run_cmd = python_bin + " -m stestr.subunit_runner.run " + ids
+        if pytest or conf.runner == "pytest":
+            run_cmd = python_bin + " -m pytest --subunit " + ids
+        else:
+            run_cmd = python_bin + " -m stestr.subunit_runner.run " + ids
 
         def run_tests():
             run_proc = [
@@ -629,7 +644,6 @@ def run_tests():
             # that are both failing and listed.
             ids = list_ids.intersection(ids)
 
-    conf = config_file.TestrConf.load_from_file(config)
     if not analyze_isolation:
         cmd = conf.get_run_command(
             ids,
@@ -645,6 +659,7 @@ def run_tests():
             top_dir=top_dir,
             test_path=test_path,
             randomize=random,
+            pytest=pytest,
         )
         if isolated:
             result = 0
@@ -669,6 +684,7 @@ def run_tests():
                     randomize=random,
                     test_path=test_path,
                     top_dir=top_dir,
+                    pytest=pytest,
                 )
 
                 run_result = _run_tests(
@@ -724,6 +740,7 @@ def run_tests():
                 randomize=random,
                 test_path=test_path,
                 top_dir=top_dir,
+                pytest=pytest,
             )
             if not _run_tests(cmd, until_failure):
                 # If the test was filtered, it won't have been run.

stestr/config_file.py

@@ -39,6 +39,7 @@ class TestrConf:
     top_dir = None
     parallel_class = False
     group_regex = None
+    runner = None
 
     def __init__(self, config_file, section="DEFAULT"):
         self.config_file = str(config_file)
@@ -59,6 +60,7 @@ def _load_from_configparser(self):
         self.group_regex = parser.get(
             self.section, "group_regex", fallback=self.group_regex
         )
+        self.runner = parser.get(self.section, "runner", fallback=self.runner)
 
     def _load_from_toml(self):
         with open(self.config_file) as f:
@@ -68,6 +70,7 @@ def _load_from_toml(self):
             self.top_dir = root.get("top_dir", self.top_dir)
             self.parallel_class = root.get("parallel_class", self.parallel_class)
             self.group_regex = root.get("group_regex", self.group_regex)
+            self.runner = root.get("runner", self.runner)
 
     @classmethod
     def load_from_file(cls, config):
@@ -113,6 +116,7 @@ def get_run_command(
         exclude_regex=None,
         randomize=False,
         parallel_class=None,
+        pytest=False,
     ):
         """Get a test_processor.TestProcessorFixture for this config file
 
@@ -158,6 +162,8 @@ def get_run_command(
             stestr scheduler by class. If both this and the corresponding
             config file option which includes `group-regex` are set, this value
             will be used.
+        :param bool pytest: Set to true to use pytest as the test runner instead of
+            the stestr stdlib based unittest runner
 
         :returns: a TestProcessorFixture object for the specified config file
             and any arguments passed into this function
@@ -198,12 +204,48 @@ def get_run_command(
         if os.path.exists('"%s"' % python):
             python = '"%s"' % python
 
-        command = (
-            '%s -m stestr.subunit_runner.run discover -t "%s" "%s" '
-            "$LISTOPT $IDOPTION" % (python, top_dir, test_path)
-        )
-        listopt = "--list"
-        idoption = "--load-list $IDFILE"
+        if not pytest and self.runner is not None:
+            if self.runner == "pytest":
+                pytest = True
+            elif self.runner == "unittest":
+                pytest = False
+            else:
+                raise RuntimeError(
+                    f"Specified runner argument value: {self.runner} in "
+                    "config file is not valid. Only pytest or unittest can be "
+                    "specified in the config file."
+                )
+        if pytest:
+            if sys.platform == "win32":
+                command = (
+                    '%s -m pytest -s --subunit --rootdir="%s" "%s" '
+                    "$LISTOPT $IDOPTION"
+                    % (
+                        python,
+                        top_dir,
+                        test_path,
+                    )
+                )
+
+            else:
+                command = (
+                    '%s -m pytest --subunit --rootdir="%s" "%s" '
+                    "$LISTOPT $IDOPTION"
+                    % (
+                        python,
+                        top_dir,
+                        test_path,
+                    )
+                )
+            listopt = "--co"
+            idoption = "--load-list $IDFILE"
+        else:
+            command = (
+                '%s -m stestr.subunit_runner.run discover -t "%s" "%s" '
+                "$LISTOPT $IDOPTION" % (python, top_dir, test_path)
+            )
+            listopt = "--list"
+            idoption = "--load-list $IDFILE"
         # If the command contains $IDOPTION read that command from config
         # Use a group regex if one is defined
         if parallel_class or self.parallel_class: