From 2f4b7e859882a6231d626eaf523f77b969c4e3bb Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 23 Mar 2023 19:06:53 -0700 Subject: [PATCH 1/5] Bump version to 0.1.5 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 885e92d..bf8993c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pytest-unflakable -version = 0.1.4 +version = 0.1.5 author = Unflakable author_email = support@unflakable.com maintainer = Unflakable From 2931aa42b2582f8e578bf1106c83c36a68f2255f Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 23 Mar 2023 19:15:37 -0700 Subject: [PATCH 2/5] Fix pytest versions in package README --- src/pytest_unflakable/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_unflakable/README.md b/src/pytest_unflakable/README.md index a75a8e7..a586d4c 100644 --- a/src/pytest_unflakable/README.md +++ b/src/pytest_unflakable/README.md @@ -23,7 +23,7 @@ for running `unittest` tests. This plugin maintains compatibility with the PyTest and Python versions listed below: ![Python versions](https://img.shields.io/pypi/pyversions/pytest-unflakable) -![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1-blue) +![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1%20%7C%207.2-blue) ## Contributing From cceab234c376f03b52b88489afec23f0bf415b04 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 5 Sep 2023 19:56:08 -0700 Subject: [PATCH 3/5] Remove Twitter badges --- README.md | 2 -- src/pytest_unflakable/README.md | 1 - 2 files changed, 3 deletions(-) diff --git a/README.md b/README.md index 4bc9c74..d176e9c 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@

-[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) - # Official Unflakable Plugins for Python This repository contains the official [Unflakable](https://unflakable.com) plugins for Python. diff --git a/src/pytest_unflakable/README.md b/src/pytest_unflakable/README.md index a586d4c..69c133b 100644 --- a/src/pytest_unflakable/README.md +++ b/src/pytest_unflakable/README.md @@ -5,7 +5,6 @@

[![PyPI version](https://img.shields.io/pypi/v/pytest-unflakable.svg)](https://pypi.org/project/pytest-unflakable) -[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) # Unflakable Plugin for PyTest From cd92dbb103772156396b803b6a86ac37621276ad Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 5 Sep 2023 20:28:19 -0700 Subject: [PATCH 4/5] Test PyTest 7.3 and 7.4 in CI --- .github/workflows/ci.yaml | 2 ++ README.md | 2 +- src/pytest_unflakable/README.md | 2 +- tox.ini | 12 +++++++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a43dc37..10b205b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,6 +77,8 @@ jobs: - pytest70 - pytest71 - pytest72 + - pytest73 + - pytest74 steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index d176e9c..85d6f81 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ for running `unittest` tests. This plugin maintains compatibility with the PyTest and Python versions listed below: ![Python versions](https://img.shields.io/pypi/pyversions/pytest-unflakable) -![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1%20%7C%207.2-blue) +![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1%20%7C%207.2%20%7C%207.3%20%7C%207.4-blue) ## Contributing diff --git a/src/pytest_unflakable/README.md b/src/pytest_unflakable/README.md index 69c133b..11fdc59 100644 --- a/src/pytest_unflakable/README.md +++ b/src/pytest_unflakable/README.md @@ -22,7 +22,7 @@ for running `unittest` tests. This plugin maintains compatibility with the PyTest and Python versions listed below: ![Python versions](https://img.shields.io/pypi/pyversions/pytest-unflakable) -![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1%20%7C%207.2-blue) +![PyTest versions](https://img.shields.io/badge/pytest-6.2%20%7C%207.0%20%7C%207.1%20%7C%207.2%20%7C%207.3%20%7C%207.4-blue) ## Contributing diff --git a/tox.ini b/tox.ini index 1aca426..82d1ea8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # For more information about tox, see https://tox.readthedocs.io/en/latest/ [tox] -envlist = pytest{62,70,71,72},flake8,mypy,pycodestyle +envlist = pytest{62,70,71,72,73,74},flake8,mypy,pycodestyle [testenv] extras = dev @@ -26,6 +26,16 @@ deps = pytest>=7.2.0,<7.3.0 pytest-xdist==2.5.0 +[testenv:pytest73] +deps = + pytest>=7.3.0,<7.4.0 + pytest-xdist==2.5.0 + +[testenv:pytest74] +deps = + pytest>=7.4.0,<7.5.0 + pytest-xdist==2.5.0 + [testenv:flake8] commands = flake8 src tests From 6224680a91ec506ac1682b3f4f2d12b80327c5fd Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Fri, 8 Sep 2023 22:12:50 -0700 Subject: [PATCH 5/5] Fix PyTest plugin when xdist is not installed The pytest_configure_node() hook is only allowed when xdist is installed. This also updates the tests to run with and without xdist. --- .github/workflows/ci.yaml | 6 + setup.cfg | 1 - src/pytest_unflakable/__init__.py | 43 +++- src/pytest_unflakable/_plugin.py | 62 +++--- tests/test_unflakable.py | 355 +++++++----------------------- tox.ini | 38 +++- 6 files changed, 182 insertions(+), 323 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10b205b..fb166b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,11 +74,17 @@ jobs: - "3.11" tox_env: - pytest62 + - pytest62-xdist - pytest70 + - pytest70-xdist - pytest71 + - pytest71-xdist - pytest72 + - pytest72-xdist - pytest73 + - pytest73-xdist - pytest74 + - pytest74-xdist steps: - uses: actions/checkout@v3 diff --git a/setup.cfg b/setup.cfg index bf8993c..6adbab3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,7 +47,6 @@ dev = flake8-quotes mypy py>=1.9.0 - pytest-xdist>=2.0.0 requests-mock[fixture] types-requests types-setuptools diff --git a/src/pytest_unflakable/__init__.py b/src/pytest_unflakable/__init__.py index b5b9fe8..2676605 100644 --- a/src/pytest_unflakable/__init__.py +++ b/src/pytest_unflakable/__init__.py @@ -5,13 +5,16 @@ import argparse import os +import pprint +import sys from typing import TYPE_CHECKING import pytest import logging +from ._api import get_test_suite_manifest from ._git import get_current_git_commit, get_current_git_branch -from ._plugin import UnflakablePlugin, QuarantineMode +from ._plugin import UnflakablePlugin, QuarantineMode, UnflakableXdistHooks if TYPE_CHECKING: Config = pytest.Config @@ -130,7 +133,7 @@ def pytest_configure(config: Config) -> None: if config.getoption('unflakable_suite_id') is None: raise pytest.UsageError('missing required argument --test-suite-id') - # pytest-xdist workers don't make API calls and amy not have the API key available. + # pytest-xdist workers don't make API calls and may not have the API key available. if is_xdist_worker: api_key = '' elif config.option.unflakable_api_key_path is not None: @@ -143,6 +146,7 @@ def pytest_configure(config: Config) -> None: branch = config.option.unflakable_branch commit = config.option.unflakable_commit + test_suite_id = config.option.unflakable_suite_id git_auto_detect = not config.getoption('unflakable_no_git_auto_detect', False) if git_auto_detect and not is_xdist_worker: if commit is None: @@ -153,10 +157,34 @@ def pytest_configure(config: Config) -> None: branch = get_current_git_branch(commit, logger) logger.debug('auto-detected branch `%s`', branch) + insecure_disable_tls_validation = config.getoption( + 'unflakable_insecure_disable_tls_validation', False) + manifest = None if is_xdist_worker and 'unflakable_manifest' in config.workerinput: # type: ignore - worker_manifest = config.workerinput['unflakable_manifest'] # type: ignore + manifest = config.workerinput['unflakable_manifest'] # type: ignore + logger.debug( + f'xdist worker received manifest for test suite {test_suite_id}: ' + f'{pprint.pformat(manifest)}' + ) else: - worker_manifest = None + try: + manifest = get_test_suite_manifest( + test_suite_id=test_suite_id, + api_key=api_key, + base_url=config.option.unflakable_base_url, + insecure_disable_tls_validation=insecure_disable_tls_validation, + logger=logger, + ) + # IOError is the base class for `requests.RequestException`. + except IOError as e: + sys.stderr.write( + ('ERROR: Failed to get Unflakable manifest: %s\nTest failures will NOT be' + ' quarantined.\n') % (repr(e))), + + if config.pluginmanager.hasplugin('xdist'): + config.pluginmanager.register( + UnflakableXdistHooks(logger=logger, worker_manifest=manifest) + ) config.pluginmanager.register(UnflakablePlugin( api_key=api_key, @@ -164,13 +192,12 @@ def pytest_configure(config: Config) -> None: branch=branch, commit=commit, failure_retries=config.option.unflakable_failure_retries, - insecure_disable_tls_validation=config.getoption( - 'unflakable_insecure_disable_tls_validation', False), + insecure_disable_tls_validation=insecure_disable_tls_validation, quarantine_mode=quarantine_mode, - test_suite_id=config.option.unflakable_suite_id, + test_suite_id=test_suite_id, upload_results=not is_xdist_worker and ( not config.getoption('unflakable_no_upload_results', False)), logger=logger, - worker_manifest=worker_manifest, + manifest=manifest, is_xdist_worker=is_xdist_worker, )) diff --git a/src/pytest_unflakable/_plugin.py b/src/pytest_unflakable/_plugin.py index 8e76627..cc89968 100644 --- a/src/pytest_unflakable/_plugin.py +++ b/src/pytest_unflakable/_plugin.py @@ -8,16 +8,13 @@ ) import logging -import pprint import pytest import _pytest -import sys from time import time from datetime import datetime, timezone from ._api import ( - get_test_suite_manifest, create_test_suite_run, build_test_suite_run_url, CreateTestSuiteRunRequest, @@ -152,6 +149,30 @@ def count_towards_summary(self) -> bool: ItemReports = Dict[CallPhase, UnflakableReport] +class UnflakableXdistHooks: + logger: logging.Logger + + def __init__( + self, + logger: logging.Logger, + worker_manifest: Optional[TestSuiteManifest], + ): + self.logger = logger + self.manifest = worker_manifest + + # This is a `xdist.workermanage.WorkerController`, but pytest-xdist doesn't provide types. + def pytest_configure_node(self, node: Any) -> None: + """ + Hook called by pytest-xdist to configure each worker node. + + We leverage this hook to send the manifest to the worker. + """ + nodeid = node.workerinput['workerid'] + self.logger.debug(f'called hook pytest_configure_node: {nodeid}') + if self.manifest is not None: + node.workerinput['unflakable_manifest'] = self.manifest + + class UnflakablePlugin: api_key: str base_url: Optional[str] @@ -186,7 +207,7 @@ def __init__( test_suite_id: str, upload_results: bool, logger: logging.Logger, - worker_manifest: Optional[TestSuiteManifest], + manifest: Optional[TestSuiteManifest], is_xdist_worker: bool, ): self.api_key = api_key @@ -203,26 +224,7 @@ def __init__( self.is_xdist_worker = is_xdist_worker self.item_reports = {} - self.manifest = worker_manifest - if self.manifest is None: - try: - self.manifest = get_test_suite_manifest( - test_suite_id=self.test_suite_id, - api_key=self.api_key, - base_url=self.base_url, - insecure_disable_tls_validation=self.insecure_disable_tls_validation, - logger=self.logger, - ) - # IOError is the base class for `requests.RequestException`. - except IOError as e: - sys.stderr.write( - ('ERROR: Failed to get Unflakable manifest: %s\nTest failures will NOT be' - ' quarantined.\n') % (repr(e))), - else: - logger.debug( - f'xdist worker received manifest for test suite {self.test_suite_id}: ' - f'{pprint.pformat(self.manifest)}' - ) + self.manifest = manifest self.quarantined_tests = set([ (quarantined_test['filename'], tuple(quarantined_test['name'])) for @@ -495,18 +497,6 @@ def pytest_sessionstart(self, session: pytest.Session) -> None: self.session = session self.start_time = time() - # This is a `xdist.workermanage.WorkerController`, but pytest-xdist doesn't provide types. - def pytest_configure_node(self, node: Any) -> None: - """ - Hook called by pytest-xdist to configure each worker node. - - We leverage this hook to send the manifest to the worker. - """ - nodeid = node.workerinput['workerid'] - self.logger.debug(f'called hook pytest_configure_node: {nodeid}') - if self.manifest is not None: - node.workerinput['unflakable_manifest'] = self.manifest - def _build_test_suite_run_request( self, session: pytest.Session, diff --git a/tests/test_unflakable.py b/tests/test_unflakable.py index b2810f4..e2f70ef 100644 --- a/tests/test_unflakable.py +++ b/tests/test_unflakable.py @@ -1,4 +1,5 @@ """Tests for pytest_unflakable plugin.""" +import os # Copyright (c) 2022-2023 Developer Innovations, LLC @@ -38,15 +39,43 @@ def _3platform() -> None: pass -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], +TEST_PARAMS_XDIST_ARG_NAMES = ['xdist'] +TEST_PARAMS_XDIST_ARG_VALUES = ( + [ + pytest.param(False, id='not_xdist'), + ] + ([ + pytest.param(True, id='xdist'), + ] if os.environ.get('TEST_XDIST') == '1' else []) +) + +TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES = ['verbose', 'xdist'] +TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES = ( + [ + pytest.param(False, False, id='not_verbose-not_xdist'), + pytest.param(True, False, id='verbose-not_xdist'), + ] + ([ + pytest.param(False, True, id='not_verbose-xdist'), + pytest.param(True, True, id='verbose-xdist'), + ] if os.environ.get('TEST_XDIST') == '1' else []) +) + +TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES = ['verbose', 'quarantined', 'xdist'] +TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES = ( + [ + pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), + pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), + pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), + pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), + ] + ([ + pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), + pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), + pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), + pytest.param(True, True, True, id='verbose-quarantined-xdist'), + ] if os.environ.get('TEST_XDIST') == '1' else []) ) + + +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_flaky( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -93,15 +122,7 @@ def test_flaky(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_quarantine_flaky( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -154,15 +175,7 @@ def test_flaky(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_flaky_until_last_attempt( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -210,15 +223,7 @@ def test_flaky(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_all_statuses( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -316,15 +321,7 @@ def test_skipped(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_class_all_statuses( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -423,15 +420,7 @@ def test_skipped(self): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_nested_classes( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -526,15 +515,7 @@ def test_flaky(self): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_unittest_all_statuses( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -639,15 +620,7 @@ def test_skipped(self): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_multiple_files( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -713,15 +686,7 @@ def test_quarantined(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_quarantine_mode_ignore_failures( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -779,15 +744,7 @@ def test_quarantined(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_quarantine_mode_no_quarantine( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -845,15 +802,7 @@ def test_quarantined(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_quarantine_mode_skip_tests( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -906,15 +855,7 @@ def test_quarantined(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_parameterized( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -976,10 +917,7 @@ def test_with_param(p): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_empty_collection( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1000,15 +938,7 @@ def test_empty_collection( ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_all_skipped( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1038,15 +968,7 @@ def test_skipped(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_skipped_and_pass( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1089,10 +1011,7 @@ def test_skipped(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_git_detached_head( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1120,10 +1039,7 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_no_git_repo( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1153,10 +1069,7 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_no_git_auto_detect( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1186,10 +1099,7 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_git_cli_args( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1216,20 +1126,12 @@ def test_pass(): expected_commit='CLI_COMMIT', expect_xdist=xdist, extra_args=[ - '--branch', 'CLI_BRANCH', '--commit', 'CLI_COMMIT' - ] + (XDIST_ARGS if xdist else []), + '--branch', 'CLI_BRANCH', '--commit', 'CLI_COMMIT' + ] + (XDIST_ARGS if xdist else []), ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_no_retries( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1268,10 +1170,7 @@ def test_flaky(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_api_key_environ( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1303,10 +1202,7 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_plugin_disabled( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1347,10 +1243,7 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_no_upload_results( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1403,15 +1296,7 @@ def test_quarantined(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_select_subset( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1489,19 +1374,8 @@ def test_pass(): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_setup_failure( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1573,19 +1447,8 @@ def test_setup_fail(setup_fail): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_setup_flaky( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1661,19 +1524,8 @@ def test_setup_flaky(setup_flaky): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_teardown_failure( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1755,19 +1607,8 @@ def test_teardown_fail(teardown_fail): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_teardown_flaky( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1847,15 +1688,7 @@ def test_teardown_flaky(teardown_flaky): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_xfail_pass( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1892,15 +1725,7 @@ def test_xfail(): ) -@pytest.mark.parametrize( - ['verbose', 'xdist'], - [ - pytest.param(False, False, id='not_verbose-not_xdist'), - pytest.param(False, True, id='not_verbose-xdist'), - pytest.param(True, False, id='verbose-not_xdist'), - pytest.param(True, True, id='verbose-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_XDIST_ARG_NAMES, TEST_PARAMS_VERBOSE_XDIST_ARG_VALUES) def test_xfail_fail( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -1942,19 +1767,8 @@ def test_xfail(): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_xfail_fail_strict( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -2013,19 +1827,8 @@ def test_xfail(): ) -@pytest.mark.parametrize( - ['verbose', 'quarantined', 'xdist'], - [ - pytest.param(False, False, False, id='not_verbose-not_quarantined-not_xdist'), - pytest.param(False, True, False, id='not_verbose-quarantined-not_xdist'), - pytest.param(True, False, False, id='verbose-not_quarantined-not_xdist'), - pytest.param(True, True, False, id='verbose-quarantined-not_xdist'), - pytest.param(False, False, True, id='not_verbose-not_quarantined-xdist'), - pytest.param(False, True, True, id='not_verbose-quarantined-xdist'), - pytest.param(True, False, True, id='verbose-not_quarantined-xdist'), - pytest.param(True, True, True, id='verbose-quarantined-xdist'), - ], -) +@pytest.mark.parametrize(TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_NAMES, + TEST_PARAMS_VERBOSE_QUARANTINED_XDIST_ARG_VALUES) def test_xfail_flaky_strict( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -2089,10 +1892,7 @@ def test_xfail(): ) -@pytest.mark.parametrize( - ['xdist'], - [pytest.param(False, id='not_xdist'), pytest.param(True, id='xdist')], -) +@pytest.mark.parametrize(TEST_PARAMS_XDIST_ARG_NAMES, TEST_PARAMS_XDIST_ARG_VALUES) def test_warnings( pytester: pytest.Pytester, requests_mock: requests_mock.Mocker, @@ -2244,6 +2044,7 @@ def test_pass2(): ) +@pytest.mark.skipif(os.environ.get('TEST_XDIST') != '1', reason='xdist is disabled') @pytest.mark.parametrize( ['verbose', 'quarantined'], [ diff --git a/tox.ini b/tox.ini index 82d1ea8..5980e13 100644 --- a/tox.ini +++ b/tox.ini @@ -1,40 +1,76 @@ # For more information about tox, see https://tox.readthedocs.io/en/latest/ [tox] -envlist = pytest{62,70,71,72,73,74},flake8,mypy,pycodestyle +envlist = pytest{62,70,71,72,73,74},pytest{62,70,71,72,73,74}-xdist,flake8,mypy,pycodestyle [testenv] extras = dev commands = pytest {posargs:tests} [testenv:pytest62] +deps = + pytest>=6.2.0,<6.3.0 + +[testenv:pytest62-xdist] deps = pytest>=6.2.0,<6.3.0 pytest-xdist==2.3.0 +setenv = + TEST_XDIST = 1 [testenv:pytest70] +deps = + pytest>=7.0.0,<7.1.0 + +[testenv:pytest70-xdist] deps = pytest>=7.0.0,<7.1.0 pytest-xdist==2.4.0 +setenv = + TEST_XDIST = 1 [testenv:pytest71] +deps = + pytest>=7.1.0,<7.2.0 + +[testenv:pytest71-xdist] deps = pytest>=7.1.0,<7.2.0 pytest-xdist==2.5.0 +setenv = + TEST_XDIST = 1 [testenv:pytest72] +deps = + pytest>=7.2.0,<7.3.0 + +[testenv:pytest72-xdist] deps = pytest>=7.2.0,<7.3.0 pytest-xdist==2.5.0 +setenv = + TEST_XDIST = 1 [testenv:pytest73] +deps = + pytest>=7.3.0,<7.4.0 + +[testenv:pytest73-xdist] deps = pytest>=7.3.0,<7.4.0 pytest-xdist==2.5.0 +setenv = + TEST_XDIST = 1 [testenv:pytest74] +deps = + pytest>=7.4.0,<7.5.0 + +[testenv:pytest74-xdist] deps = pytest>=7.4.0,<7.5.0 pytest-xdist==2.5.0 +setenv = + TEST_XDIST = 1 [testenv:flake8] commands = flake8 src tests