diff --git a/.github/workflows/_checks.yaml b/.github/workflows/_checks.yaml index b9770683..5496f792 100644 --- a/.github/workflows/_checks.yaml +++ b/.github/workflows/_checks.yaml @@ -139,7 +139,7 @@ jobs: operating_systems: '["ubuntu-latest", "windows-latest"]' python_version_for_codecov: "3.14" operating_system_for_codecov: ubuntu-latest - tests_concurrency: "1" + tests_concurrency: "4" integration_tests: name: Integration tests (${{ matrix.python-version }}, ${{ matrix.os }}) diff --git a/pyproject.toml b/pyproject.toml index 7af1b22c..b6adbeae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,7 +198,7 @@ known-first-party = ["apify_client", "crawlee"] max-branches = 18 [tool.pytest.ini_options] -addopts = "-r a --verbose" +addopts = "-r a --verbose --dist worksteal" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" timeout = 1800 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 49d1e4c2..a76dc448 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -34,6 +34,19 @@ _SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() _MAX_BUILD_ATTEMPTS = 2 +# The Actors in these suites install browsers or Scrapy into their images, so their platform builds take several +# times longer than the rest of the suite. +_HEAVY_IMAGE_SUITES = ('tests/e2e/test_crawlee/', 'tests/e2e/test_scrapy/') + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Schedule the tests with the heaviest Actor images first. + + pytest-xdist distributes the collection in order, so the longest tests have to come first for the workers to + finish at roughly the same time. + """ + items.sort(key=lambda item: not item.nodeid.startswith(_HEAVY_IMAGE_SUITES)) + @pytest.fixture(scope='session') def apify_token() -> str: @@ -110,12 +123,16 @@ def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> None: @pytest.fixture(scope='session') def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: """Build the package wheel if it hasn't been built yet, and return the path to the wheel.""" + # `getbasetemp()` is per-xdist-worker, so both the lock and the indicator file live in its shared parent + # directory, which every worker sees. + shared_tmp_path = tmp_path_factory.getbasetemp().parent + # Make sure the wheel is not being built concurrently across all the pytest-xdist runners, # through locking the building process with a temp file. - with FileLock(tmp_path_factory.getbasetemp().parent / 'sdk_wheel_build.lock'): - # Make sure the wheel is built exactly once across across all the pytest-xdist runners, - # through an indicator file saying that the wheel was already built. - was_wheel_built_this_test_run_file = tmp_path_factory.getbasetemp() / f'wheel_was_built_in_run_{testrun_uid}' + with FileLock(shared_tmp_path / 'sdk_wheel_build.lock'): + # Make sure the wheel is built exactly once across all the pytest-xdist runners, through an indicator + # file saying that the wheel was already built. + was_wheel_built_this_test_run_file = shared_tmp_path / f'wheel_was_built_in_run_{testrun_uid}' if not was_wheel_built_this_test_run_file.exists(): subprocess.run( args=[sys.executable, '-m', 'build'], diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index d26844e0..2644afa5 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -435,8 +435,10 @@ async def test_reboot_proceeds_when_event_listener_exceeds_timeout( """Test that a hanging pre-reboot event listener does not block reboot beyond the timeout.""" apify_client_async_patcher.patch('run', 'reboot', return_value=None) + release_listener = asyncio.Event() + async def hanging_listener(*_args: object) -> None: - await asyncio.sleep(60) + await release_listener.wait() async with Actor: Actor.configuration.is_at_home = True @@ -451,6 +453,9 @@ async def hanging_listener(*_args: object) -> None: custom_after_sleep=timedelta(milliseconds=1), ) + # Let the listener finish, so that exiting the context does not wait it out. + release_listener.set() + # The timeout was honored and logged. assert any('Pre-reboot event listeners did not finish within timeout' in r.message for r in caplog.records) diff --git a/tests/unit/test_apify_storages.py b/tests/unit/test_apify_storages.py index cbd624b7..d69cc72c 100644 --- a/tests/unit/test_apify_storages.py +++ b/tests/unit/test_apify_storages.py @@ -1,5 +1,5 @@ -import asyncio import json +import os from datetime import UTC, datetime from pathlib import Path from unittest import mock @@ -136,11 +136,12 @@ async def test_txt_input_missing_metadata(input_test_configuration: Configuratio kvs_path = Path(input_test_configuration.storage_dir) / 'key_value_stores' / 'default' input_file = kvs_path / f'{input_test_configuration.input_key}.txt' input_file.write_text(EXAMPLE_TXT_INPUT) + # Backdate the file so that a rewrite by `KeyValueStore.open` shows up as a changed mtime whatever the + # filesystem's timestamp granularity is. + backdated_time = input_file.stat().st_mtime - 10 + os.utime(input_file, (backdated_time, backdated_time)) last_modified = input_file.stat().st_mtime - # Make sure that filesystem has enough time to detect changes - await asyncio.sleep(1) - kvs = await KeyValueStore.open( storage_client=ApifyFileSystemStorageClient(), configuration=input_test_configuration ) @@ -156,11 +157,12 @@ async def test_json_input_missing_metadata(input_test_configuration: Configurati kvs_path = Path(input_test_configuration.storage_dir) / 'key_value_stores' / 'default' input_file = kvs_path / f'{input_test_configuration.input_key}{suffix}' input_file.write_text(EXAMPLE_JSON_INPUT) + # Backdate the file so that a rewrite by `KeyValueStore.open` shows up as a changed mtime whatever the + # filesystem's timestamp granularity is. + backdated_time = input_file.stat().st_mtime - 10 + os.utime(input_file, (backdated_time, backdated_time)) last_modified = input_file.stat().st_mtime - # Make sure that filesystem has enough time to detect changes - await asyncio.sleep(1) - kvs = await KeyValueStore.open( storage_client=ApifyFileSystemStorageClient(), configuration=input_test_configuration ) @@ -176,11 +178,12 @@ async def test_bytes_input_missing_metadata(input_test_configuration: Configurat kvs_path = Path(input_test_configuration.storage_dir) / 'key_value_stores' / 'default' input_file = kvs_path / f'{input_test_configuration.input_key}{suffix}' input_file.write_bytes(EXAMPLE_BYTES_INPUT) + # Backdate the file so that a rewrite by `KeyValueStore.open` shows up as a changed mtime whatever the + # filesystem's timestamp granularity is. + backdated_time = input_file.stat().st_mtime - 10 + os.utime(input_file, (backdated_time, backdated_time)) last_modified = input_file.stat().st_mtime - # Make sure that filesystem has enough time to detect changes - await asyncio.sleep(1) - kvs = await KeyValueStore.open( storage_client=ApifyFileSystemStorageClient(), configuration=input_test_configuration )