diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 9cf18b9072c..56230c96bf1 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -237,7 +237,7 @@ jobs: PIP_USER: 1 run: >- PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}" - pytest --junitxml=junit.xml + pytest --junitxml=junit.xml -m 'not dev_mode and not autobahn' shell: bash - name: Re-run the failing tests with maximum verbosity if: failure() @@ -280,6 +280,98 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + autobahn: + permissions: + contents: read # to fetch code (actions/checkout) + + name: Autobahn testsuite + needs: gen_llhttp + strategy: + matrix: + pyver: ['3.14'] + no-extensions: [''] + os: [ubuntu] + fail-fast: true + runs-on: ${{ matrix.os }}-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: true + - name: Setup Python ${{ matrix.pyver }} + id: python-install + uses: actions/setup-python@v6 + with: + allow-prereleases: true + python-version: ${{ matrix.pyver }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "dir=$(pip cache dir)" >> "${GITHUB_OUTPUT}" + shell: bash + - name: Cache PyPI + uses: actions/cache@v5.0.3 + with: + key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} + path: ${{ steps.pip-cache.outputs.dir }} + restore-keys: | + pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Install dependencies + env: + DEPENDENCY_GROUP: test${{ endsWith(matrix.pyver, 't') && '-ft' || '' }} + run: | + python -Im pip install -r requirements/${{ env.DEPENDENCY_GROUP }}.in -c requirements/${{ env.DEPENDENCY_GROUP }}.txt + - name: Restore llhttp generated files + if: ${{ matrix.no-extensions == '' }} + uses: actions/download-artifact@v8 + with: + name: llhttp + path: vendor/llhttp/build/ + - name: Cythonize + if: ${{ matrix.no-extensions == '' }} + run: | + make cythonize + - name: Install self + env: + AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} + run: python -m pip install -e . + - name: Run unittests + env: + COLOR: yes + AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} + PIP_USER: 1 + run: >- + PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}" + pytest --junitxml=junit.xml --numprocesses=0 -m autobahn + shell: bash + - name: Turn coverage into xml + env: + COLOR: 'yes' + PIP_USER: 1 + run: | + python -m coverage xml + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + files: ./coverage.xml + flags: >- + CI-GHA,OS-${{ + runner.os + }},VM-${{ + matrix.os + }},Py-${{ + steps.python-install.outputs.python-version + }} + token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + benchmark: name: Benchmark needs: @@ -330,20 +422,21 @@ jobs: needs: - lint - test + - autobahn runs-on: ubuntu-latest steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} - name: Trigger codecov notification uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true run_command: send-notifications - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} pre-deploy: name: Pre-Deploy diff --git a/CHANGES/12173.contrib.rst b/CHANGES/12173.contrib.rst new file mode 100644 index 00000000000..7c1bd3d1c1b --- /dev/null +++ b/CHANGES/12173.contrib.rst @@ -0,0 +1 @@ +Fixed and reworked ``autobahn`` tests -- by :user:`Dreamsorcerer`. diff --git a/setup.cfg b/setup.cfg index dce66515bad..2d7e24b0374 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,8 +61,7 @@ addopts = --cov=aiohttp --cov=tests/ - # run tests that are not marked with dev_mode - -m "not dev_mode" + -m "not dev_mode and not autobahn and not internal" filterwarnings = error ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning @@ -95,6 +94,7 @@ minversion = 3.8.2 testpaths = tests/ xfail_strict = true markers = + autobahn: Autobahn testsuite. Should be run as a separate job. dev_mode: mark test to run in dev mode. internal: tests which may cause issues for packagers, but should be run in aiohttp's CI. skip_blockbuster: mark test to skip the blockbuster fixture. diff --git a/tests/autobahn/client/client.py b/tests/autobahn/client/client.py index 5dbcd6ff654..2a58ff8bf33 100644 --- a/tests/autobahn/client/client.py +++ b/tests/autobahn/client/client.py @@ -2,42 +2,31 @@ import asyncio -import aiohttp +from aiohttp import ClientSession, WSMsgType async def client(url: str, name: str) -> None: - async with aiohttp.ClientSession() as session: - async with session.ws_connect(url + "/getCaseCount") as ws: + async with ClientSession(base_url=url) as session: + async with session.ws_connect("/getCaseCount") as ws: msg = await ws.receive() - assert msg.type is aiohttp.WSMsgType.TEXT + assert msg.type is WSMsgType.TEXT num_tests = int(msg.data) - print("running %d cases" % num_tests) for i in range(1, num_tests + 1): - print("running test case:", i) - text_url = url + "/runCase?case=%d&agent=%s" % (i, name) - async with session.ws_connect(text_url) as ws: + async with session.ws_connect( + "/runCase", params={"case": i, "agent": name} + ) as ws: async for msg in ws: - if msg.type is aiohttp.WSMsgType.TEXT: + if msg.type is WSMsgType.TEXT: await ws.send_str(msg.data) - elif msg.type is aiohttp.WSMsgType.BINARY: + elif msg.type is WSMsgType.BINARY: await ws.send_bytes(msg.data) else: break - url = url + "/updateReports?agent=%s" % name - async with session.ws_connect(url) as ws: - print("finally requesting %s" % url) + async with session.ws_connect("/updateReports", params={"agent": name}) as ws: + pass -async def run(url: str, name: str) -> None: - try: - await client(url, name) - except Exception: - import traceback - - traceback.print_exc() - - -if __name__ == "__main__": - asyncio.run(run("http://localhost:9001", "aiohttp")) +if __name__ == "__main__": # pragma: no branch + asyncio.run(client("http://localhost:9001", "aiohttp")) diff --git a/tests/autobahn/client/fuzzingserver.json b/tests/autobahn/client/fuzzingserver.json index 5f8bf31d203..8530d15bd36 100644 --- a/tests/autobahn/client/fuzzingserver.json +++ b/tests/autobahn/client/fuzzingserver.json @@ -1,12 +1,11 @@ - { "url": "ws://localhost:9001", - "options": {"failByDrop": false}, + "options": {"failByDrop": true}, "outdir": "./reports/clients", "webport": 8080, "cases": ["*"], - "exclude-cases": ["12.*", "13.*"], + "exclude-cases": [], "exclude-agent-cases": {} } diff --git a/tests/autobahn/server/fuzzingclient.json b/tests/autobahn/server/fuzzingclient.json index 0ed2f84acf8..bf34731a2e8 100644 --- a/tests/autobahn/server/fuzzingclient.json +++ b/tests/autobahn/server/fuzzingclient.json @@ -1,16 +1,16 @@ { - "options": { "failByDrop": false }, + "options": {"failByDrop": true}, "outdir": "./reports/servers", "servers": [ { "agent": "AutobahnServer", "url": "ws://localhost:9001", - "options": { "version": 18 } + "options": {"version": 18} } ], "cases": ["*"], - "exclude-cases": ["12.*", "13.*"], + "exclude-cases": [], "exclude-agent-cases": {} } diff --git a/tests/autobahn/server/server.py b/tests/autobahn/server/server.py index 47122ae2578..1b0560847b8 100644 --- a/tests/autobahn/server/server.py +++ b/tests/autobahn/server/server.py @@ -9,24 +9,15 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse: ws = web.WebSocketResponse(autoclose=False) - is_ws = ws.can_prepare(request) - if not is_ws: - raise web.HTTPBadRequest() - await ws.prepare(request) request.app[websockets].append(ws) - while True: - msg = await ws.receive() - + async for msg in ws: if msg.type is web.WSMsgType.TEXT: await ws.send_str(msg.data) elif msg.type is web.WSMsgType.BINARY: await ws.send_bytes(msg.data) - elif msg.type is web.WSMsgType.CLOSE: - await ws.close() - break else: break @@ -34,22 +25,17 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse: async def on_shutdown(app: web.Application) -> None: - ws_list = app[websockets] - for ws in set(ws_list): + for ws in app[websockets]: await ws.close(code=WSCloseCode.GOING_AWAY, message=b"Server shutdown") -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no branch logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s" ) app = web.Application() - l: list[web.WebSocketResponse] = [] - app[websockets] = l + app[websockets] = [] app.router.add_route("GET", "/", wshandler) app.on_shutdown.append(on_shutdown) - try: - web.run_app(app, port=9001) - except KeyboardInterrupt: - print("Server stopped at http://127.0.0.1:9001") + web.run_app(app, port=9001) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 8824e7ffb2b..1a0015e6755 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,9 +1,9 @@ import json +import pprint import subprocess -import sys -from collections.abc import Generator +from collections.abc import Iterator from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import pytest from pytest import TempPathFactory @@ -15,6 +15,9 @@ DockerException = python_on_whales.DockerException docker = python_on_whales.docker +# (Test number, test status, test report) +Result = tuple[str, str, dict[str, object] | None] + @pytest.fixture(scope="session") def report_dir(tmp_path_factory: TempPathFactory) -> Path: @@ -22,15 +25,12 @@ def report_dir(tmp_path_factory: TempPathFactory) -> Path: @pytest.fixture(scope="session", autouse=True) -def build_autobahn_testsuite() -> Generator[None, None, None]: - try: - docker.build( - file="tests/autobahn/Dockerfile.autobahn", - tags=["autobahn-testsuite"], - context_path=".", - ) - except DockerException: - pytest.skip("The docker daemon is not running.") +def build_autobahn_testsuite() -> Iterator[None]: + docker.build( + file="tests/autobahn/Dockerfile.autobahn", + tags=["autobahn-testsuite"], + context_path=".", + ) try: yield @@ -38,25 +38,51 @@ def build_autobahn_testsuite() -> Generator[None, None, None]: docker.image.remove(x="autobahn-testsuite") -def get_failed_tests(report_path: str, name: str) -> list[dict[str, Any]]: - path = Path(report_path) - result_summary = json.loads((path / "index.json").read_text())[name] - failed_messages = [] - PASS = {"OK", "INFORMATIONAL"} - entry_fields = {"case", "description", "expectation", "expected", "received"} - for results in result_summary.values(): - if results["behavior"] in PASS and results["behaviorClose"] in PASS: - continue - report = json.loads((path / results["reportfile"]).read_text()) - failed_messages.append({field: report[field] for field in entry_fields}) - return failed_messages +def get_report(path: Path, result: dict[str, str]) -> dict[str, object] | None: + if result["behaviorClose"] == "OK": + return None + return json.loads((path / result["reportfile"]).read_text()) # type: ignore[no-any-return] + +def get_test_results(path: Path, name: str) -> tuple[Result, ...]: + results = json.loads((path / "index.json").read_text())[name] + return tuple( + (k, r["behaviorClose"], get_report(path, r)) for k, r in results.items() + ) -@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") -@pytest.mark.xfail + +def process_xfail( + results: tuple[Result, ...], xfail: dict[str, str] +) -> list[dict[str, object]]: + failed = [] + for number, status, details in results: + if number in xfail: + assert status not in {"OK", "INFORMATIONAL"} # Strict xfail + assert details is not None + if details["result"] == xfail[number]: + continue + if status not in {"OK", "INFORMATIONAL"}: # pragma: no cover + assert details is not None + pprint.pprint(details) + failed.append(details) + return failed + + +@pytest.mark.autobahn def test_client(report_dir: Path, request: pytest.FixtureRequest) -> None: + client = subprocess.Popen( + ( + "wait-for-it", + "-s", + "localhost:9001", + "--", + "coverage", + "run", + "-a", + "tests/autobahn/client/client.py", + ) + ) try: - print("Starting autobahn-testsuite server") autobahn_container = docker.run( detach=True, image="autobahn-testsuite", @@ -64,53 +90,64 @@ def test_client(report_dir: Path, request: pytest.FixtureRequest) -> None: publish=[(9001, 9001)], remove=True, volumes=[ - (f"{request.path.parent}/client", "/config"), - (f"{report_dir}", "/reports"), + (request.path.parent / "client", "/config"), + (report_dir, "/reports"), ], ) - print("Running aiohttp test client") - client = subprocess.Popen( - ["wait-for-it", "-s", "localhost:9001", "--"] - + [sys.executable] - + ["tests/autobahn/client/client.py"] - ) client.wait() finally: - print("Stopping client and server") client.terminate() client.wait() autobahn_container.stop() - failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp") - - assert not failed_messages, "\n".join( - "\n\t".join( - f"{field}: {msg[field]}" - for field in ("case", "description", "expectation", "expected", "received") - ) - for msg in failed_messages - ) - - -@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") -@pytest.mark.xfail + results = get_test_results(report_dir / "clients", "aiohttp") + xfail = { + "3.4": "Actual events match at least one expected.", + "7.9.5": "The close code should have been 1002 or empty", + "9.1.4": "Did not receive message within 100 seconds.", + "9.1.5": "Did not receive message within 100 seconds.", + "9.1.6": "Did not receive message within 100 seconds.", + "9.2.4": "Did not receive message within 10 seconds.", + "9.2.5": "Did not receive message within 100 seconds.", + "9.2.6": "Did not receive message within 100 seconds.", + "9.3.1": "Did not receive message within 100 seconds.", + "9.3.2": "Did not receive message within 100 seconds.", + "9.3.3": "Did not receive message within 100 seconds.", + "9.3.4": "Did not receive message within 100 seconds.", + "9.3.5": "Did not receive message within 100 seconds.", + "9.3.6": "Did not receive message within 100 seconds.", + "9.3.7": "Did not receive message within 100 seconds.", + "9.3.8": "Did not receive message within 100 seconds.", + "9.3.9": "Did not receive message within 100 seconds.", + "9.4.1": "Did not receive message within 100 seconds.", + "9.4.2": "Did not receive message within 100 seconds.", + "9.4.3": "Did not receive message within 100 seconds.", + "9.4.4": "Did not receive message within 100 seconds.", + "9.4.5": "Did not receive message within 100 seconds.", + "9.4.6": "Did not receive message within 100 seconds.", + "9.4.7": "Did not receive message within 100 seconds.", + "9.4.8": "Did not receive message within 100 seconds.", + "9.4.9": "Did not receive message within 100 seconds.", + } + assert not process_xfail(results, xfail) + + +@pytest.mark.autobahn def test_server(report_dir: Path, request: pytest.FixtureRequest) -> None: + server = subprocess.Popen( + ("coverage", "run", "-a", "tests/autobahn/server/server.py") + ) try: - print("Starting aiohttp test server") - server = subprocess.Popen( - [sys.executable] + ["tests/autobahn/server/server.py"] - ) - print("Starting autobahn-testsuite client") docker.run( image="autobahn-testsuite", name="autobahn", remove=True, volumes=[ - (f"{request.path.parent}/server", "/config"), - (f"{report_dir}", "/reports"), + (request.path.parent / "server", "/config"), + (report_dir, "/reports"), ], - networks=["host"], - command=[ + networks=("host",), + command=( "wait-for-it", "-s", "localhost:9001", @@ -120,19 +157,38 @@ def test_server(report_dir: Path, request: pytest.FixtureRequest) -> None: "fuzzingclient", "--spec", "/config/fuzzingclient.json", - ], + ), ) finally: - print("Stopping client and server") server.terminate() server.wait() - failed_messages = get_failed_tests(f"{report_dir}/servers", "AutobahnServer") - - assert not failed_messages, "\n".join( - "\n\t".join( - f"{field}: {msg[field]}" - for field in ("case", "description", "expectation", "expected", "received") - ) - for msg in failed_messages - ) + results = get_test_results(report_dir / "servers", "AutobahnServer") + xfail = { + "7.9.5": "The close code should have been 1002 or empty", + "9.1.4": "Did not receive message within 100 seconds.", + "9.1.5": "Did not receive message within 100 seconds.", + "9.1.6": "Did not receive message within 100 seconds.", + "9.2.4": "Did not receive message within 10 seconds.", + "9.2.5": "Did not receive message within 100 seconds.", + "9.2.6": "Did not receive message within 100 seconds.", + "9.3.1": "Did not receive message within 100 seconds.", + "9.3.2": "Did not receive message within 100 seconds.", + "9.3.3": "Did not receive message within 100 seconds.", + "9.3.4": "Did not receive message within 100 seconds.", + "9.3.5": "Did not receive message within 100 seconds.", + "9.3.6": "Did not receive message within 100 seconds.", + "9.3.7": "Did not receive message within 100 seconds.", + "9.3.8": "Did not receive message within 100 seconds.", + "9.3.9": "Did not receive message within 100 seconds.", + "9.4.1": "Did not receive message within 100 seconds.", + "9.4.2": "Did not receive message within 100 seconds.", + "9.4.3": "Did not receive message within 100 seconds.", + "9.4.4": "Did not receive message within 100 seconds.", + "9.4.5": "Did not receive message within 100 seconds.", + "9.4.6": "Did not receive message within 100 seconds.", + "9.4.7": "Did not receive message within 100 seconds.", + "9.4.8": "Did not receive message within 100 seconds.", + "9.4.9": "Did not receive message within 100 seconds.", + } + assert not process_xfail(results, xfail)