Rework autobahn tests (aio-libs#12173)

Author: Dreamsorcerer

.github/workflows/ci-cd.yml

@@ -237,7 +237,7 @@ jobs:
         PIP_USER: 1
       run: >-
         PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}"
-        pytest --junitxml=junit.xml
+        pytest --junitxml=junit.xml -m 'not dev_mode and not autobahn'
       shell: bash
     - name: Re-run the failing tests with maximum verbosity
       if: failure()
@@ -280,6 +280,98 @@ jobs:
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
 
+  autobahn:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+
+    name: Autobahn testsuite
+    needs: gen_llhttp
+    strategy:
+      matrix:
+        pyver: ['3.14']
+        no-extensions: ['']
+        os: [ubuntu]
+      fail-fast: true
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Setup Python ${{ matrix.pyver }}
+      id: python-install
+      uses: actions/setup-python@v6
+      with:
+        allow-prereleases: true
+        python-version: ${{ matrix.pyver }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "dir=$(pip cache dir)" >> "${GITHUB_OUTPUT}"
+      shell: bash
+    - name: Cache PyPI
+      uses: actions/cache@v5.0.3
+      with:
+        key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }}
+        path: ${{ steps.pip-cache.outputs.dir }}
+        restore-keys: |
+            pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-
+    - name: Update pip, wheel, setuptools, build, twine
+      run: |
+        python -m pip install -U pip wheel setuptools build twine
+    - name: Install dependencies
+      env:
+        DEPENDENCY_GROUP: test${{ endsWith(matrix.pyver, 't') && '-ft' || '' }}
+      run: |
+        python -Im pip install -r requirements/${{ env.DEPENDENCY_GROUP }}.in -c requirements/${{ env.DEPENDENCY_GROUP }}.txt
+    - name: Restore llhttp generated files
+      if: ${{ matrix.no-extensions == '' }}
+      uses: actions/download-artifact@v8
+      with:
+        name: llhttp
+        path: vendor/llhttp/build/
+    - name: Cythonize
+      if: ${{ matrix.no-extensions == '' }}
+      run: |
+        make cythonize
+    - name: Install self
+      env:
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
+      run: python -m pip install -e .
+    - name: Run unittests
+      env:
+        COLOR: yes
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
+        PIP_USER: 1
+      run: >-
+        PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}"
+        pytest --junitxml=junit.xml --numprocesses=0 -m autobahn
+      shell: bash
+    - name: Turn coverage into xml
+      env:
+        COLOR: 'yes'
+        PIP_USER: 1
+      run: |
+        python -m coverage xml
+    - name: Upload coverage
+      uses: codecov/codecov-action@v5
+      with:
+        files: ./coverage.xml
+        flags: >-
+          CI-GHA,OS-${{
+            runner.os
+          }},VM-${{
+            matrix.os
+          }},Py-${{
+            steps.python-install.outputs.python-version
+          }}
+        token: ${{ secrets.CODECOV_TOKEN }}
+    - name: Upload test results to Codecov
+      if: ${{ !cancelled() }}
+      uses: codecov/test-results-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
   benchmark:
     name: Benchmark
     needs:
@@ -330,20 +422,21 @@ jobs:
     needs:
     - lint
     - test
+    - autobahn
 
     runs-on: ubuntu-latest
 
     steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}
     - name: Trigger codecov notification
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: true
         run_command: send-notifications
-    - name: Decide whether the needed jobs succeeded or failed
-      uses: re-actors/alls-green@release/v1
-      with:
-        jobs: ${{ toJSON(needs) }}
 
   pre-deploy:
     name: Pre-Deploy

CHANGES/12173.contrib.rst

@@ -0,0 +1 @@
+Fixed and reworked ``autobahn`` tests -- by :user:`Dreamsorcerer`.

setup.cfg

@@ -61,8 +61,7 @@ addopts =
     --cov=aiohttp
     --cov=tests/
 
-    # run tests that are not marked with dev_mode
-    -m "not dev_mode"
+    -m "not dev_mode and not autobahn and not internal"
 filterwarnings =
     error
     ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning
@@ -95,6 +94,7 @@ minversion = 3.8.2
 testpaths = tests/
 xfail_strict = true
 markers =
+    autobahn: Autobahn testsuite. Should be run as a separate job.
     dev_mode: mark test to run in dev mode.
     internal: tests which may cause issues for packagers, but should be run in aiohttp's CI.
     skip_blockbuster: mark test to skip the blockbuster fixture.

tests/autobahn/client/client.py

@@ -2,42 +2,31 @@
 
 import asyncio
 
-import aiohttp
+from aiohttp import ClientSession, WSMsgType
 
 
 async def client(url: str, name: str) -> None:
-    async with aiohttp.ClientSession() as session:
-        async with session.ws_connect(url + "/getCaseCount") as ws:
+    async with ClientSession(base_url=url) as session:
+        async with session.ws_connect("/getCaseCount") as ws:
             msg = await ws.receive()
-            assert msg.type is aiohttp.WSMsgType.TEXT
+            assert msg.type is WSMsgType.TEXT
             num_tests = int(msg.data)
-            print("running %d cases" % num_tests)
 
         for i in range(1, num_tests + 1):
-            print("running test case:", i)
-            text_url = url + "/runCase?case=%d&agent=%s" % (i, name)
-            async with session.ws_connect(text_url) as ws:
+            async with session.ws_connect(
+                "/runCase", params={"case": i, "agent": name}
+            ) as ws:
                 async for msg in ws:
-                    if msg.type is aiohttp.WSMsgType.TEXT:
+                    if msg.type is WSMsgType.TEXT:
                         await ws.send_str(msg.data)
-                    elif msg.type is aiohttp.WSMsgType.BINARY:
+                    elif msg.type is WSMsgType.BINARY:
                         await ws.send_bytes(msg.data)
                     else:
                         break
 
-        url = url + "/updateReports?agent=%s" % name
-        async with session.ws_connect(url) as ws:
-            print("finally requesting %s" % url)
+        async with session.ws_connect("/updateReports", params={"agent": name}) as ws:
+            pass
 
 
-async def run(url: str, name: str) -> None:
-    try:
-        await client(url, name)
-    except Exception:
-        import traceback
-
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    asyncio.run(run("http://localhost:9001", "aiohttp"))
+if __name__ == "__main__":  # pragma: no branch
+    asyncio.run(client("http://localhost:9001", "aiohttp"))

tests/autobahn/client/fuzzingserver.json

@@ -1,12 +1,11 @@
-
 {
    "url": "ws://localhost:9001",
 
-   "options": {"failByDrop": false},
+   "options": {"failByDrop": true},
    "outdir": "./reports/clients",
    "webport": 8080,
 
    "cases": ["*"],
-   "exclude-cases": ["12.*", "13.*"],
+   "exclude-cases": [],
    "exclude-agent-cases": {}
 }

tests/autobahn/server/fuzzingclient.json

@@ -1,16 +1,16 @@
 {
-    "options": { "failByDrop": false },
+    "options": {"failByDrop": true},
     "outdir": "./reports/servers",
 
     "servers": [
         {
             "agent": "AutobahnServer",
             "url": "ws://localhost:9001",
-            "options": { "version": 18 }
+            "options": {"version": 18}
         }
     ],
 
     "cases": ["*"],
-    "exclude-cases": ["12.*", "13.*"],
+    "exclude-cases": [],
     "exclude-agent-cases": {}
 }

tests/autobahn/server/server.py

@@ -9,47 +9,33 @@
 
 async def wshandler(request: web.Request) -> web.WebSocketResponse:
     ws = web.WebSocketResponse(autoclose=False)
-    is_ws = ws.can_prepare(request)
-    if not is_ws:
-        raise web.HTTPBadRequest()
-
     await ws.prepare(request)
 
     request.app[websockets].append(ws)
 
-    while True:
-        msg = await ws.receive()
-
+    async for msg in ws:
         if msg.type is web.WSMsgType.TEXT:
             await ws.send_str(msg.data)
         elif msg.type is web.WSMsgType.BINARY:
             await ws.send_bytes(msg.data)
-        elif msg.type is web.WSMsgType.CLOSE:
-            await ws.close()
-            break
         else:
             break
 
     return ws
 
 
 async def on_shutdown(app: web.Application) -> None:
-    ws_list = app[websockets]
-    for ws in set(ws_list):
+    for ws in app[websockets]:
         await ws.close(code=WSCloseCode.GOING_AWAY, message=b"Server shutdown")
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no branch
     logging.basicConfig(
         level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s"
     )
 
     app = web.Application()
-    l: list[web.WebSocketResponse] = []
-    app[websockets] = l
+    app[websockets] = []
     app.router.add_route("GET", "/", wshandler)
     app.on_shutdown.append(on_shutdown)
-    try:
-        web.run_app(app, port=9001)
-    except KeyboardInterrupt:
-        print("Server stopped at http://127.0.0.1:9001")
+    web.run_app(app, port=9001)