From 7ca1e0bbf2008d8b32b11944500d184ddeb8f896 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 12 May 2026 10:44:11 +0200 Subject: [PATCH 1/3] ci: Add package pre-publish check --- .github/workflows/manual_release_beta.yaml | 20 +++++++++++++++++++ .github/workflows/manual_release_stable.yaml | 21 ++++++++++++++++++++ .github/workflows/on_pull_request.yaml | 21 ++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/.github/workflows/manual_release_beta.yaml b/.github/workflows/manual_release_beta.yaml index f6ce0d81ce..3373541f83 100644 --- a/.github/workflows/manual_release_beta.yaml +++ b/.github/workflows/manual_release_beta.yaml @@ -60,6 +60,26 @@ jobs: version_number: ${{ needs.release_prepare.outputs.version_number }} ref: ${{ needs.changelog_update.outputs.changelog_commitish }} + - name: Verify built package + uses: apify/workflows/python-package-check@main + with: + package_name: crawlee + src_package_dir: src/crawlee + dist_dir: dist + python_version: "3.14" + extras: all + smoke_code: | + from crawlee.crawlers import ( + HttpCrawler, BeautifulSoupCrawler, ParselCrawler, + PlaywrightCrawler, AdaptivePlaywrightCrawler, + ) + from crawlee.storages import Dataset, KeyValueStore, RequestQueue + from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient + from crawlee import Request + HttpCrawler() + BeautifulSoupCrawler() + ParselCrawler() + # Publish the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/manual_release_stable.yaml b/.github/workflows/manual_release_stable.yaml index 2f5dca51d9..d0ca91d005 100644 --- a/.github/workflows/manual_release_stable.yaml +++ b/.github/workflows/manual_release_stable.yaml @@ -97,6 +97,27 @@ jobs: is_prerelease: "" version_number: ${{ needs.release_prepare.outputs.version_number }} ref: ${{ needs.changelog_update.outputs.changelog_commitish }} + + - name: Verify built package + uses: apify/workflows/python-package-check@main + with: + package_name: crawlee + src_package_dir: src/crawlee + dist_dir: dist + python_version: "3.14" + extras: all + smoke_code: | + from crawlee.crawlers import ( + HttpCrawler, BeautifulSoupCrawler, ParselCrawler, + PlaywrightCrawler, AdaptivePlaywrightCrawler, + ) + from crawlee.storages import Dataset, KeyValueStore, RequestQueue + from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient + from crawlee import Request + HttpCrawler() + BeautifulSoupCrawler() + ParselCrawler() + # Publish the package to PyPI using PyPA official GitHub action with OIDC authentication. - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index a13e725f3c..2778532b30 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -25,6 +25,27 @@ jobs: name: Code checks uses: ./.github/workflows/_check_code.yaml + package_check: + name: Package check + uses: apify/workflows/.github/workflows/python_package_check.yaml@main + with: + package_name: crawlee + src_package_dir: src/crawlee + dist_dir: dist + python_version: "3.14" + extras: all + smoke_code: | + from crawlee.crawlers import ( + HttpCrawler, BeautifulSoupCrawler, ParselCrawler, + PlaywrightCrawler, AdaptivePlaywrightCrawler, + ) + from crawlee.storages import Dataset, KeyValueStore, RequestQueue + from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient + from crawlee import Request + HttpCrawler() + BeautifulSoupCrawler() + ParselCrawler() + tests: name: Tests uses: ./.github/workflows/_tests.yaml From 17c95fe65f7688feebe7bec798ed7e65edea2a94 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 14 May 2026 11:30:22 +0200 Subject: [PATCH 2/3] ci: inline package-check recipe, drop shared reusable workflow indirection Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/on_pull_request.yaml | 50 ++++++++++++++++---------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index 2778532b30..b78844cf74 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -27,24 +27,38 @@ jobs: package_check: name: Package check - uses: apify/workflows/.github/workflows/python_package_check.yaml@main - with: - package_name: crawlee - src_package_dir: src/crawlee - dist_dir: dist - python_version: "3.14" - extras: all - smoke_code: | - from crawlee.crawlers import ( - HttpCrawler, BeautifulSoupCrawler, ParselCrawler, - PlaywrightCrawler, AdaptivePlaywrightCrawler, - ) - from crawlee.storages import Dataset, KeyValueStore, RequestQueue - from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient - from crawlee import Request - HttpCrawler() - BeautifulSoupCrawler() - ParselCrawler() + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.14" + + - name: Build sdist and wheel + run: uv run poe build + + - name: Verify built package + uses: apify/workflows/python-package-check@main + with: + package_name: crawlee + src_package_dir: src/crawlee + dist_dir: dist + python_version: "3.14" + extras: all + smoke_code: | + from crawlee.crawlers import ( + HttpCrawler, BeautifulSoupCrawler, ParselCrawler, + PlaywrightCrawler, AdaptivePlaywrightCrawler, + ) + from crawlee.storages import Dataset, KeyValueStore, RequestQueue + from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient + from crawlee import Request + HttpCrawler() + BeautifulSoupCrawler() + ParselCrawler() tests: name: Tests From 92ad3416bb33e130114d8f19b422c4b766af4adb Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 14 May 2026 11:39:03 +0200 Subject: [PATCH 3/3] ci: extract package_check job into _check_package.yaml Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/_check_package.yaml | 47 ++++++++++++++++++++++++++ .github/workflows/on_pull_request.yaml | 33 +----------------- 2 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/_check_package.yaml diff --git a/.github/workflows/_check_package.yaml b/.github/workflows/_check_package.yaml new file mode 100644 index 0000000000..d4b5981b0f --- /dev/null +++ b/.github/workflows/_check_package.yaml @@ -0,0 +1,47 @@ +name: Package check + +on: + # Runs when manually triggered from the GitHub UI. + workflow_dispatch: + + # Runs when invoked by another workflow. + workflow_call: + +permissions: + contents: read + +jobs: + package_check: + name: Package check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.14" + + - name: Build sdist and wheel + run: uv run poe build + + - name: Verify built package + uses: apify/workflows/python-package-check@main + with: + package_name: crawlee + src_package_dir: src/crawlee + dist_dir: dist + python_version: "3.14" + extras: all + smoke_code: | + from crawlee.crawlers import ( + HttpCrawler, BeautifulSoupCrawler, ParselCrawler, + PlaywrightCrawler, AdaptivePlaywrightCrawler, + ) + from crawlee.storages import Dataset, KeyValueStore, RequestQueue + from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient + from crawlee import Request + HttpCrawler() + BeautifulSoupCrawler() + ParselCrawler() diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index b78844cf74..991dabaae8 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -27,38 +27,7 @@ jobs: package_check: name: Package check - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Set up uv package manager - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: "3.14" - - - name: Build sdist and wheel - run: uv run poe build - - - name: Verify built package - uses: apify/workflows/python-package-check@main - with: - package_name: crawlee - src_package_dir: src/crawlee - dist_dir: dist - python_version: "3.14" - extras: all - smoke_code: | - from crawlee.crawlers import ( - HttpCrawler, BeautifulSoupCrawler, ParselCrawler, - PlaywrightCrawler, AdaptivePlaywrightCrawler, - ) - from crawlee.storages import Dataset, KeyValueStore, RequestQueue - from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient - from crawlee import Request - HttpCrawler() - BeautifulSoupCrawler() - ParselCrawler() + uses: ./.github/workflows/_check_package.yaml tests: name: Tests