diff --git a/.github/workflows/docker-image-test.yml b/.github/workflows/docker-image-test.yml index bd3d8d00c..eab5c9ff2 100644 --- a/.github/workflows/docker-image-test.yml +++ b/.github/workflows/docker-image-test.yml @@ -61,11 +61,24 @@ jobs: fail-fast: false matrix: include: - - { dockerfile: Dockerfile-15, target: "", name: 15 } - - { dockerfile: Dockerfile-17, target: "", name: 17 } - - { dockerfile: Dockerfile-orioledb-17, target: "", name: orioledb-17 } - - { dockerfile: Dockerfile-multigres, target: variant-17, name: multigres-17 } - - { dockerfile: Dockerfile-multigres, target: variant-orioledb-17, name: multigres-orioledb-17 } + # CHANGED: Dockerfile-15/17 replaced by parameterised Dockerfile-supabase. + # pg_version is passed as --build-arg PG_VERSION to select the PostgreSQL version. + - dockerfile: Dockerfile-supabase + name: 15 + pg_version: "15" + - dockerfile: Dockerfile-supabase + name: 17 + pg_version: "17" + - dockerfile: Dockerfile-orioledb-17 + name: orioledb-17 + pg_version: "17" + # CHANGED: base_dockerfile causes the build step to build the supabase base image + # locally first and pass it as SUPABASE_IMAGE. The variant-orioledb-17 entry was + # removed — that target does not exist in Dockerfile-multigres. + - dockerfile: Dockerfile-multigres + name: multigres-17 + pg_version: "17" + base_dockerfile: Dockerfile-supabase steps: - name: Checkout Repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -89,32 +102,45 @@ jobs: - name: Build Docker image run: | echo "Building ${{ matrix.name }}..." - TARGET_ARG="" - if [ -n "${{ matrix.target }}" ]; then - TARGET_ARG="--target ${{ matrix.target }}" + # CHANGED: pass PG_VERSION for Dockerfile-supabase and Dockerfile-multigres builds. + PG_VERSION_ARG="" + if [ -n "${{ matrix.pg_version }}" ]; then + PG_VERSION_ARG="--build-arg PG_VERSION=${{ matrix.pg_version }}" fi - docker build -f "${{ matrix.dockerfile }}" $TARGET_ARG \ + # CHANGED: layered images (multigres) need their base image available locally. + # Each matrix job runs on an isolated runner, so we build the base inline here + # and pass it as SUPABASE_IMAGE rather than pulling from a registry. + BASE_IMAGE_ARG="" + if [ -n "${{ matrix.base_dockerfile }}" ]; then + docker build -f "${{ matrix.base_dockerfile }}" \ + --build-arg PG_VERSION=${{ matrix.pg_version }} \ + --target production \ + -t "pg-docker-test:base-${{ matrix.name }}" \ + . + BASE_IMAGE_ARG="--build-arg SUPABASE_IMAGE=pg-docker-test:base-${{ matrix.name }}" + fi + docker build -f "${{ matrix.dockerfile }}" --target production $PG_VERSION_ARG $BASE_IMAGE_ARG \ -t "pg-docker-test:${{ matrix.name }}" \ -t "supabase-postgres:${{ matrix.name }}-analyze" \ . - name: Run image size analysis - if: ${{ matrix.target == '' }} + if: ${{ matrix.base_dockerfile == '' }} run: | echo "=== Image Size Analysis for ${{ matrix.name }} ===" - nix run --accept-flake-config .#image-size-analyzer -- --image Dockerfile-${{ matrix.name }} --no-build + nix run --accept-flake-config .#image-size-analyzer -- --image ${{ matrix.dockerfile }} --pg-version ${{ matrix.pg_version }} --no-build - name: Run Docker image tests - if: ${{ matrix.target == '' }} + if: ${{ matrix.base_dockerfile == '' }} run: | echo "=== Running tests for ${{ matrix.name }} ===" - nix run --accept-flake-config .#docker-image-test -- --no-build Dockerfile-${{ matrix.name }} + nix run --accept-flake-config .#docker-image-test -- --no-build --pg-version ${{ matrix.pg_version }} ${{ matrix.dockerfile }} - name: Run multigres Docker image tests - if: ${{ matrix.target != '' }} + if: ${{ matrix.base_dockerfile != '' }} run: | echo "=== Running tests for ${{ matrix.name }} ===" - nix run --accept-flake-config .#docker-image-test -- --no-build --target ${{ matrix.target }} ${{ matrix.dockerfile }} + nix run --accept-flake-config .#docker-image-test -- --no-build --target production ${{ matrix.dockerfile }} - name: Show container logs on failure if: failure() @@ -130,6 +156,7 @@ jobs: run: | docker ps -a --filter "name=pg-test-${{ matrix.name }}" -q | xargs -r docker rm -f || true docker rmi "pg-docker-test:${{ matrix.name }}" || true + docker rmi "pg-docker-test:base-${{ matrix.name }}" || true # CHANGED: remove ephemeral base image built for layered builds docker rmi "supabase-postgres:${{ matrix.name }}-analyze" || true skip-notification: diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index fccf54c27..bfb6d5cb7 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -19,6 +19,8 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} + base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} + layered_matrix: ${{ steps.set-matrix.outputs.layered_matrix }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -26,45 +28,20 @@ jobs: - name: Generate build matrix id: set-matrix run: | - nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) - let base_matrix = ($versions | each { |ver| - let version = ($ver | str trim) - let dockerfile = $"Dockerfile-($version)" - if ($dockerfile | path exists) { - { - version: $version, - dockerfile: $dockerfile, - target: "production" - } - } else { - null - } - } | compact) - - # Discover multigres variants by checking for matching targets in Dockerfile-multigres - let multigres_matrix = ($versions | each { |ver| - let version = ($ver | str trim) - let mg_version = $"multigres-($version)" - let mg_dockerfile = "Dockerfile-multigres" - let mg_target = $"variant-($version)" - if ($mg_dockerfile | path exists) and (open --raw $mg_dockerfile | str contains $"AS ($mg_target)") { - { - version: $mg_version, - dockerfile: $mg_dockerfile, - target: $mg_target - } - } else { - null - } - } | compact) - - let matrix = ($base_matrix | append $multigres_matrix) - - let matrix_config = { - include: $matrix - } - - $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' + nix run nixpkgs#nushell -- -c ' + let releases = (open ansible/vars.yml | get postgres_release) + let base = (open ansible/vars.yml | get release_matrix_base + | each { |e| $e | insert tag ($releases | get $e.release_key) }) + let layered = (open ansible/vars.yml | get release_matrix_layered + | each { |e| + let ver = ($releases | get $e.release_key) + $e | insert tag $"($ver)($e.tag_suffix)" | insert base_tag $ver + }) + let combined = ($base | append $layered) + $"base_matrix=({include: $base} | to json -r)" | save --append $env.GITHUB_OUTPUT + $"layered_matrix=({include: $layered} | to json -r)" | save --append $env.GITHUB_OUTPUT + $"matrix_config=({include: $combined} | to json -r)" | save --append $env.GITHUB_OUTPUT + ' build: needs: prepare strategy: @@ -89,11 +66,11 @@ jobs: | str join "\n" | save --append $env.GITHUB_OUTPUT ' - build_release_image: + build_base_images: needs: [prepare, build] strategy: matrix: - postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + postgres: ${{ fromJson(needs.prepare.outputs.base_matrix).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} timeout-minutes: 180 @@ -109,52 +86,56 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Get image tag - id: image - run: | - if [[ "${{ matrix.arch }}" == "arm64" ]]; then - pg_version=$(nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.postgres.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } - ') - echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT - else - pg_version=$(nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.postgres.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } - ') - echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT - fi - id: build uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 with: push: true build-args: | ${{ needs.build.outputs.build_args }} + PG_VERSION=${{ matrix.postgres.pg_version }} target: ${{ matrix.postgres.target }} - tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + tags: supabase/postgres:${{ matrix.postgres.tag }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} + + build_layered_images: + needs: [prepare, build, build_base_images] + strategy: + matrix: + postgres: ${{ fromJson(needs.prepare.outputs.layered_matrix).include }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} + timeout-minutes: 180 + steps: + - name: Checkout Repo + uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - uses: ./.github/actions/nix-install-ephemeral + - run: docker context create builders + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + with: + endpoint: builders + - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - id: build + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + PG_VERSION=${{ matrix.postgres.pg_version }} + SUPABASE_IMAGE=supabase/postgres:${{ matrix.postgres.base_tag }}_${{ matrix.arch }} + target: ${{ matrix.postgres.target }} + tags: supabase/postgres:${{ matrix.postgres.tag }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} merge_manifest: - needs: [prepare, build, build_release_image] + needs: [prepare, build, build_base_images, build_layered_images] strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} @@ -170,20 +151,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version - run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - let pg_version = if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' + run: echo "pg_version=supabase/postgres:${{ matrix.tag }}" >> $GITHUB_OUTPUT - name: Output version id: output_version run: | @@ -195,7 +163,7 @@ jobs: - name: Upload Results Artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merge_results-${{ matrix.version }} + name: merge_results-${{ matrix.tag }} path: results.txt if-no-files-found: warn - name: Merge multi-arch manifests diff --git a/.github/workflows/manual-docker-release.yml b/.github/workflows/manual-docker-release.yml index 22cad1642..0cc3816a3 100644 --- a/.github/workflows/manual-docker-release.yml +++ b/.github/workflows/manual-docker-release.yml @@ -1,4 +1,4 @@ -name: Manual Docker Artifacts Release +name: Manual Docker Artifacts Release on: workflow_dispatch: @@ -6,7 +6,7 @@ on: postgresVersion: description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78' required: false - + permissions: id-token: write contents: read @@ -16,6 +16,8 @@ jobs: runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} + base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} + layered_matrix: ${{ steps.set-matrix.outputs.layered_matrix }} steps: - uses: ./.github/actions/nix-install-ephemeral - name: Checkout Repo @@ -23,45 +25,20 @@ jobs: - name: Generate build matrix id: set-matrix run: | - nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) - let base_matrix = ($versions | each { |ver| - let version = ($ver | str trim) - let dockerfile = $"Dockerfile-($version)" - if ($dockerfile | path exists) { - { - version: $version, - dockerfile: $dockerfile, - target: "production" - } - } else { - null - } - } | compact) - - # Discover multigres variants by checking for matching targets in Dockerfile-multigres - let multigres_matrix = ($versions | each { |ver| - let version = ($ver | str trim) - let mg_version = $"multigres-($version)" - let mg_dockerfile = "Dockerfile-multigres" - let mg_target = $"variant-($version)" - if ($mg_dockerfile | path exists) and (open --raw $mg_dockerfile | str contains $"AS ($mg_target)") { - { - version: $mg_version, - dockerfile: $mg_dockerfile, - target: $mg_target - } - } else { - null - } - } | compact) - - let matrix = ($base_matrix | append $multigres_matrix) - - let matrix_config = { - include: $matrix - } - - $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' + nix run nixpkgs#nushell -- -c ' + let releases = (open ansible/vars.yml | get postgres_release) + let base = (open ansible/vars.yml | get release_matrix_base + | each { |e| $e | insert tag ($releases | get $e.release_key) }) + let layered = (open ansible/vars.yml | get release_matrix_layered + | each { |e| + let ver = ($releases | get $e.release_key) + $e | insert tag $"($ver)($e.tag_suffix)" | insert base_tag $ver + }) + let combined = ($base | append $layered) + $"base_matrix=({include: $base} | to json -r)" | save --append $env.GITHUB_OUTPUT + $"layered_matrix=({include: $layered} | to json -r)" | save --append $env.GITHUB_OUTPUT + $"matrix_config=({include: $combined} | to json -r)" | save --append $env.GITHUB_OUTPUT + ' build: needs: prepare strategy: @@ -73,24 +50,55 @@ jobs: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - uses: ./.github/actions/nix-install-ephemeral - - name: Set PostgreSQL version environment variable - run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV - - id: args run: | nix run nixpkgs#nushell -- -c ' - open ansible/vars.yml - | items { |key value| {name: $key, item: $value} } - | where { |it| ($it.item | describe) == "string" } - | each { |it| $"($it.name)=($it.item)" } - | str join "\n" + open ansible/vars.yml + | items { |key value| {name: $key, item: $value} } + | where { |it| ($it.item | describe) == "string" } + | each { |it| $"($it.name)=($it.item)" } + | str join "\n" | save --append $env.GITHUB_OUTPUT - ' - build_release_image: + ' + build_base_images: needs: [prepare, build] strategy: matrix: - postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + postgres: ${{ fromJson(needs.prepare.outputs.base_matrix).include }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && 'blacksmith-8vcpu-ubuntu-2404' || 'large-linux-arm' }} + timeout-minutes: 180 + steps: + - name: Checkout Repo + uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - uses: ./.github/actions/nix-install-ephemeral + - run: docker context create builders + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + with: + endpoint: builders + - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - id: build + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + PG_VERSION=${{ matrix.postgres.pg_version }} + target: ${{ matrix.postgres.target }} + tags: supabase/postgres:${{ inputs.postgresVersion || matrix.postgres.tag }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} + + build_layered_images: + needs: [prepare, build, build_base_images] + strategy: + matrix: + postgres: ${{ fromJson(needs.prepare.outputs.layered_matrix).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && 'blacksmith-8vcpu-ubuntu-2404' || 'large-linux-arm' }} timeout-minutes: 180 @@ -106,62 +114,23 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Get image tag - id: image - run: | - if [[ "${{ matrix.arch }}" == "arm64" ]]; then - pg_version=$(nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.postgres.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - let final_version = if "${{ inputs.postgresVersion }}" != "" { - "${{ inputs.postgresVersion }}" - } else { - $base_tag - } - if $is_multigres { $"($final_version)-multigres" | str trim } else { $final_version | str trim } - ') - echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT - else - pg_version=$(nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.postgres.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - let final_version = if "${{ inputs.postgresVersion }}" != "" { - "${{ inputs.postgresVersion }}" - } else { - $base_tag - } - if $is_multigres { $"($final_version)-multigres" | str trim } else { $final_version | str trim } - ') - echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT - fi - id: build uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 with: push: true build-args: | ${{ needs.build.outputs.build_args }} + PG_VERSION=${{ matrix.postgres.pg_version }} + SUPABASE_IMAGE=supabase/postgres:${{ inputs.postgresVersion || matrix.postgres.base_tag }}_${{ matrix.arch }} target: ${{ matrix.postgres.target }} - tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + tags: supabase/postgres:${{ inputs.postgresVersion != '' && format('{0}-multigres', inputs.postgresVersion) || matrix.postgres.tag }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} + merge_manifest: - needs: [prepare, build, build_release_image] + needs: [prepare, build, build_base_images, build_layered_images] strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} @@ -177,20 +146,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version - run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" - let is_multigres = ($version | str starts-with "multigres-") - let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } - let release_key = if ($base_version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($base_version)" - } - let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - let pg_version = if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' + run: echo "pg_version=supabase/postgres:${{ inputs.postgresVersion != '' && (contains(matrix.tag, '-multigres') && format('{0}-multigres', inputs.postgresVersion) || inputs.postgresVersion) || matrix.tag }}" >> $GITHUB_OUTPUT - name: Output version id: output_version run: | @@ -202,7 +158,7 @@ jobs: - name: Upload Results Artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merge_results-${{ matrix.version }} + name: merge_results-${{ matrix.tag }} path: results.txt if-no-files-found: warn - name: Merge multi-arch manifests @@ -228,12 +184,12 @@ jobs: nix run nixpkgs#nushell -- -c ' # Parse the matrix configuration directly let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) - + # Get versions directly from include array - let versions = ($matrix_config.include | get version) - + let versions = ($matrix_config.include | get tag) + echo "Versions: $versions" - + # Convert the versions to a comma-separated string let versions_str = ($versions | str join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV @@ -249,7 +205,7 @@ jobs: # Get all results files and process them in one go let files = (ls **/results.txt | get name) echo $"Found files: ($files)" - + let matrix = { include: ( $files @@ -257,17 +213,17 @@ jobs: | each { |content| $content | lines } # Split into lines | flatten # Flatten the nested lists | where { |line| $line != "" } # Filter empty lines - | each { |line| + | each { |line| # Extract just the version part after the last colon let version = ($line | parse "supabase/postgres:{version}" | get version.0) {version: $version} } ) } - + let json_output = ($matrix | to json -r) # -r for raw output echo $"Debug output: ($json_output)" - + $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT ' - name: Debug Combined Results @@ -285,5 +241,5 @@ jobs: matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ inputs.postgresVersion != '' && (contains(matrix.version, '-multigres') && format('{0}-multigres', inputs.postgresVersion) || inputs.postgresVersion) || matrix.version }} + version: ${{ matrix.version }} secrets: inherit diff --git a/Dockerfile-multigres b/Dockerfile-multigres index bddb8407b..43cb2110d 100644 --- a/Dockerfile-multigres +++ b/Dockerfile-multigres @@ -1,321 +1,71 @@ -# syntax=docker/dockerfile:1.6 -# Multigres PostgreSQL 17 variants — vanilla and OrioleDB +# syntax=docker/dockerfile:1.2 +# 1.2: minimum version for COPY --chmod +# Multigres PostgreSQL image — layered on top of the supabase base image. # -# Build targets: -# docker build -f Dockerfile-multigres --target variant-17 -t multigres-17 . -# docker build -f Dockerfile-multigres --target variant-orioledb-17 -t multigres-orioledb-17 . - -#################### -# Stage 0: Nix base — shared Alpine + Nix setup for all builders -#################### -FROM alpine:3.23 AS nix-base - -RUN apk add --no-cache \ - bash \ - coreutils \ - curl \ - shadow \ - sudo \ - xz - -RUN addgroup -S postgres && \ - adduser -S -h /var/lib/postgresql -s /bin/bash -G postgres postgres && \ - addgroup -S wal-g && \ - adduser -S -s /bin/bash -G wal-g wal-g - -RUN cat < /tmp/extra-nix.conf -extra-experimental-features = nix-command flakes -extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com -extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= -EOF -RUN curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --no-channel-add --yes --nix-extra-conf-file /tmp/extra-nix.conf -ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" -RUN nix --version - -WORKDIR /nixpg -COPY . . - -#################### -# Stage 1a: Nix builder — PostgreSQL 17 -#################### -FROM nix-base AS nix-builder-17 - -RUN nix profile add path:.#psql_17_slim/bin path:.#pg-backrest path:.#pgctld - -RUN nix store gc - -RUN nix profile add path:.#supabase-groonga && \ - mkdir -p /tmp/groonga-plugins && \ - cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ - -RUN nix store gc - -#################### -# Stage 1b: Nix builder — OrioleDB 17 -#################### -FROM nix-base AS nix-builder-orioledb-17 - -RUN nix profile add path:.#psql_orioledb-17_slim/bin path:.#pg-backrest path:.#pgctld - -RUN nix store gc - -RUN nix profile add path:.#supabase-groonga && \ - mkdir -p /tmp/groonga-plugins && \ - cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ - -RUN nix store gc - -#################### -# Stage 2: Gosu builder -#################### -FROM alpine:3.23 AS gosu-builder - -ARG TARGETARCH -ARG GOSU_VERSION=1.19 -ARG GO_VERSION=1.26.1 - -RUN apk add --no-cache curl git - -# Install Go -RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" | tar -C /usr/local -xz -ENV PATH="/usr/local/go/bin:${PATH}" - -# Build gosu from source -RUN git clone --depth 1 --branch "${GOSU_VERSION}" https://github.com/tianon/gosu.git /gosu && \ - cd /gosu && \ - CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/local/bin/gosu . && \ - chmod +x /usr/local/bin/gosu +# The supabase image must be built first: +# docker build -f Dockerfile-supabase -t supabase-postgres:17 . +# docker build -f Dockerfile-multigres -t multigres:17 . +# +# Override the base image at build time: +# docker build -f Dockerfile-multigres \ +# --build-arg SUPABASE_IMAGE=registry.example.com/supabase-postgres:17 \ +# -t multigres:17 . #################### -# Stage 3: Shared base — runtime Alpine + config + migrations -# Both variants inherit from this stage. No /nix dependency here. +# Stage 1: pgctld builder #################### -FROM alpine:3.23 AS base - -RUN apk add --no-cache \ - bash \ - curl \ - openssh \ - procps \ - shadow \ - su-exec \ - tzdata \ - musl-locales \ - musl-locales-lang \ - && rm -rf /var/cache/apk/* - -RUN addgroup -S postgres && \ - adduser -S -G postgres -h /var/lib/postgresql -s /bin/bash postgres && \ - addgroup -S wal-g && \ - adduser -S -G wal-g -s /bin/bash wal-g && \ - adduser postgres wal-g - -RUN mkdir -p \ - /usr/lib/postgresql/bin \ - /usr/lib/postgresql/share/postgresql \ - /usr/share/postgresql \ - /var/lib/postgresql/data \ - /var/log/postgresql \ - /var/run/postgresql \ - /etc/postgresql \ - /etc/postgresql-custom \ - && chown -R postgres:postgres \ - /usr/lib/postgresql \ - /var/lib/postgresql \ - /usr/share/postgresql \ - /var/log/postgresql \ - /var/run/postgresql \ - /etc/postgresql \ - /etc/postgresql-custom - -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-jsonlog.conf /etc/postgresql/logging.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf -COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf -COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +FROM golang:1.24-alpine AS pgctld-builder -# Apply settings shared by all variants: -# - enable unix socket, session preload, config includes -# - strip timescaledb and pgsodium (absent from all pg17 builds) -# - comment out db_user_namespace (not supported in pg17) -RUN sed -i \ - -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ - -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ - -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ - # skip wal-g - unused by multigres - # -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" \ - -e "s/ timescaledb,//g" \ - -e "s/ pgsodium,//g" \ - -e "s/db_user_namespace = off/#db_user_namespace = off/g" \ - # skip - managed by pgctld - -e "s|^data_directory |#data_directory |g" \ - /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - chown -R postgres:postgres /etc/postgresql-custom && \ - mkdir -p /usr/share/postgresql/extension/ && \ - ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ - chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh +# Pinned to the commit that introduced --pg-initdb-sql-dirs (MUL-484) +ARG PGCTLD_REV=1e3bad798972600778ee27eb08ab7d34cc8be8e9 -COPY migrations/db /docker-entrypoint-initdb.d/ -COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql -COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql +RUN apk add --no-cache git -ENV PGDATA=/var/lib/postgresql/data -ENV POSTGRES_HOST=/var/run/postgresql -ENV POSTGRES_USER=supabase_admin -ENV POSTGRES_DB=postgres -ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 -ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins +RUN git clone https://github.com/multigres/multigres.git /multigres && \ + cd /multigres && \ + git checkout ${PGCTLD_REV} && \ + # Copy pico CSS assets before build (mirrors pgctld.nix preBuild step) + cp external/pico/pico.* go/common/web/templates/css/ 2>/dev/null || true && \ + CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/local/bin/pgctld ./go/cmd/pgctld #################### -# Stage 4a: variant-17 — PostgreSQL 17 (vanilla) +# Stage 2: Multigres image #################### -FROM base AS variant-17 +# SUPABASE_IMAGE is set by the release workflow via --build-arg, derived from +# PG_VERSION and the release matrix in ansible/vars.yml. +ARG SUPABASE_IMAGE=supabase-postgres:17 +FROM ${SUPABASE_IMAGE} AS production -COPY --from=nix-builder-17 /nix /nix -COPY --from=nix-builder-17 /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins -COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu +# Install pgbackrest (available in Alpine community repo) +RUN apk add --no-cache pgbackrest -RUN for f in /nix/var/nix/profiles/default/bin/*; do \ - ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ - ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ - done && \ - ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ - ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ - ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true +# Copy pgctld binary; keep it separate so the wrapper script can reference it cleanly +COPY --from=pgctld-builder /usr/local/bin/pgctld /usr/local/bin/pgctld-bin -RUN chown -R postgres:postgres /usr/lib/postgresql && \ - chown -R postgres:postgres /usr/share/postgresql - -# Can't use /etc/pgctld because it's a mount point +# pgctld config template — /etc/pgctld is a mount point in k8s so use a custom dir COPY docker/pgctld/postgresql.conf.tmpl /etc/pgctld-custom/postgresql.conf.tmpl -# Wrapper: injects --postgres-config-template on every pgctld call AND bridges -# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so -# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper. -COPY --chmod=755 docker/pgctld/pgctld-wrapper /usr/local/bin/pgctld -ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/postgresql.conf.tmpl - -# Strip extensions absent from pg17 vanilla build -RUN sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql-custom/supautils.conf - -# Generate a single SQL manifest that pgctld runs via --init-db-sql-file after initdb. -# Creates the postgres role, runs init-scripts as postgres (matching migrate.sh), -# then runs migrations as supabase_admin. -RUN set -e && \ - manifest=/docker-entrypoint-initdb.d/multigres-init.sql && \ - printf -- "-- Auto-generated: run init-scripts and migrations after initdb\n" > "$manifest" && \ - printf "DO \$\$ BEGIN\n IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'postgres') THEN\n CREATE ROLE postgres SUPERUSER LOGIN;\n END IF;\nEND \$\$;\n" >> "$manifest" && \ - printf "ALTER DATABASE postgres OWNER TO postgres;\n\n" >> "$manifest" && \ - printf "SET SESSION AUTHORIZATION postgres;\n" >> "$manifest" && \ - for f in $(ls /docker-entrypoint-initdb.d/init-scripts/*.sql 2>/dev/null | sort); do \ - printf '\\ir init-scripts/%s\n' "$(basename "$f")" >> "$manifest"; \ - done && \ - printf "\nRESET SESSION AUTHORIZATION;\n\n" >> "$manifest" && \ - for f in $(ls /docker-entrypoint-initdb.d/migrations/*.sql 2>/dev/null | sort); do \ - printf '\\ir migrations/%s\n' "$(basename "$f")" >> "$manifest"; \ - done && \ - chown postgres:postgres "$manifest" - -ENV POSTGRES_INITDB_SQL_FILES=/docker-entrypoint-initdb.d/multigres-init.sql -ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" -ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive - -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD pg_isready -U postgres -h localhost || true -STOPSIGNAL SIGINT -USER postgres -EXPOSE 5432 - -ENTRYPOINT ["tail"] -CMD ["-f", "/dev/null"] - -#################### -# Stage 4b: variant-orioledb-17 — PostgreSQL 17 with OrioleDB -#################### -FROM base AS variant-orioledb-17 - -COPY --from=nix-builder-orioledb-17 /nix /nix -COPY --from=nix-builder-orioledb-17 /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins -COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu - -RUN for f in /nix/var/nix/profiles/default/bin/*; do \ - ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ - ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ - done && \ - ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ - ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ - ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true - -RUN chown -R postgres:postgres /usr/lib/postgresql && \ - chown -R postgres:postgres /usr/share/postgresql - -# Strip extensions absent from orioledb-17 build -RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' \ - /etc/postgresql-custom/supautils.conf - -# Add orioledb to shared_preload_libraries and configure as default table access method -# Rewind: 1200s window, 100k transactions, 1280 x 8KB = 10MB buffer -RUN sed -i "s/\(shared_preload_libraries.*\)'/\1, orioledb'/" /etc/postgresql/postgresql.conf && \ - echo "default_table_access_method = 'orioledb'" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.enable_rewind = true" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_max_time = 1200" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_max_transactions = 100000" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_buffers = 1280" >> /etc/postgresql/postgresql.conf - -# Register orioledb before initdb migrations run -RUN echo "CREATE EXTENSION orioledb;" > /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql && \ - chown postgres:postgres /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql - -# pgctld calls initdb without locale flags, so it picks up LANG from the environment. -# OrioleDB requires C, POSIX, or ICU collations; override the base stage's en_US.UTF-8. -ENV LANG=C -ENV LC_ALL=C -ENV LANGUAGE=C - -# Can't use /etc/pgctld because it's a mount point -COPY docker/pgctld/orioledb-postgresql.conf.tmpl /etc/pgctld-custom/orioledb-postgresql.conf.tmpl - -# Wrapper: injects --postgres-config-template on every pgctld call AND bridges -# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so -# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper. -COPY --chmod=755 docker/pgctld/pgctld-wrapper /usr/local/bin/pgctld -ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/orioledb-postgresql.conf.tmpl +# Wrapper: injects --postgres-config-template on every pgctld call so unmodified +# k8s manifests and local provisioner commands work without extra flags +COPY --chmod=0755 docker/pgctld/pgctld /usr/local/bin/pgctld -# Regenerate manifest after orioledb added 00-pre-init.sql to init-scripts/ -RUN set -e && \ - manifest=/docker-entrypoint-initdb.d/multigres-init.sql && \ - printf -- "-- Auto-generated: run init-scripts and migrations after initdb\n" > "$manifest" && \ - printf "DO \$\$ BEGIN\n IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'postgres') THEN\n CREATE ROLE postgres SUPERUSER LOGIN;\n END IF;\nEND \$\$;\n" >> "$manifest" && \ - printf "ALTER DATABASE postgres OWNER TO postgres;\n\n" >> "$manifest" && \ - printf "SET SESSION AUTHORIZATION postgres;\n" >> "$manifest" && \ - for f in $(ls /docker-entrypoint-initdb.d/init-scripts/*.sql 2>/dev/null | sort); do \ - printf '\\ir init-scripts/%s\n' "$(basename "$f")" >> "$manifest"; \ - done && \ - printf "\nRESET SESSION AUTHORIZATION;\n\n" >> "$manifest" && \ - for f in $(ls /docker-entrypoint-initdb.d/migrations/*.sql 2>/dev/null | sort); do \ - printf '\\ir migrations/%s\n' "$(basename "$f")" >> "$manifest"; \ - done && \ - chown postgres:postgres "$manifest" +# /etc/postgresql/postgresql.conf is not modified here: pgctld renders its own +# config from postgresql.conf.tmpl and passes it directly to PostgreSQL, so the +# supabase base config (including wal-g and data_directory settings) is never loaded. -ENV POSTGRES_INITDB_SQL_FILES=/docker-entrypoint-initdb.d/multigres-init.sql -ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" -ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive +# wal-g is not used in Multigres (pgbackrest handles backups); remove inherited files. +RUN rm -f \ + /etc/postgresql-custom/wal-g.conf \ + /home/postgres/wal_fetch.sh \ + /root/wal_change_ownership.sh -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD pg_isready -U postgres -h localhost || true -STOPSIGNAL SIGINT +# No HEALTHCHECK defined here: inherits the probe from the supabase base image. +# Kubernetes ignores Docker HEALTHCHECK entirely — use readinessProbe in the Pod spec. +# STOPSIGNAL inherited from supabase base image (SIGINT — smart shutdown). USER postgres -EXPOSE 5432 -ENTRYPOINT ["tail"] -CMD ["-f", "/dev/null"] +# pgctld is the cluster lifecycle manager for Multigres: it handles initdb, +# config templating, replication setup, and coordinated restarts. Running it +# as PID 1 ensures it receives stop signals directly and can shut down +# PostgreSQL cleanly before the container exits. +ENTRYPOINT ["/usr/local/bin/pgctld"] diff --git a/Dockerfile-supabase b/Dockerfile-supabase new file mode 100644 index 000000000..5bab2e87c --- /dev/null +++ b/Dockerfile-supabase @@ -0,0 +1,220 @@ +# syntax=docker/dockerfile:1.3 +# 1.3: minimum version for heredoc support in RUN (used for nix config below) +# Supabase PostgreSQL image with Nix extensions — parameterised by major version. +# +# Build (defaults to PostgreSQL 17): +# docker build -f Dockerfile-supabase -t supabase-postgres:17 . +# +# Build for a different PostgreSQL version: +# docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . + +ARG PG_VERSION=17 + +#################### +# Stage 1: Nix builder +#################### +FROM alpine:3.23 AS nix-builder + +ARG PG_VERSION + +# Install dependencies for nix installer (coreutils for GNU cp, sudo for installer) +RUN apk add --no-cache \ + bash \ + coreutils \ + curl \ + shadow \ + sudo \ + xz + +# Create users (Alpine syntax) +RUN addgroup -S postgres && \ + adduser -S -h /var/lib/postgresql -s /bin/bash -G postgres postgres && \ + addgroup -S wal-g && \ + adduser -S -s /bin/bash -G wal-g wal-g + +# Create nix config +RUN cat < /tmp/extra-nix.conf +extra-experimental-features = nix-command flakes +extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com +extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= +EOF +RUN curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --no-channel-add --yes --nix-extra-conf-file /tmp/extra-nix.conf +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" +RUN nix --version + +WORKDIR /nixpg +COPY . . + +# Build PostgreSQL with extensions +RUN nix profile add path:.#psql_${PG_VERSION}_slim/bin + +RUN nix store gc + +# Build groonga and copy plugins +RUN nix profile add path:.#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +RUN nix store gc + +#################### +# Stage 2: Gosu builder +#################### +FROM alpine:3.23 AS gosu-builder + +ARG TARGETARCH +ARG GOSU_VERSION=1.19 +ARG GO_VERSION=1.26.1 + +RUN apk add --no-cache curl git + +# Install Go +RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" | tar -C /usr/local -xz +ENV PATH="/usr/local/go/bin:${PATH}" + +# Build gosu from source +RUN git clone --depth 1 --branch "${GOSU_VERSION}" https://github.com/tianon/gosu.git /gosu && \ + cd /gosu && \ + CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/local/bin/gosu . && \ + chmod +x /usr/local/bin/gosu + +#################### +# Stage 3: Final production image +#################### +FROM alpine:3.23 AS production + +ARG PG_VERSION + +# Install minimal runtime dependencies +RUN apk add --no-cache \ + bash \ + curl \ + shadow \ + su-exec \ + tzdata \ + musl-locales \ + musl-locales-lang \ + && rm -rf /var/cache/apk/* + +# Create postgres user/group +RUN addgroup -S postgres && \ + adduser -S -G postgres -h /var/lib/postgresql -s /bin/bash postgres && \ + addgroup -S wal-g && \ + adduser -S -G wal-g -s /bin/bash wal-g && \ + adduser postgres wal-g + +# Copy Nix store and profiles from builder (profile already created by nix profile install) +COPY --from=nix-builder /nix /nix + +# Copy groonga plugins +COPY --from=nix-builder /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins + +# Copy gosu +COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu + +# Setup PostgreSQL directories +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + /var/lib/postgresql/data \ + /var/run/postgresql \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql \ + && chown -R postgres:postgres /var/run/postgresql + +# Create symbolic links for binaries +RUN for f in /nix/var/nix/profiles/default/bin/*; do \ + ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ + ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ + done + +# Create symbolic links for PostgreSQL shares +RUN ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ + ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true + +# Set permissions +RUN chown -R postgres:postgres /usr/lib/postgresql && \ + chown -R postgres:postgres /usr/share/postgresql + +# Setup configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql/postgresql.conf.d +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +# Configure PostgreSQL settings +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + chown -R postgres:postgres /etc/postgresql-custom && \ + ln -s /etc/postgresql/postgresql.conf.d /etc/postgresql-custom/conf.d + +# pg17+ does not ship timescaledb or plv8; db_user_namespace was removed in pg17. +# Applied conditionally so the same Dockerfile works for pg15 (where these are valid). +RUN if [ "${PG_VERSION}" -ge 17 ]; then \ + sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql/postgresql.conf && \ + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g' /etc/postgresql/postgresql.conf && \ + sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql-custom/supautils.conf; \ + fi + +# Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# Add entrypoint script from the official postgres Docker library (version-matched) +ADD --chmod=0755 \ + https://raw.githubusercontent.com/docker-library/postgres/6edb0a8c4def40c371514b34aef9037ec82d9110/${PG_VERSION}/alpine3.23/docker-entrypoint.sh \ + /usr/local/bin/docker-entrypoint.sh + +# Setup pgsodium key script +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + +# Environment variables +ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" +ENV PGDATA=/var/lib/postgresql/data +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins +# Point to minimal glibc locales included in slim Nix package for initdb locale support +ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive + +# Marks the container unhealthy after 10 failed pg_isready probes, which blocks +# dependent services in Docker Compose (depends_on: condition: service_healthy). +# Kubernetes ignores Docker HEALTHCHECK entirely — use readinessProbe in the Pod spec. +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost + +# SIGINT triggers PostgreSQL smart shutdown: waits for active sessions to finish +# before stopping. This avoids interrupting in-flight transactions but can delay +# pod termination if long-running queries are active. +# Consider SIGTERM (fast shutdown) to disconnect clients immediately, which +# respects Kubernetes terminationGracePeriodSeconds more predictably. +STOPSIGNAL SIGINT +EXPOSE 5432 + +# No USER directive: the entrypoint starts as root to fix volume ownership and +# set up permissions, then drops to the postgres user via gosu before exec'ing +# the postgres process. This follows the standard official PostgreSQL image pattern. +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/README.md b/README.md index b509bdddd..966cb48f9 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Here's a comprehensive overview of the project's directory structure: | docker/nix/ | Nix-based Docker build configurations | | Dockerfile-15 | Docker image definition for PostgreSQL 15 | | Dockerfile-17 | Docker image definition for PostgreSQL 17 | +| Dockerfile-supabase | Parameterised supabase base image (supports any PostgreSQL major version via `--build-arg PG_VERSION=17`) | +| Dockerfile-multigres | Multigres image layered on top of `Dockerfile-supabase` | | **tests/** | Integration and system tests | | testinfra/ | Infrastructure tests using pytest framework | | tests/ | General integration test suites | @@ -161,6 +163,34 @@ docker build -f Dockerfile-15 -t supabase-postgres:15 . docker build -f Dockerfile-17 -t supabase-postgres:17 . ``` +#### Supabase base image (version-parameterised) + +`Dockerfile-supabase` is a single Dockerfile that can target any supported PostgreSQL major version via `--build-arg`: + +```bash +# Build for PostgreSQL 17 (default) +docker build -f Dockerfile-supabase -t supabase-postgres:17 . + +# Build for PostgreSQL 15 +docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . +``` + +#### Multigres image + +`Dockerfile-multigres` layers on top of the supabase image, adding `pgctld` and `pgbackrest`. Build the supabase image first, then point `SUPABASE_IMAGE` at it: + +```bash +# Build supabase base, then multigres on top +docker build -f Dockerfile-supabase -t supabase-postgres:17 . +docker build -f Dockerfile-multigres -t multigres:17 . + +# Target a different PostgreSQL version +docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . +docker build -f Dockerfile-multigres \ + --build-arg SUPABASE_IMAGE=supabase-postgres:15 \ + -t multigres:15 . +``` + ## Next Steps Now that you understand the basics of Supabase Postgres: diff --git a/ansible/vars.yml b/ansible/vars.yml index 6a49cb27a..ffd0283eb 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,12 +8,44 @@ postgres_major: - "17" - orioledb-17 -# Full version strings for each major version +# Full version strings for each major version. +# +# This is the source of truth for Postgres versions used in the Dockerfiles, and +# is used to derive image tags and base images in the release matrix. postgres_release: postgresorioledb-17: "17.6.0.086-orioledb" postgres17: "17.6.1.129" postgres15: "15.14.1.129" +# Docker release matrix — base images built first, layered images built on top. +# tag and base_tag are derived at build time from postgres_release via release_key. +# tag_suffix is appended to the release version to form the final image tag. +release_matrix_base: + - dockerfile: Dockerfile-supabase + target: production + pg_version: "15" + release_key: postgres15 + - dockerfile: Dockerfile-supabase + target: production + pg_version: "17" + release_key: postgres17 + +release_matrix_layered: + - dockerfile: Dockerfile-multigres + target: production + pg_version: "17" + release_key: postgres17 + tag_suffix: "-multigres" + + # OrioleDB uses a patched version of Postgres 17 so it builds from scratch but + # logically belongs as a layered image. The layered build job passes SUPABASE_IMAGE + # as a build arg which Docker silently ignores since it is undeclared in this Dockerfile. + - dockerfile: Dockerfile-orioledb-17 + target: production + pg_version: "17" + release_key: postgresorioledb-17 + tag_suffix: "" + # Non Postgres Extensions pgbouncer_release: 1.25.1 pgbouncer_release_checksum: sha256:6e566ae92fe3ef7f6a1b9e26d6049f7d7ca39c40e29e7b38f6d5500ae15d8465 diff --git a/docker/pgctld/pgctld b/docker/pgctld/pgctld new file mode 100755 index 000000000..f2780714d --- /dev/null +++ b/docker/pgctld/pgctld @@ -0,0 +1,6 @@ +#!/bin/sh +exec /usr/local/bin/pgctld-bin \ + --postgres-config-template /etc/pgctld-custom/postgresql.conf.tmpl \ + --pg-initdb-sql-dirs supabase_admin:/docker-entrypoint-initdb.d/init-scripts \ + --pg-initdb-sql-dirs supabase_admin:/docker-entrypoint-initdb.d/migrations \ + "$@" diff --git a/nix/packages/docker-image-test.nix b/nix/packages/docker-image-test.nix index fe69f0932..5cb0eb52e 100644 --- a/nix/packages/docker-image-test.nix +++ b/nix/packages/docker-image-test.nix @@ -41,6 +41,7 @@ writeShellApplication { HTTP_MOCK_PID="" KEEP_CONTAINER=false TARGET="" + PG_VERSION="" # Colors for output RED='\033[0;31m' @@ -67,14 +68,17 @@ writeShellApplication { --keep Keep the container running after tests (for debugging) --target TARGET Build target (required for Dockerfile-multigres) Values: variant-17, variant-orioledb-17 + --pg-version VER PostgreSQL major version (required for Dockerfile-supabase) + Values: 15, 17 Examples: nix run .#docker-image-test -- Dockerfile-17 nix run .#docker-image-test -- Dockerfile-15 nix run .#docker-image-test -- Dockerfile-orioledb-17 nix run .#docker-image-test -- --no-build Dockerfile-17 + nix run .#docker-image-test -- --pg-version 17 Dockerfile-supabase + nix run .#docker-image-test -- --no-build --pg-version 15 Dockerfile-supabase nix run .#docker-image-test -- --target variant-17 Dockerfile-multigres - nix run .#docker-image-test -- --target variant-orioledb-17 Dockerfile-multigres nix run .#docker-image-test -- --no-build --target variant-17 Dockerfile-multigres EOF } @@ -85,6 +89,17 @@ writeShellApplication { Dockerfile-15) echo "15 5436" ;; Dockerfile-17) echo "17 5435" ;; Dockerfile-orioledb-17) echo "orioledb-17 5437" ;; + Dockerfile-supabase) + if [[ -z "$PG_VERSION" ]]; then + log_error "Dockerfile-supabase requires --pg-version (15 or 17)" + exit 1 + fi + case "$PG_VERSION" in + 15) echo "15 5436" ;; + 17) echo "17 5435" ;; + *) log_error "Unknown --pg-version: $PG_VERSION (expected 15 or 17)"; exit 1 ;; + esac + ;; Dockerfile-multigres) case "''${TARGET}" in variant-17) echo "multigres-17 5438" ;; @@ -97,7 +112,7 @@ writeShellApplication { ;; *) log_error "Unknown Dockerfile: $dockerfile" - log_error "Supported: Dockerfile-15, Dockerfile-17, Dockerfile-orioledb-17, Dockerfile-multigres" + log_error "Supported: Dockerfile-15, Dockerfile-17, Dockerfile-supabase, Dockerfile-orioledb-17, Dockerfile-multigres" exit 1 ;; esac @@ -422,6 +437,7 @@ writeShellApplication { --no-build) skip_build=true; shift ;; --keep) KEEP_CONTAINER=true; shift ;; --target) TARGET="$2"; shift; shift ;; + --pg-version) PG_VERSION="$2"; shift; shift ;; -*) log_error "Unknown option: $1"; print_help; exit 1 ;; *) dockerfile="$1"; shift ;; esac @@ -452,8 +468,12 @@ writeShellApplication { if [[ -n "$TARGET" ]]; then target_arg="--target $TARGET" fi + local pg_version_arg="" + if [[ -n "$PG_VERSION" ]]; then + pg_version_arg="--build-arg PG_VERSION=$PG_VERSION" + fi # shellcheck disable=SC2086 - if ! docker build -f "$REPO_ROOT/$dockerfile" $target_arg -t "$IMAGE_TAG" "$REPO_ROOT"; then + if ! docker build -f "$REPO_ROOT/$dockerfile" $target_arg $pg_version_arg -t "$IMAGE_TAG" "$REPO_ROOT"; then log_error "Failed to build image" exit 1 fi diff --git a/nix/packages/image-size-analyzer.nix b/nix/packages/image-size-analyzer.nix index c12b1e040..3987e9851 100644 --- a/nix/packages/image-size-analyzer.nix +++ b/nix/packages/image-size-analyzer.nix @@ -31,8 +31,9 @@ runCommand "image-size-analyzer" # Default values OUTPUT_JSON=false NO_BUILD=false + PG_VERSION="" declare -a IMAGES=() - ALL_DOCKERFILES=("Dockerfile-15" "Dockerfile-17" "Dockerfile-orioledb-17") + ALL_DOCKERFILES=("Dockerfile-15" "Dockerfile-17" "Dockerfile-supabase" "Dockerfile-orioledb-17") TIMESTAMP=$(date +%s) TEMP_DIR="/tmp/image-size-analyzer-$TIMESTAMP" @@ -45,7 +46,9 @@ runCommand "image-size-analyzer" Options: --json Output results as JSON instead of TUI --image DOCKERFILE Analyze specific Dockerfile (can be used multiple times) - Valid values: Dockerfile-15, Dockerfile-17, Dockerfile-orioledb-17 + Valid values: Dockerfile-15, Dockerfile-17, Dockerfile-supabase, Dockerfile-orioledb-17 + --pg-version VER PostgreSQL major version (required when --image Dockerfile-supabase) + Values: 15, 17 --no-build Skip building images, analyze existing ones --help Show this help message @@ -74,6 +77,14 @@ runCommand "image-size-analyzer" NO_BUILD=true shift ;; + --pg-version) + if [[ -z "$2" ]]; then + echo "Error: --pg-version requires a value" + exit 1 + fi + PG_VERSION="$2" + shift 2 + ;; --image) if [[ -z "$2" ]]; then echo "Error: --image requires a value" @@ -141,8 +152,16 @@ runCommand "image-size-analyzer" # Get tag name from Dockerfile name get_tag() { local dockerfile=$1 - local suffix=''${dockerfile#Dockerfile-} - echo "supabase-postgres:$suffix-analyze" + if [[ "$dockerfile" == "Dockerfile-supabase" ]]; then + if [[ -z "$PG_VERSION" ]]; then + echo "Error: --pg-version required for Dockerfile-supabase" >&2 + exit 1 + fi + echo "supabase-postgres:$PG_VERSION-analyze" + else + local suffix=''${dockerfile#Dockerfile-} + echo "supabase-postgres:$suffix-analyze" + fi } # Build a single image @@ -152,7 +171,12 @@ runCommand "image-size-analyzer" tag=$(get_tag "$dockerfile") echo "Building $dockerfile as $tag..." - if ! docker build -f "$dockerfile" -t "$tag" . ; then + local pg_version_arg="" + if [[ -n "$PG_VERSION" ]]; then + pg_version_arg="--build-arg PG_VERSION=$PG_VERSION" + fi + # shellcheck disable=SC2086 + if ! docker build -f "$dockerfile" $pg_version_arg -t "$tag" . ; then echo "Error: Failed to build $dockerfile" return 1 fi