diff --git a/olm/README.md b/olm/README.md index e77563b..1d9cc68 100644 --- a/olm/README.md +++ b/olm/README.md @@ -25,35 +25,28 @@ which is the source of the certification process. ## Secret and Listener Operators -The manifest generation for these two operators is only partially automated. -You start with the script below and then manually update the cluster service version. -To generate the manifests for the secret operator version 24.11.1, run: +Use the `scripts/generate-olm.py` script in each operator repository (secret and listener) like this: -```bash -./olm/build-manifests.sh -r 24.11.1 \ - -c $HOME/repo/stackable/openshift-certified-operators \ - -o $HOME/repo/stackable/secret-operator +```shell +cd $HOME/repo/openshift/openshift-certified-operators/operators/stackable-secret-operator + +uv run --script scripts/generate-olm.py \ +--output-dir $HOME/repo/openshift/openshift-certified-operators/operators/stackable-secret-operator \ +--version \ +--openshift-versions v4.18-v4.21 ``` Where: -- `-r `: the release number (mandatory). This must be a semver-compatible value to patch-level e.g. 23.1.0. -- `-c `: the output folder for the manifest files -- `-o `: directory of the operator repository - -Similarly for the listener operator run: - -```bash -./olm/build-manifests.sh -r 24.11.1 \ - -c $HOME/repo/stackable/openshift-certified-operators \ - -o $HOME/repo/stackable/listener-operator -``` +- `--version `: the release number (mandatory). Example: `26.3.0`. +- `--output-dir `: location of the certified operators repository. +- `--openshift-versions `: catalogs where this bundle is published. Example: `v4.18-v4.21`. ## All Other Operators ```bash ./olm/build-manifests.py \ - --openshift-versions 'v4.14-v4.16' \ + --openshift-versions 'v4.18-v4.21' \ --release 24.11.1 \ --repo-operator ~/repo/stackable/hbase-operator ``` @@ -71,7 +64,7 @@ To build operator bundles run: ```bash ./olm/build-bundles.sh \ -c $HOME/repo/stackable/openshift-certified-operators \ - -r 24.11.1 \ + -r 26.3.0 \ -o listener \ -d ``` diff --git a/olm/build-manifests.py b/olm/build-manifests.py index 8ac322a..f178ad5 100755 --- a/olm/build-manifests.py +++ b/olm/build-manifests.py @@ -1,5 +1,27 @@ -#!/usr/bin/env python -# vim: filetype=python syntax=python tabstop=4 expandtab +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml", +# ] +# /// +""" +(Re)Generate Stackable operator manifests for the Operator Lifecycle Manager (OLM). + +The script renders the Helm chart, looks up image digests on +quay.io, and writes a complete OLM bundle under deploy/olm//. + +Usage: + + uv run --script olm/generate-olm.py --version 26.3.0 --repo-operator ~/repo/stackable/airflow-operator --output-dir deploy/olm + + # Or directly with python3 (PyYAML must be installed): + python3 olm/generate-olm.py --version 26.3.0 --repo-operator ~/repo/stackable/airflow-operator --openshift-versions v4.18-v4.21 + +Requirements: + - uv (https://docs.astral.sh/uv/) — installs PyYAML automatically + - helm (https://helm.sh) +""" import argparse import json @@ -13,24 +35,10 @@ import urllib.parse import urllib.request -try: - import yaml -except ModuleNotFoundError: - print( - "Module 'pyyaml' not found. Install using: pip install -r olm/requirements.txt" - ) - sys.exit(1) +import yaml __version__ = "0.0.1" -DESCRIPTION = """ -(Re)Generate manifests for the Operator Lifecycle Manager (OLM). - -Example: - ./olm/build-manifests.py --release 24.3.0 --repo-operator ~/repo/stackable/airflow-operator -""" - - class ManifestException(Exception): pass @@ -38,7 +46,9 @@ class ManifestException(Exception): def parse_args(argv: list[str]) -> argparse.Namespace: """Parse command line args.""" parser = argparse.ArgumentParser( - description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter + description="Generate OLM bundle manifests for Stackable operators.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, ) parser.add_argument( "--version", @@ -219,8 +229,6 @@ def generate_csv_related_images( def generate_manifests(args: argparse.Namespace) -> list[dict]: logging.debug("start generate_manifests") - # Parse CRDs as generated by Rust serde. - crds = generate_crds(args.repo_operator) # Parse Helm manifests manifests = generate_helm_templates(args) @@ -254,19 +262,16 @@ def generate_manifests(args: argparse.Namespace) -> list[dict]: except KeyError: pass - owned_crds = to_owned_crds(crds) - # Generate the CSV csv = generate_csv( args, - owned_crds, cluster_permissions, deployments, related_images, ) logging.debug("finish generate_manifests") - return [csv, *crds, *manifests] + return [csv, *manifests] def filter_op_objects(args: argparse.Namespace, manifests) -> tuple[dict, dict, dict]: @@ -304,7 +309,6 @@ def write_manifests(args: argparse.Namespace, manifests: list[dict]) -> None: os.makedirs(manifests_dir) # The following objects are written as separate files: - # - crds # - cluster service version # - cluster roles # - services @@ -349,45 +353,8 @@ def write_manifests(args: argparse.Namespace, manifests: list[dict]) -> None: except FileExistsError: raise ManifestException("Destintation directory already exists") - -def to_owned_crds(crds: list[dict]) -> list[dict]: - logging.debug("start to_owned_crds") - owned_crd_dicts = [] - for c in crds: - for v in c["spec"]["versions"]: - ### Extract CRD description from different properties - description = "No description available" - try: - # we use this field instead of schema.openAPIV3Schema.description - # because that one is not set by the Rust->CRD serialization - description = v["schema"]["openAPIV3Schema"]["properties"]["spec"][ - "description" - ] - except KeyError: - pass - try: - # The OPA CRD has this field set - description = v["schema"]["openAPIV3Schema"]["description"] - except KeyError: - pass - - owned_crd_dicts.append( - { - "name": c["metadata"]["name"], - "displayName": c["metadata"]["name"], - "kind": c["spec"]["names"]["kind"], - "version": v["name"], - "description": description, - } - ) - - logging.debug("finish to_owned_crds") - return owned_crd_dicts - - def generate_csv( args: argparse.Namespace, - owned_crds: list[dict], cluster_permissions: list[tuple[str, dict]], deployments: list[dict], related_images: list[dict[str, str]], @@ -414,12 +381,6 @@ def generate_csv( f"https://github.com/stackabletech/{args.op_name}" ) - # Commented out as it caused problems with the certification pipeline. - # result["metadata"]["annotations"]["olm.skipRange"] = f'< {args.release}' - - ### 1. Add list of owned crds - result["spec"]["customresourcedefinitions"]["owned"] = owned_crds - ### 2. Add list of related images result["spec"]["relatedImages"] = related_images @@ -522,26 +483,6 @@ def generate_helm_templates(args: argparse.Namespace) -> list[dict]: ) -def generate_crds(repo_operator: pathlib.Path) -> list[dict]: - logging.debug(f"start generate_crds for {repo_operator}") - crd_path = ( - repo_operator / "deploy" / "helm" / repo_operator.name / "crds" / "crds.yaml" - ) - - logging.info(f"Reading CRDs from {crd_path}") - crds = list(yaml.load_all(crd_path.read_text(), Loader=yaml.SafeLoader)) - for crd in crds: - if crd["kind"] == "CustomResourceDefinition": - # Remove the helm.sh/resource-policy annotation - del crd["metadata"]["annotations"]["helm.sh/resource-policy"] - else: - raise ManifestException( - f'Expected "CustomResourceDefinition" but found kind "{crd["kind"]}" in CRD file "{crd_path}"' - ) - logging.debug("finish generate_crds") - return crds - - def quay_image(images: list[tuple[str, str]]) -> list[dict[str, str]]: """Get the images for the operator from quay.io. See: https://docs.quay.io/api/swagger""" logging.debug("start op_image") diff --git a/olm/build-manifests.sh b/olm/build-manifests.sh deleted file mode 100755 index 7173a58..0000000 --- a/olm/build-manifests.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env bash -# -# NOTE: This script is intended to be used only with the secret and listener ops. -# For all other operators, use the Python equivalent script: build-manifests.py -# -# Helper script to (re)generate OLM package manifests (skeletons). -# -# Usage: -# -# ./olm/build-manifests.sh -r -c -o -# -# Example: -# -# ./olm/build-manifests.sh -r 24.11.0 -c $HOME/repo/stackable/openshift-certified-operators -o $HOME/repo/stackable/secret-operator -# -# Before running the script: -# * Ensure the certified-operators and operator repos are located on the correct branches (the branches are not supplied as arguments). -# * Update the supported OpenShift version range in the `generate_metadata()` function. -# -# The generated manifests need to be updated manually with the following steps: -# * Copy the cluster service version file from the previous package version. -# * Replace the contents of the deployment, and cluster role with the `template.spec` and `rules` from the newly generated files. -# * Remove the unused generated files : service account, operator cluster role (not the product cluster role), role binding, deployment. -# * Remove all Helm labels in all remaining files (including all labels from the cluster role). -# * Update image tags and hashes - -set -euo pipefail -set -x - -# CLI args -OPENSHIFT_ROOT="" -OP_ROOT="" -RELEASE_VERSION="" - -# derived from CLI args -PRODUCT="" -OPERATOR="" -MANIFESTS_DIR="" -METADATA_DIR="" - -# Location of the stackable-utils repo. -# Needed to build path to custom OLM values.yaml files. -UTILS_REPO_DIR=$(pwd) - -generate_metadata() { - - # generate metadata - rm -r -f "$METADATA_DIR" - mkdir -p "$METADATA_DIR" - - pushd "$METADATA_DIR" - - cat >annotations.yaml <<-ANNOS ---- -annotations: - operators.operatorframework.io.bundle.channel.default.v1: "${RELEASE}" - operators.operatorframework.io.bundle.channels.v1: "stable,${RELEASE}" - operators.operatorframework.io.bundle.manifests.v1: manifests/ - operators.operatorframework.io.bundle.mediatype.v1: registry+v1 - operators.operatorframework.io.bundle.metadata.v1: metadata/ - operators.operatorframework.io.bundle.package.v1: stackable-${OPERATOR} - - com.redhat.openshift.versions: v4.18-v4.20 -ANNOS - - popd -} - -generate_manifests() { - # generate manifests - rm -r -f "$MANIFESTS_DIR" - mkdir -p "$MANIFESTS_DIR" - - pushd "$MANIFESTS_DIR" - - # If available, split the CRD into individual manifests - # The secret op installs the CRDs by its self upon startup if not already there. - if [ -f "$OP_ROOT/deploy/helm/$OPERATOR/crds/crds.yaml" ]; then - cat "$OP_ROOT/deploy/helm/$OPERATOR/crds/crds.yaml" | yq -s '.spec.names.kind' - fi - - # Expand the Helm chart to individual files by applying both the default values.yaml - # and the OLM specific values.yaml. - # Each document is saved in file named "{.kind}-{.metadata.name}.yaml" to reduce the number of - # name collisions when multiple objects share the same name. - helm template "$OPERATOR" \ - --values "$OP_ROOT/deploy/helm/$OPERATOR/values.yaml" \ - --values "$UTILS_REPO_DIR/olm/resources/values/$OPERATOR/values.yaml" \ - "$OP_ROOT/deploy/helm/$OPERATOR" | yq -s '(.kind // "kind-unknown") + "-" + (.metadata.name // "name-unknown")' - popd - } - -parse_inputs() { - while [[ "$#" -gt 0 ]]; do - case $1 in - -r) - RELEASE_VERSION="$2" - RELEASE="$(cut -d'.' -f1,2 <<<"$RELEASE_VERSION")" - shift - ;; - -o) - OP_ROOT="$2" - shift - ;; - -c) - OPENSHIFT_ROOT="$2" - shift - ;; - *) - echo "Unknown parameter passed: $1" - exit 1 - ;; - esac - shift - done - - # e.g. "airflow" instead of "airflow-operator", "spark-k8s" instead of "spark-k8s-operator" - PRODUCT=$(basename "${OP_ROOT}" | rev | cut -d- -f2- | rev) - - OPERATOR="$PRODUCT-operator" - MANIFESTS_DIR="$OPENSHIFT_ROOT/operators/stackable-$OPERATOR/$RELEASE_VERSION/manifests" - METADATA_DIR="$OPENSHIFT_ROOT/operators/stackable-$OPERATOR/$RELEASE_VERSION/metadata" - - if [ "stackable-utils" != $(basename "$UTILS_REPO_DIR") ]; then - echo "ERROR: Looks like you are not calling this script from the base [stackable-utils] repository." - echo "ERROR: This is required to properly reference files under the [resources] folder." - exit 1 - fi -} - -maybe_print_help() { - SCRIPT_NAME=$(basename "$0") - if [ -z "$RELEASE_VERSION" ] || [ -z "$OP_ROOT" ] || [ -z "$OPENSHIFT_ROOT" ]; then - cat <<-HELP - (Re)generate OLM manifest skeletons. - - Usage: - - $SCRIPT_NAME -r -c -o - - Options: - -r : Release version - -c : Path to the RH certified operator repository - -o : Path to the Stackable operator repository - - Example: - - $SCRIPT_NAME -r 23.11.0 -c $HOME/repo/stackable/openshift-certified-operators -o $HOME/repo/stackable/zookeeper-operator -HELP - - exit 1 - fi -} - -patch_cluster_roles() { - pushd "$MANIFESTS_DIR" - - # Add nonroot-v2 SCC to product cluster role - if [ -f "$PRODUCT-clusterrole.yml" ]; then - yq -i '.rules += { "apiGroups": [ "security.openshift.io" ], "resources": [ "securitycontextconstraints" ], "resourceNames": ["nonroot-v2" ], "verbs": ["use"]}' "$PRODUCT-clusterrole.yml" - fi - - # Add nonroot-v2 SCC to operator cluster role - if [ -f "$OPERATOR-clusterrole.yml" ]; then - yq -i '.rules += { "apiGroups": [ "security.openshift.io" ], "resources": [ "securitycontextconstraints" ], "resourceNames": ["nonroot-v2" ], "verbs": ["use"]}' "$OPERATOR-clusterrole.yml" - fi - - popd - -} - -main() { - parse_inputs "$@" - maybe_print_help - generate_metadata - generate_manifests - patch_cluster_roles -} - -main "$@" diff --git a/olm/resources/csv.yaml b/olm/resources/csv.yaml index d948bfe..3300cd2 100644 --- a/olm/resources/csv.yaml +++ b/olm/resources/csv.yaml @@ -56,9 +56,6 @@ spec: - supported: true type: AllNamespaces - customresourcedefinitions: - owned: - - placeholder relatedImages: - name: operator name placeholder image: quay.io image placeholder