From b88c9167ddfb605c30fdf0acbd1bc7dc4a196dea Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Sun, 1 Mar 2026 09:57:29 +0000 Subject: [PATCH 1/6] Add pyproject.toml so the projet can be self-installed --- .gitignore | 23 +++++++++++---- __init__.py | 0 pyproject.toml | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 6 deletions(-) delete mode 100644 __init__.py create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 25fea9d2..f20bb618 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,20 @@ -*.sublime-workspace -*.code-workspace -.vscode __pycache__/ -*.pyc -*~ -*.swp .conda +.DS_Store +.env.* .idea +.vscode +*.bak +*.code-workspace +*.egg +*.egg-info +*.egg-info/ +*.log +*.pyc +*.sublime-workspace +*.swp +*.tmp +*.venv +*~ +uv.lock +venv/ diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f78777bd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[project] +name = "SimSys_Scripts" +version = "0.1.0" +description = "Simulation Systems Scripts" +requires-python = ">=3.10" + +[project.optional-dependencies] +dev = [ + "pytest", + "networkx", + "PyYAML", +] + +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = [ + "fcm_bdiff*", + "kgo_updates*", + "umdp3_fixer*", + "lfric_macros*", + "lfric_styling*", + "github_scripts*", + "nightly_testing*", + "gh_review_project*", + "script_umdp3_checker*", + "script_copyright_checker*", +] + +[tool.setuptools] +script-files = [ + "sbin/gh_add_user", + "sbin/gh_manage_milestones", + "sbin/gh_manage_labels", + "sbin/tagman", +] + +[tool.pytest.ini_options] +testpaths = [ + "gh_review_project", + "github_scripts", + "script_umdp3_checker", + "lfric_macros", + "lfric_styling", + "nightly_testing", + "umdp3_fixer", +] + +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +ignore = [ + "E501", # line too long + "I001", # isort: skip file +] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.ty.environment] +root = [ + ".", + "./fcm_bdiff", + "./gh_review_project", + "./github_scripts", + "./script_umdp3_checker", + "./lfric_macros", + "./lfric_styling", + "./nightly_testing", + "./umdp3_fixer", +] From 2ba52bee21b57e28905c22af8a39ae1050b609af Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Sun, 1 Mar 2026 11:26:35 +0000 Subject: [PATCH 2/6] Add venv install script and rename LICENSE --- LICENSE => LICENCE | 2 +- install | 63 ++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 65 insertions(+), 2 deletions(-) rename LICENSE => LICENCE (98%) create mode 100644 install diff --git a/LICENSE b/LICENCE similarity index 98% rename from LICENSE rename to LICENCE index ad21699e..7ceca5a1 100644 --- a/LICENSE +++ b/LICENCE @@ -1,4 +1,4 @@ -BSD 3-Clause License +BSD 3-Clause Licence Copyright (c) 2023, Met Office All rights reserved. diff --git a/install b/install new file mode 100644 index 00000000..6e3b007b --- /dev/null +++ b/install @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# Installs the package into a virtual environment (.venv) at the repository root. +# +# Usage: +# ./install [--uv] [--python PYTHON] +# +# --uv Use `uv` to create and sync the virtual environment +# (alternative to venv + pip). +# --python PYTHON Path to the Python interpreter to use (default: python3.12). +# +# Requirements: +# - Python 3.12 or higher +# - pip (bundled with Python) +# - uv (optional, only required when --uv flag is passed) +# +# Once installed, activate the environment with: +# source .venv/bin/activate + +set -euo pipefail + +USE_UV=false +PYTHON="python3.12" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --uv) USE_UV=true ;; + --python) + if [[ -z "${2:-}" ]]; then + echo "Error: --python requires an argument." >&2 + exit 1 + fi + PYTHON="$2" + shift + ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac + shift +done + +if $USE_UV; then + # Option 2: uv - alternative for users who prefer uv for environment management. + # Syncs dependencies declared in pyproject.toml / uv.lock. + if ! command -v uv &>/dev/null; then + echo "Error: 'uv' is not installed or not on PATH." >&2 + exit 1 + fi + uv sync ${PYTHON:+--python "$PYTHON"} + echo "Installation complete. Activate the environment with: source .venv/bin/activate" +else + # Option 1: venv + pip - recommended for Met Office users. + # Ensures packages are resolved via Met Office Artifactory rather than PyPI. + if ! command -v "$PYTHON" &>/dev/null; then + echo "Error: '$PYTHON' is not installed or not on PATH." >&2 + exit 1 + fi + "$PYTHON" -m venv .venv + # shellcheck source=/dev/null + source .venv/bin/activate + pip install -e . + echo "Installation complete. Virtual environment is active." +fi diff --git a/pyproject.toml b/pyproject.toml index f78777bd..299e2fd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ target-version = "py310" select = ["E", "F", "W", "I"] ignore = [ "E501", # line too long - "I001", # isort: skip file + "I001", # isort: skip unsorted-imports ] [tool.ruff.format] From 5cc918f10e34ba8bb29151c408c67f7bbfe81513 Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Sun, 1 Mar 2026 22:28:49 +0000 Subject: [PATCH 3/6] Update venv install script and README --- .github/linters/.ruff.toml | 8 ++++++-- README.md | 37 +++++++++++++++++++++++++++++++++++++ install => install-vnev.sh | 36 ++++++++++++++++++++---------------- pyproject.toml | 13 +++++++++++-- 4 files changed, 74 insertions(+), 20 deletions(-) rename install => install-vnev.sh (58%) diff --git a/.github/linters/.ruff.toml b/.github/linters/.ruff.toml index 9cb5f660..70aeef4a 100644 --- a/.github/linters/.ruff.toml +++ b/.github/linters/.ruff.toml @@ -1,12 +1,16 @@ cache-dir = "/tmp/.ruff_cache" line-length = 88 -output-format = "grouped" +# output-format = "grouped" +preview = true [lint] # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" # undefined-local-with-import-star-usage -ignore = ["F405"] +ignore = [ + "F405", # line too long + "I001", # isort: unsorted-imports +] [format] # Like Black, indent with spaces, rather than tabs. diff --git a/README.md b/README.md index ed3cb1fc..209af5f7 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,40 @@ owned and maintained by the Simulation Systems and Deployment (SSD) team. Also contains a copy of `script_updater.sh` which is intended to live in the fcm repositories to pull from this repository. + +## Install venv with dev tools + +- After cloning the repo, run + +```sh +# venv + pip: recommended for Met Office use +python3.12 -m venv .venv +.venv/bin/pip install -e ".[dev]" + +# Or, via uv (if available) +uv sync --extra dev --python 3.12 +``` + +- Activate the virtual environment + +```sh +source .venv/bin/activate +``` + +A helper script is also available to do this + +```sh +# via venv + pip +source ./install-venv.sh +# via uv +source ./install-venv.sh [--uv] [--python PYTHON] +``` + +## Lint and Format Python + +```sh +# Lint +ruff check . --config .github/linters/.ruff.toml +# Check Format +ruff format --check --preview --config .github/linters/.ruff.toml +``` diff --git a/install b/install-vnev.sh similarity index 58% rename from install rename to install-vnev.sh index 6e3b007b..17489fa0 100644 --- a/install +++ b/install-vnev.sh @@ -3,7 +3,7 @@ # Installs the package into a virtual environment (.venv) at the repository root. # # Usage: -# ./install [--uv] [--python PYTHON] +# source ./install-venv.sh [--uv] [--python PYTHON] # # --uv Use `uv` to create and sync the virtual environment # (alternative to venv + pip). @@ -14,13 +14,9 @@ # - pip (bundled with Python) # - uv (optional, only required when --uv flag is passed) # -# Once installed, activate the environment with: -# source .venv/bin/activate - -set -euo pipefail USE_UV=false -PYTHON="python3.12" +PYTHON="" # Empty by default; resolved differently per install path # Parse arguments while [[ $# -gt 0 ]]; do @@ -29,12 +25,12 @@ while [[ $# -gt 0 ]]; do --python) if [[ -z "${2:-}" ]]; then echo "Error: --python requires an argument." >&2 - exit 1 + return 1 fi PYTHON="$2" shift ;; - *) echo "Unknown argument: $1" >&2; exit 1 ;; + *) echo "Unknown argument: $1" >&2; return 1 ;; esac shift done @@ -42,22 +38,30 @@ done if $USE_UV; then # Option 2: uv - alternative for users who prefer uv for environment management. # Syncs dependencies declared in pyproject.toml / uv.lock. + # The Python version is resolved from pyproject.toml unless --python is specified. if ! command -v uv &>/dev/null; then echo "Error: 'uv' is not installed or not on PATH." >&2 - exit 1 + return 1 fi - uv sync ${PYTHON:+--python "$PYTHON"} - echo "Installation complete. Activate the environment with: source .venv/bin/activate" + uv sync --extra dev ${PYTHON:+--python "$PYTHON"} else # Option 1: venv + pip - recommended for Met Office users. # Ensures packages are resolved via Met Office Artifactory rather than PyPI. + # Falls back to python3.12 as the minimum version required by pyproject.toml. + PYTHON="${PYTHON:-python3.12}" if ! command -v "$PYTHON" &>/dev/null; then echo "Error: '$PYTHON' is not installed or not on PATH." >&2 - exit 1 + return 1 fi "$PYTHON" -m venv .venv - # shellcheck source=/dev/null - source .venv/bin/activate - pip install -e . - echo "Installation complete. Virtual environment is active." + .venv/bin/pip install -e ".[dev]" fi +# Activate the environment after installation. This allows users to immediately +# start using the installed packages without needing to run the +# activation command separately. +echo "Installation complete. Activating the environment..." +#shellcheck disable=SC1091 +source .venv/bin/activate +echo "Environment activated. + To deactivate, run: deactivate. + To remove the environment, delete the .venv directory." diff --git a/pyproject.toml b/pyproject.toml index 299e2fd2..ab00a655 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,13 +2,15 @@ name = "SimSys_Scripts" version = "0.1.0" description = "Simulation Systems Scripts" -requires-python = ">=3.10" +requires-python = ">=3.12" [project.optional-dependencies] dev = [ "pytest", "networkx", "PyYAML", + "ruff", + "ty", ] [build-system] @@ -53,7 +55,14 @@ line-length = 88 target-version = "py310" [tool.ruff.lint] -select = ["E", "F", "W", "I"] +select = [ + "E", # pycodestyle errors: Basic PEP 8 violations + "F", # pyflakes errors: Logical errors in code + "W", # pycodestyle warnings: PEP 8 warnings + "I", # isort errors: Import sorting issues + "UP", # pyupgrade: Modernisation rules, enforce f-strings for formatting + "B", # flake8-bugbear: Code quality issues +] ignore = [ "E501", # line too long "I001", # isort: skip unsorted-imports From 87513ad01b3ffbcfc0e3bf5b1b4a33328bb2f05a Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Sun, 1 Mar 2026 23:00:05 +0000 Subject: [PATCH 4/6] ruff format check --- README.md | 2 +- fcm_bdiff/fcm_bdiff.py | 6 +- github_scripts/git_bdiff.py | 9 +- pyproject.toml | 1 + script_umdp3_checker/search_lists.py | 16 +- suite_report.py | 14 +- umdp3_fixer/styling.py | 1372 +++++++++++++------------- umdp3_fixer/whitespace.py | 12 +- 8 files changed, 713 insertions(+), 719 deletions(-) diff --git a/README.md b/README.md index 209af5f7..f7c9bf8e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ fcm repositories to pull from this repository. ## Install venv with dev tools -- After cloning the repo, run +- After cloning the repository, run ```sh # venv + pip: recommended for Met Office use diff --git a/fcm_bdiff/fcm_bdiff.py b/fcm_bdiff/fcm_bdiff.py index 74a84c2a..51073bae 100644 --- a/fcm_bdiff/fcm_bdiff.py +++ b/fcm_bdiff/fcm_bdiff.py @@ -37,10 +37,10 @@ class FCMBase: getting information from an instance of the same class. Note that the version for Git has a small handful of methods, mostly - internal and some propeties. These are kept as close as possible to + internal and some properties. These are kept as close as possible to version in git_bdiff.py. - Attributes used to navigate the horros of FCM and thus used in this + Attributes used to navigate the horrors of FCM and thus used in this package are therefore preceded with an '_' and shouldn't be what is being referred to outwith this class. Nor should the original 'functions'... @@ -85,7 +85,7 @@ def get_branch_name(self): """ Get the branch name from the branch URL. Not sure how useful this will be in FCM world. - For now, define it to be the contants of the URL after .*/main/ has been + For now, define it to be the contents of the URL after .*/main/ has been stripped off. i.e. it will start with trunk/... or branches/... """ diff --git a/github_scripts/git_bdiff.py b/github_scripts/git_bdiff.py index 2db51ddf..4bb82c6e 100644 --- a/github_scripts/git_bdiff.py +++ b/github_scripts/git_bdiff.py @@ -168,9 +168,12 @@ def has_diverged(self): def files(self): """Iterate over files changed on the branch.""" - for line in self.run_git( - ["diff", "--name-only", "--diff-filter=AMX", self.ancestor] - ): + for line in self.run_git([ + "diff", + "--name-only", + "--diff-filter=AMX", + self.ancestor, + ]): if line != "": yield line diff --git a/pyproject.toml b/pyproject.toml index ab00a655..c197ce0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ select = [ ignore = [ "E501", # line too long "I001", # isort: skip unsorted-imports + "UP030", # Allow explicit positional integers in .format() ] [tool.ruff.format] diff --git a/script_umdp3_checker/search_lists.py b/script_umdp3_checker/search_lists.py index 7ee2c74b..b193811a 100644 --- a/script_umdp3_checker/search_lists.py +++ b/script_umdp3_checker/search_lists.py @@ -146,15 +146,13 @@ } # Retired if-defs (placeholder - would be loaded from configuration) -retired_ifdefs = set( - [ - "VATPOLES", - "A12_4A", - "A12_3A", - "UM_JULES", - "A12_2A", - ] -) +retired_ifdefs = set([ + "VATPOLES", + "A12_4A", + "A12_3A", + "UM_JULES", + "A12_2A", +]) # Deprecated C identifiers deprecated_c_identifiers = {"gets", "tmpnam", "tempnam", "mktemp"} diff --git a/suite_report.py b/suite_report.py index 67148cdf..bff139cb 100755 --- a/suite_report.py +++ b/suite_report.py @@ -1992,14 +1992,12 @@ def print_report(self): suite_dir = self.suite_path except Exception: suite_dir = "--cylc_suite_dir--" - trac_log.extend( - [ - "There has been an exception in SuiteReport.print_report()", - "See output for more information", - "rose-stem suite output will be in the files :\n", - f"~/cylc-run/{suite_dir}/log/scheduler/log", - ] - ) + trac_log.extend([ + "There has been an exception in SuiteReport.print_report()", + "See output for more information", + "rose-stem suite output will be in the files :\n", + f"~/cylc-run/{suite_dir}/log/scheduler/log", + ]) finally: # Pick up user specified log path if available, # otherwise default to cyclc suite dir. diff --git a/umdp3_fixer/styling.py b/umdp3_fixer/styling.py index 98e76fdb..a7af2cca 100755 --- a/umdp3_fixer/styling.py +++ b/umdp3_fixer/styling.py @@ -106,687 +106,685 @@ "TYPE", ] -KEYWORDS = set( - [ - "abort", - "abs", - "abstract", - "access", - "achar", - "acos", - "acosd", - "acosh", - "action", - "adjustl", - "adjustr", - "advance", - "aimag", - "aint", - "alarm", - "algama", - "all", - "allocatable", - "allocate", - "allocated", - "alog", - "alog10", - "amax0", - "amax1", - "amin0", - "amin1", - "amod", - "and", - "anint", - "any", - "asin", - "asind", - "asinh", - "assign", - "assignment", - "associate", - "associated", - "asynchronous", - "atan", - "atan2", - "atan2d", - "atand", - "atanh", - "atomic", - "atomic_add", - "atomic_and", - "atomic_cas", - "atomic_define", - "atomic_fetch_add", - "atomic_fetch_and", - "atomic_fetch_or", - "atomic_fetch_xor", - "atomic_int_kind", - "atomic_logical_kind", - "atomic_or", - "atomic_ref", - "atomic_xor", - "backspace", - "backtrace", - "barrier", - "besj0", - "besj1", - "besjn", - "bessel_j0", - "bessel_j1", - "bessel_jn", - "bessel_y0", - "bessel_y1", - "bessel_yn", - "besy0", - "besy1", - "besyn", - "bge", - "bgt", - "bind", - "bit_size", - "blank", - "ble", - "block", - "blt", - "btest", - "c_alert", - "c_associated", - "c_backspace", - "c_bool", - "c_carriage_return", - "c_char", - "c_double", - "c_double_complex", - "c_f_pointer", - "c_f_procpointer", - "c_float", - "c_float128", - "c_float128_complex", - "c_float_complex", - "c_form_feed", - "c_funloc", - "c_funptr", - "c_horizontal_tab", - "c_int", - "c_int128_t", - "c_int16_t", - "c_int32_t", - "c_int64_t", - "c_int8_t", - "c_int_fast128_t", - "c_int_fast16_t", - "c_int_fast32_t", - "c_int_fast64_t", - "c_int_fast8_t", - "c_int_least128_t", - "c_int_least16_t", - "c_int_least32_t", - "c_int_least64_t", - "c_int_least8_t", - "c_intmax_t", - "c_intptr_t", - "c_loc", - "c_long", - "c_long_double", - "c_long_double_complex", - "c_long_long", - "c_new_line", - "c_null_char", - "c_null_funptr", - "c_null_ptr", - "c_ptr", - "c_ptrdiff_t", - "c_short", - "c_signed_char", - "c_size_t", - "c_sizeof", - "c_vertical_tab", - "cabs", - "call", - "case", - "ccos", - "cdabs", - "cdcos", - "cdexp", - "cdlog", - "cdsin", - "cdsqrt", - "ceiling", - "cexp", - "char", - "character", - "character_kinds", - "character_storage_size", - "chdir", - "chmod", - "class", - "clog", - "close", - "cmplx", - "co_broadcast", - "co_max", - "co_min", - "co_reduce", - "co_sum", - "codimension", - "command_argument_count", - "common", - "compiler_options", - "compiler_version", - "complex", - "concurrent", - "conjg", - "contains", - "contiguous", - "continue", - "convert", - "copyin", - "copyprivate", - "cos", - "cosd", - "cosh", - "cotan", - "cotand", - "count", - "cpp", - "cpu_time", - "cqabs", - "cqcos", - "cqexp", - "cqlog", - "cqsin", - "cqsqrt", - "critical", - "cshift", - "csin", - "csqrt", - "ctime", - "cycle", - "dabs", - "dacos", - "dacosh", - "dasin", - "dasinh", - "data", - "datan", - "datan2", - "datanh", - "date_and_time", - "dbesj0", - "dbesj1", - "dbesjn", - "dbesy0", - "dbesy1", - "dbesyn", - "dble", - "dcmplx", - "dconjg", - "dcos", - "dcosh", - "ddim", - "deallocate", - "decode", - "default", - "deferred", - "delim", - "derf", - "derfc", - "dexp", - "dfloat", - "dgamma", - "digits", - "dim", - "dimag", - "dimension", - "dint", - "direct", - "dlgama", - "dlog", - "dlog10", - "dmax1", - "dmin1", - "dmod", - "dnint", - "do", - "dot_product", - "double", - "dprod", - "dreal", - "dshiftl", - "dshiftr", - "dsign", - "dsin", - "dsinh", - "dsqrt", - "dtan", - "dtanh", - "dtime", - "elemental", - "else", - "elsewhere", - "encode", - "end", - "endfile", - "entry", - "enum", - "enumerator", - "eor", - "eoshift", - "epsilon", - "equivalence", - "eqv", - "erf", - "erfc", - "erfc_scaled", - "errmsg", - "error", - "error_unit", - "etime", - "event_query", - "execute_command_line", - "exist", - "exit", - "exp", - "exponent", - "extends", - "extends_type_of", - "external", - "false", - "fdate", - "fget", - "fgetc", - "file", - "file_storage_size", - "final", - "firstprivate", - "float", - "floor", - "flush", - "fmt", - "fnum", - "forall", - "form", - "format", - "formatted", - "fpp", - "fput", - "fputc", - "fraction", - "free", - "fseek", - "fstat", - "ftell", - "function", - "gamma", - "generic", - "gerror", - "get_command", - "get_command_argument", - "get_environment_variable", - "getarg", - "getcwd", - "getenv", - "getgid", - "getlog", - "getpid", - "getuid", - "gmtime", - "go", - "hostnm", - "huge", - "hypot", - "iabs", - "iachar", - "iall", - "iand", - "iany", - "iargc", - "ibclr", - "ibits", - "ibset", - "ichar", - "idate", - "idim", - "idint", - "idnint", - "ieee_class", - "ieee_class_type", - "ieee_copy_sign", - "ieee_is_finite", - "ieee_is_nan", - "ieee_is_negative", - "ieee_is_normal", - "ieee_logb", - "ieee_negative_denormal", - "ieee_negative_inf", - "ieee_negative_normal", - "ieee_negative_zero", - "ieee_next_after", - "ieee_positive_denormal", - "ieee_positive_inf", - "ieee_positive_normal", - "ieee_positive_zero", - "ieee_quiet_nan", - "ieee_rem", - "ieee_rint", - "ieee_scalb", - "ieee_selected_real_kind", - "ieee_signaling_nan", - "ieee_support_datatype", - "ieee_support_denormal", - "ieee_support_divide", - "ieee_support_inf", - "ieee_support_nan", - "ieee_support_sqrt", - "ieee_support_standard", - "ieee_unordered", - "ieee_value", - "ieor", - "ierrno", - "if", - "ifix", - "imag", - "image_index", - "images", - "imagpart", - "implicit", - "import", - "in", - "include", - "index", - "inout", - "input_unit", - "inquire", - "int", - "int16", - "int2", - "int32", - "int64", - "int8", - "integer", - "integer_kinds", - "intent", - "interface", - "intrinsic", - "iomsg", - "ior", - "iostat", - "iostat_end", - "iostat_eor", - "iostat_inquire_internal_unit", - "iparity", - "iqint", - "irand", - "is", - "is_iostat_end", - "is_iostat_eor", - "isatty", - "ishft", - "ishftc", - "isign", - "isnan", - "iso_c_binding", - "iso_fortran_env", - "itime", - "kill", - "kind", - "lastprivate", - "lbound", - "lcobound", - "leadz", - "len", - "len_trim", - "lgamma", - "lge", - "lgt", - "link", - "lle", - "llt", - "lnblnk", - "loc", - "lock", - "lock_type", - "log", - "log10", - "log_gamma", - "logical", - "logical_kinds", - "long", - "lshift", - "lstat", - "ltime", - "malloc", - "maskl", - "maskr", - "master", - "matmul", - "max", - "max0", - "max1", - "maxexponent", - "maxloc", - "maxval", - "mclock", - "mclock8", - "memory", - "merge", - "merge_bits", - "min", - "min0", - "min1", - "minexponent", - "minloc", - "minval", - "mod", - "module", - "modulo", - "move_alloc", - "mvbits", - "name", - "named", - "namelist", - "nearest", - "neqv", - "new_line", - "nextrec", - "nint", - "nml", - "non_intrinsic", - "non_overridable", - "none", - "nopass", - "norm2", - "not", - "null", - "nullify", - "num_images", - "number", - "numeric_storage_size", - "only", - "open", - "opened", - "operator", - "optional", - "or", - "ordered", - "out", - "output_unit", - "pack", - "pad", - "parallel", - "parameter", - "parity", - "pass", - "perror", - "pointer", - "popcnt", - "poppar", - "position", - "precision", - "present", - "print", - "private", - "procedure", - "product", - "program", - "protected", - "public", - "pure", - "qabs", - "qacos", - "qasin", - "qatan", - "qatan2", - "qcmplx", - "qconjg", - "qcos", - "qcosh", - "qdim", - "qerf", - "qerfc", - "qexp", - "qgamma", - "qimag", - "qlgama", - "qlog", - "qlog10", - "qmax1", - "qmin1", - "qmod", - "qnint", - "qsign", - "qsin", - "qsinh", - "qsqrt", - "qtan", - "qtanh", - "radix", - "ran", - "rand", - "random_number", - "random_seed", - "range", - "rank", - "read", - "readwrite", - "real", - "real128", - "real32", - "real64", - "real_kinds", - "realpart", - "rec", - "recl", - "record", - "recursive", - "reduction", - "rename", - "repeat", - "reshape", - "result", - "return", - "rewind", - "rewrite", - "rrspacing", - "rshift", - "same_type_as", - "save", - "scale", - "scan", - "secnds", - "second", - "sections", - "select", - "selected_char_kind", - "selected_int_kind", - "selected_real_kind", - "sequence", - "sequential", - "set_exponent", - "shape", - "shared", - "shifta", - "shiftl", - "shiftr", - "short", - "sign", - "signal", - "sin", - "sind", - "sinh", - "size", - "sizeof", - "sleep", - "sngl", - "source", - "spacing", - "spread", - "sqrt", - "srand", - "stat", - "stat_failed_image", - "stat_locked", - "stat_locked_other_image", - "stat_stopped_image", - "stat_unlocked", - "status", - "stop", - "storage_size", - "structure", - "submodule", - "subroutine", - "sum", - "symlnk", - "sync", - "system", - "system_clock", - "tan", - "tand", - "tanh", - "target", - "task", - "taskwait", - "then", - "this_image", - "threadprivate", - "time", - "time8", - "tiny", - "to", - "trailz", - "transfer", - "transpose", - "trim", - "true", - "ttynam", - "type", - "ubound", - "ucobound", - "umask", - "unformatted", - "unit", - "unlink", - "unlock", - "unpack", - "use", - "value", - "verif", - "verify", - "volatile", - "wait", - "where", - "while", - "workshare", - "write", - "xor", - "zabs", - "zcos", - "zexp", - "zlog", - "zsin", - "zsqrt", - ] -) +KEYWORDS = set([ + "abort", + "abs", + "abstract", + "access", + "achar", + "acos", + "acosd", + "acosh", + "action", + "adjustl", + "adjustr", + "advance", + "aimag", + "aint", + "alarm", + "algama", + "all", + "allocatable", + "allocate", + "allocated", + "alog", + "alog10", + "amax0", + "amax1", + "amin0", + "amin1", + "amod", + "and", + "anint", + "any", + "asin", + "asind", + "asinh", + "assign", + "assignment", + "associate", + "associated", + "asynchronous", + "atan", + "atan2", + "atan2d", + "atand", + "atanh", + "atomic", + "atomic_add", + "atomic_and", + "atomic_cas", + "atomic_define", + "atomic_fetch_add", + "atomic_fetch_and", + "atomic_fetch_or", + "atomic_fetch_xor", + "atomic_int_kind", + "atomic_logical_kind", + "atomic_or", + "atomic_ref", + "atomic_xor", + "backspace", + "backtrace", + "barrier", + "besj0", + "besj1", + "besjn", + "bessel_j0", + "bessel_j1", + "bessel_jn", + "bessel_y0", + "bessel_y1", + "bessel_yn", + "besy0", + "besy1", + "besyn", + "bge", + "bgt", + "bind", + "bit_size", + "blank", + "ble", + "block", + "blt", + "btest", + "c_alert", + "c_associated", + "c_backspace", + "c_bool", + "c_carriage_return", + "c_char", + "c_double", + "c_double_complex", + "c_f_pointer", + "c_f_procpointer", + "c_float", + "c_float128", + "c_float128_complex", + "c_float_complex", + "c_form_feed", + "c_funloc", + "c_funptr", + "c_horizontal_tab", + "c_int", + "c_int128_t", + "c_int16_t", + "c_int32_t", + "c_int64_t", + "c_int8_t", + "c_int_fast128_t", + "c_int_fast16_t", + "c_int_fast32_t", + "c_int_fast64_t", + "c_int_fast8_t", + "c_int_least128_t", + "c_int_least16_t", + "c_int_least32_t", + "c_int_least64_t", + "c_int_least8_t", + "c_intmax_t", + "c_intptr_t", + "c_loc", + "c_long", + "c_long_double", + "c_long_double_complex", + "c_long_long", + "c_new_line", + "c_null_char", + "c_null_funptr", + "c_null_ptr", + "c_ptr", + "c_ptrdiff_t", + "c_short", + "c_signed_char", + "c_size_t", + "c_sizeof", + "c_vertical_tab", + "cabs", + "call", + "case", + "ccos", + "cdabs", + "cdcos", + "cdexp", + "cdlog", + "cdsin", + "cdsqrt", + "ceiling", + "cexp", + "char", + "character", + "character_kinds", + "character_storage_size", + "chdir", + "chmod", + "class", + "clog", + "close", + "cmplx", + "co_broadcast", + "co_max", + "co_min", + "co_reduce", + "co_sum", + "codimension", + "command_argument_count", + "common", + "compiler_options", + "compiler_version", + "complex", + "concurrent", + "conjg", + "contains", + "contiguous", + "continue", + "convert", + "copyin", + "copyprivate", + "cos", + "cosd", + "cosh", + "cotan", + "cotand", + "count", + "cpp", + "cpu_time", + "cqabs", + "cqcos", + "cqexp", + "cqlog", + "cqsin", + "cqsqrt", + "critical", + "cshift", + "csin", + "csqrt", + "ctime", + "cycle", + "dabs", + "dacos", + "dacosh", + "dasin", + "dasinh", + "data", + "datan", + "datan2", + "datanh", + "date_and_time", + "dbesj0", + "dbesj1", + "dbesjn", + "dbesy0", + "dbesy1", + "dbesyn", + "dble", + "dcmplx", + "dconjg", + "dcos", + "dcosh", + "ddim", + "deallocate", + "decode", + "default", + "deferred", + "delim", + "derf", + "derfc", + "dexp", + "dfloat", + "dgamma", + "digits", + "dim", + "dimag", + "dimension", + "dint", + "direct", + "dlgama", + "dlog", + "dlog10", + "dmax1", + "dmin1", + "dmod", + "dnint", + "do", + "dot_product", + "double", + "dprod", + "dreal", + "dshiftl", + "dshiftr", + "dsign", + "dsin", + "dsinh", + "dsqrt", + "dtan", + "dtanh", + "dtime", + "elemental", + "else", + "elsewhere", + "encode", + "end", + "endfile", + "entry", + "enum", + "enumerator", + "eor", + "eoshift", + "epsilon", + "equivalence", + "eqv", + "erf", + "erfc", + "erfc_scaled", + "errmsg", + "error", + "error_unit", + "etime", + "event_query", + "execute_command_line", + "exist", + "exit", + "exp", + "exponent", + "extends", + "extends_type_of", + "external", + "false", + "fdate", + "fget", + "fgetc", + "file", + "file_storage_size", + "final", + "firstprivate", + "float", + "floor", + "flush", + "fmt", + "fnum", + "forall", + "form", + "format", + "formatted", + "fpp", + "fput", + "fputc", + "fraction", + "free", + "fseek", + "fstat", + "ftell", + "function", + "gamma", + "generic", + "gerror", + "get_command", + "get_command_argument", + "get_environment_variable", + "getarg", + "getcwd", + "getenv", + "getgid", + "getlog", + "getpid", + "getuid", + "gmtime", + "go", + "hostnm", + "huge", + "hypot", + "iabs", + "iachar", + "iall", + "iand", + "iany", + "iargc", + "ibclr", + "ibits", + "ibset", + "ichar", + "idate", + "idim", + "idint", + "idnint", + "ieee_class", + "ieee_class_type", + "ieee_copy_sign", + "ieee_is_finite", + "ieee_is_nan", + "ieee_is_negative", + "ieee_is_normal", + "ieee_logb", + "ieee_negative_denormal", + "ieee_negative_inf", + "ieee_negative_normal", + "ieee_negative_zero", + "ieee_next_after", + "ieee_positive_denormal", + "ieee_positive_inf", + "ieee_positive_normal", + "ieee_positive_zero", + "ieee_quiet_nan", + "ieee_rem", + "ieee_rint", + "ieee_scalb", + "ieee_selected_real_kind", + "ieee_signaling_nan", + "ieee_support_datatype", + "ieee_support_denormal", + "ieee_support_divide", + "ieee_support_inf", + "ieee_support_nan", + "ieee_support_sqrt", + "ieee_support_standard", + "ieee_unordered", + "ieee_value", + "ieor", + "ierrno", + "if", + "ifix", + "imag", + "image_index", + "images", + "imagpart", + "implicit", + "import", + "in", + "include", + "index", + "inout", + "input_unit", + "inquire", + "int", + "int16", + "int2", + "int32", + "int64", + "int8", + "integer", + "integer_kinds", + "intent", + "interface", + "intrinsic", + "iomsg", + "ior", + "iostat", + "iostat_end", + "iostat_eor", + "iostat_inquire_internal_unit", + "iparity", + "iqint", + "irand", + "is", + "is_iostat_end", + "is_iostat_eor", + "isatty", + "ishft", + "ishftc", + "isign", + "isnan", + "iso_c_binding", + "iso_fortran_env", + "itime", + "kill", + "kind", + "lastprivate", + "lbound", + "lcobound", + "leadz", + "len", + "len_trim", + "lgamma", + "lge", + "lgt", + "link", + "lle", + "llt", + "lnblnk", + "loc", + "lock", + "lock_type", + "log", + "log10", + "log_gamma", + "logical", + "logical_kinds", + "long", + "lshift", + "lstat", + "ltime", + "malloc", + "maskl", + "maskr", + "master", + "matmul", + "max", + "max0", + "max1", + "maxexponent", + "maxloc", + "maxval", + "mclock", + "mclock8", + "memory", + "merge", + "merge_bits", + "min", + "min0", + "min1", + "minexponent", + "minloc", + "minval", + "mod", + "module", + "modulo", + "move_alloc", + "mvbits", + "name", + "named", + "namelist", + "nearest", + "neqv", + "new_line", + "nextrec", + "nint", + "nml", + "non_intrinsic", + "non_overridable", + "none", + "nopass", + "norm2", + "not", + "null", + "nullify", + "num_images", + "number", + "numeric_storage_size", + "only", + "open", + "opened", + "operator", + "optional", + "or", + "ordered", + "out", + "output_unit", + "pack", + "pad", + "parallel", + "parameter", + "parity", + "pass", + "perror", + "pointer", + "popcnt", + "poppar", + "position", + "precision", + "present", + "print", + "private", + "procedure", + "product", + "program", + "protected", + "public", + "pure", + "qabs", + "qacos", + "qasin", + "qatan", + "qatan2", + "qcmplx", + "qconjg", + "qcos", + "qcosh", + "qdim", + "qerf", + "qerfc", + "qexp", + "qgamma", + "qimag", + "qlgama", + "qlog", + "qlog10", + "qmax1", + "qmin1", + "qmod", + "qnint", + "qsign", + "qsin", + "qsinh", + "qsqrt", + "qtan", + "qtanh", + "radix", + "ran", + "rand", + "random_number", + "random_seed", + "range", + "rank", + "read", + "readwrite", + "real", + "real128", + "real32", + "real64", + "real_kinds", + "realpart", + "rec", + "recl", + "record", + "recursive", + "reduction", + "rename", + "repeat", + "reshape", + "result", + "return", + "rewind", + "rewrite", + "rrspacing", + "rshift", + "same_type_as", + "save", + "scale", + "scan", + "secnds", + "second", + "sections", + "select", + "selected_char_kind", + "selected_int_kind", + "selected_real_kind", + "sequence", + "sequential", + "set_exponent", + "shape", + "shared", + "shifta", + "shiftl", + "shiftr", + "short", + "sign", + "signal", + "sin", + "sind", + "sinh", + "size", + "sizeof", + "sleep", + "sngl", + "source", + "spacing", + "spread", + "sqrt", + "srand", + "stat", + "stat_failed_image", + "stat_locked", + "stat_locked_other_image", + "stat_stopped_image", + "stat_unlocked", + "status", + "stop", + "storage_size", + "structure", + "submodule", + "subroutine", + "sum", + "symlnk", + "sync", + "system", + "system_clock", + "tan", + "tand", + "tanh", + "target", + "task", + "taskwait", + "then", + "this_image", + "threadprivate", + "time", + "time8", + "tiny", + "to", + "trailz", + "transfer", + "transpose", + "trim", + "true", + "ttynam", + "type", + "ubound", + "ucobound", + "umask", + "unformatted", + "unit", + "unlink", + "unlock", + "unpack", + "use", + "value", + "verif", + "verify", + "volatile", + "wait", + "where", + "while", + "workshare", + "write", + "xor", + "zabs", + "zcos", + "zexp", + "zlog", + "zsin", + "zsqrt", +]) def replace_patterns(line, str_continuation): @@ -1117,13 +1115,11 @@ def apply_styling(lines): # if we are a (pp) continuation, save the partial line if pp_continuation: - pp_line_previous = "".join( - [ - re.sub(r"\\\s*$", "", pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line, - ] - ) + pp_line_previous = "".join([ + re.sub(r"\\\s*$", "", pp_line_previous), + re.sub(r"&\s*$", "", line_previous), + line, + ]) line_previous = "" pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) pseudo_str_continuation = is_str_continuation(pseudo_line, str_continuation) diff --git a/umdp3_fixer/whitespace.py b/umdp3_fixer/whitespace.py index 1b3cd6ab..f0f9155b 100755 --- a/umdp3_fixer/whitespace.py +++ b/umdp3_fixer/whitespace.py @@ -190,13 +190,11 @@ def apply_whitespace_fixes(lines, striptrailingspace=True, keywordsplit=True): # if we are a (pp) continuation, save the partial line if pp_continuation: - pp_line_previous = "".join( - [ - re.sub(r"\\\s*$", "", pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line, - ] - ) + pp_line_previous = "".join([ + re.sub(r"\\\s*$", "", pp_line_previous), + re.sub(r"&\s*$", "", line_previous), + line, + ]) line_previous = "" pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) pseudo_str_continuation = is_str_continuation(pseudo_line, str_continuation) From eac6492e95c8512a2b91378adceef79f334c6a5a Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:30:04 +0000 Subject: [PATCH 5/6] Apply ruff check/format --- gh_review_project/workload.py | 4 +- github_scripts/get_git_sources.py | 9 +- github_scripts/suite_data.py | 31 +++---- github_scripts/suite_report_git.py | 23 +++-- github_scripts/tests/test_get_git_sources.py | 4 +- kgo_updates/kgo_update/kgo_update.py | 65 ++++++------- lfric_macros/apply_macros.py | 4 +- lfric_macros/release_lfric.py | 6 +- lfric_macros/validate_rose_meta.py | 2 +- lfric_styling/lfric_styling.py | 4 +- pyproject.toml | 1 + .../checker_dispatch_tables.py | 12 +-- script_umdp3_checker/umdp3_checker_rules.py | 91 +++++++++---------- script_umdp3_checker/umdp3_conformance.py | 62 ++++++------- suite_report.py | 22 ++--- umdp3_fixer/ampersands.py | 32 +++---- umdp3_fixer/fstring_parse.py | 12 +-- umdp3_fixer/indentation.py | 20 ++-- umdp3_fixer/rosestem_branch_checker.py | 15 +-- umdp3_fixer/styling.py | 38 ++++---- umdp3_fixer/umdp3_fixer.py | 28 ++---- umdp3_fixer/whitespace.py | 12 +-- 22 files changed, 229 insertions(+), 268 deletions(-) diff --git a/gh_review_project/workload.py b/gh_review_project/workload.py index 63748adf..965e530e 100644 --- a/gh_review_project/workload.py +++ b/gh_review_project/workload.py @@ -218,12 +218,12 @@ def main(total: bool, test: bool, capture_project: bool, file: Path): # Create tables for each combination of reviewers and reposotories tables = {} - ## Table for non-LFRic repositories + # Table for non-LFRic repositories repo_list = other_repo_list(data, lfric_repositories) reviewers = teams["SSD"].get_team_members() tables["SSD"] = build_table(data, reviewers, repo_list) - ## Table for LFRic repositories + # Table for LFRic repositories repo_list = lfric_repositories reviewers = [] for team in teams.values(): diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 80374c13..8bc5cb89 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -10,7 +10,6 @@ import re import subprocess from datetime import datetime -from typing import Optional, Union from pathlib import Path from shutil import rmtree import shlex @@ -21,7 +20,7 @@ def run_command( command: str, check: bool = True, capture: bool = True, timeout: int = 600 -) -> Optional[subprocess.CompletedProcess]: +) -> subprocess.CompletedProcess | None: """ Run a subprocess command and return the result object Inputs: @@ -92,7 +91,7 @@ def datetime_str() -> str: def clone_and_merge( dependency: str, - opts: Union[list, dict], + opts: list | dict, loc: Path, use_mirrors: bool, mirror_loc: Path, @@ -167,7 +166,7 @@ def get_source( def merge_source( - source: Union[Path, str], + source: Path | str, ref: str, dest: Path, repo: str, @@ -364,7 +363,7 @@ def clone_repo(repo_source: str, repo_ref: str, loc: Path) -> None: run_command(command) -def sync_repo(repo_source: Union[str, Path], repo_ref: str, loc: Path) -> None: +def sync_repo(repo_source: str | Path, repo_ref: str, loc: Path) -> None: """ Rsync a local git clone and checkout the provided ref """ diff --git a/github_scripts/suite_data.py b/github_scripts/suite_data.py index e06b702d..1bbdbf69 100644 --- a/github_scripts/suite_data.py +++ b/github_scripts/suite_data.py @@ -16,7 +16,6 @@ import yaml from collections import defaultdict from pathlib import Path -from typing import Dict, List, Optional, Set, Union from git_bdiff import GitBDiff, GitInfo from get_git_sources import clone_repo, sync_repo @@ -43,7 +42,7 @@ def __init__(self) -> None: self.task_states = {} self.temp_directory = None - def get_um_failed_configs(self) -> Set[str]: + def get_um_failed_configs(self) -> set[str]: """ Read through failed UM rose_ana tasks """ @@ -76,7 +75,7 @@ def read_um_section(self, change: str) -> str: section = "Unknown" return section - def get_changed_um_section(self) -> Set[str]: + def get_changed_um_section(self) -> set[str]: """ Read through bdiff of UM source and find code owner section for each changed file @@ -113,7 +112,7 @@ def get_changed_um_section(self) -> Set[str]: return changed_sections - def get_um_owners(self, filename: str) -> Dict: + def get_um_owners(self, filename: str) -> dict: """ Read UM Code Owners file and write to a dictionary """ @@ -151,7 +150,7 @@ def get_um_owners(self, filename: str) -> Dict: return owners - def parse_tasks(self) -> Dict[str, List[str]]: + def parse_tasks(self) -> dict[str, list[str]]: """ Read through the tasks run, sorting by state """ @@ -233,7 +232,7 @@ def determine_primary_source(self) -> str: return "unknown" - def read_rose_conf(self) -> Dict[str, str]: + def read_rose_conf(self) -> dict[str, str]: """ Read the suite rose-suite.conf file into a dictionary """ @@ -281,19 +280,19 @@ def find_unknown_dependency(self, dependency: str) -> str: pattern = re.compile(rf"{dependency.upper()} SOURCE CLONE=(\S+)") log_file = self.suite_path / "log" / "scheduler" / "log" - with open(log_file, "r") as f: + with open(log_file) as f: for line in f: if match := pattern.search(line): return match.group(1).rstrip("/") raise RuntimeError(f"Unable to find source for dependency {dependency}") - def read_dependencies(self) -> Dict[str, Dict]: + def read_dependencies(self) -> dict[str, dict]: """ Read the suite dependencies from the dependencies.yaml file - this is assumed to have been copied to the suite_path directory """ - with open(self.suite_path / "dependencies.yaml", "r") as stream: + with open(self.suite_path / "dependencies.yaml") as stream: dependencies = yaml.safe_load(stream) for dependency, data in dependencies.items(): if data["source"] is None: @@ -307,7 +306,7 @@ def get_workflow_id(self) -> str: Read cylc install log for workflow id """ - with open(self.suite_path / "log" / "scheduler" / "log", "r") as f: + with open(self.suite_path / "log" / "scheduler" / "log") as f: for line in f: match = re.search(r"INFO - Workflow: (\S+\/\w+)", line) try: @@ -344,7 +343,7 @@ def get_suite_starttime(self) -> str: starttime = row[0] return starttime.split("+")[0] - def read_groups_run(self) -> List[str]: + def read_groups_run(self) -> list[str]: """ Read in groups run as part of suite from the cylc database file """ @@ -360,7 +359,7 @@ def read_groups_run(self) -> List[str]: groups = ["suite_default"] return groups - def get_task_states(self) -> Dict[str, str]: + def get_task_states(self) -> dict[str, str]: """ Query the database and return a dictionary of states. This is assumed to be in suite_path/log/db @@ -374,8 +373,8 @@ def get_task_states(self) -> Dict[str, str]: return data def query_suite_database( - self, database: Path, selections: List[str], source: str - ) -> List[tuple]: + self, database: Path, selections: list[str], source: str + ) -> list[tuple]: """ Create an sql statement and query provided database. Return the result """ @@ -392,8 +391,8 @@ def query_suite_database( return data def run_command( - self, command: Union[str, List[str]], shell: bool = False, rval: bool = False - ) -> Optional[subprocess.CompletedProcess]: + self, command: str | list[str], shell: bool = False, rval: bool = False + ) -> subprocess.CompletedProcess | None: """ Run a subprocess command and return the result object Inputs: diff --git a/github_scripts/suite_report_git.py b/github_scripts/suite_report_git.py index 0f472f69..fbed4d48 100755 --- a/github_scripts/suite_report_git.py +++ b/github_scripts/suite_report_git.py @@ -17,12 +17,11 @@ from contextlib import contextmanager from pathlib import Path from tempfile import mkdtemp -from typing import Dict, List, Set, Tuple from suite_data import SuiteData -def create_markdown_row(*columns: str, header=False) -> List[str]: +def create_markdown_row(*columns: str, header=False) -> list[str]: """ Join any number of columns into a markdown formatted table row Will attempt to format column entries as str @@ -74,10 +73,10 @@ def __init__(self, suite_path: Path) -> None: self.suite_starttime: str = self.get_suite_starttime() self.workflow_id: str = self.get_workflow_id() self.cylc_url: str = self.generate_cylc_url() - self.task_states: Dict[str, str] = self.get_task_states() - self.groups: List[str] = self.read_groups_run() - self.rose_data: Dict[str, str] = self.read_rose_conf() - self.dependencies: Dict[str, Dict] = self.read_dependencies() + self.task_states: dict[str, str] = self.get_task_states() + self.groups: list[str] = self.read_groups_run() + self.rose_data: dict[str, str] = self.read_rose_conf() + self.dependencies: dict[str, dict] = self.read_dependencies() self.primary_source: str = self.determine_primary_source() self.temp_directory = Path(mkdtemp()) self.clone_sources() @@ -85,7 +84,7 @@ def __init__(self, suite_path: Path) -> None: self.populate_gitbdiff() self.trac_log = [] - def parse_local_source(self, source: str) -> Tuple[str, str]: + def parse_local_source(self, source: str) -> tuple[str, str]: """ Find the branch name or hash and remote reference for a given source """ @@ -165,7 +164,7 @@ def create_dependency_table(self) -> None: self.trac_log.append("") - def create_task_tables(self, parsed_tasks: Dict[str, List[str]]) -> None: + def create_task_tables(self, parsed_tasks: dict[str, list[str]]) -> None: """ Create tables containing summary of task states and number of tasks won """ @@ -203,12 +202,12 @@ def create_task_tables(self, parsed_tasks: Dict[str, List[str]]) -> None: self.trac_log.extend(create_markdown_row(task, state)) self.trac_log.append(self.close_collapsed) - def create_um_code_owner_table(self, owners: Dict) -> None: + def create_um_code_owner_table(self, owners: dict) -> None: """ Create a table of required UM code owner approvals """ - changed_sections: Set[str] = self.get_changed_um_section() + changed_sections: set[str] = self.get_changed_um_section() if changed_sections: self.trac_log.extend( create_markdown_row("Section", "Owner", "Deputy", "State", header=True) @@ -221,12 +220,12 @@ def create_um_code_owner_table(self, owners: Dict) -> None: else: self.trac_log.append("* No UM Code Owners Required") - def create_um_config_owner_table(self, owners: Dict) -> None: + def create_um_config_owner_table(self, owners: dict) -> None: """ Create a table of required UM config owner approvals """ - failed_configs: Set[str] = self.get_um_failed_configs() + failed_configs: set[str] = self.get_um_failed_configs() if not failed_configs: self.trac_log.append("No UM Config Owners Required") return diff --git a/github_scripts/tests/test_get_git_sources.py b/github_scripts/tests/test_get_git_sources.py index ea6685b1..72e62220 100644 --- a/github_scripts/tests/test_get_git_sources.py +++ b/github_scripts/tests/test_get_git_sources.py @@ -136,10 +136,10 @@ def test_merge_sources(setup_sources): is None ) # Test Local Source Doesn't Merge - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match="Local source cannot be merged"): merge_source(setup_sources / "merge1", "merge1", target_clone, "SimSys_Scripts") # Test Local Source without ref raises error - with pytest.raises(Exception): + with pytest.raises(ValueError, match="Local source must have a ref"): merge_source(setup_sources / "merge0", "", target_clone, "SimSys_Scripts") diff --git a/kgo_updates/kgo_update/kgo_update.py b/kgo_updates/kgo_update/kgo_update.py index bc6ce86e..4dd20ae9 100755 --- a/kgo_updates/kgo_update/kgo_update.py +++ b/kgo_updates/kgo_update/kgo_update.py @@ -92,7 +92,7 @@ def write_update_script(kgo_dirs, new_dirname, script): # Write a sub-header for this KGO directory to help visually split # up the file (for when the user checks it by eye) script.write("#" * (len(new_kgo_dir) + 4) + "\n") - script.write("# {0} #\n".format(new_kgo_dir)) + script.write(f"# {new_kgo_dir} #\n") script.write("#" * (len(new_kgo_dir) + 4) + "\n") script.write(f"echo 'Installing {new_kgo_dir}'\n\n") @@ -103,8 +103,8 @@ def write_update_script(kgo_dirs, new_dirname, script): copy_description = ["# Files to be copied from suite: "] copy_commands = [] mkdir_commands = [] - mkdir_commands.append("echo 'mkdir -p {0}'".format(new_kgo_dir)) - mkdir_commands.append("mkdir -p {0}".format(new_kgo_dir)) + mkdir_commands.append(f"echo 'mkdir -p {new_kgo_dir}'") + mkdir_commands.append(f"mkdir -p {new_kgo_dir}") kgo_files = sorted(update_dict.keys()) for kgo_file in kgo_files: @@ -114,8 +114,8 @@ def write_update_script(kgo_dirs, new_dirname, script): subdir = os.path.dirname(kgo_file) if subdir != "": full_subdir = os.path.join(new_kgo_dir, subdir) - mkdir_commands.append("echo 'mkdir -p {0}'".format(full_subdir)) - mkdir_commands.append("mkdir -p {0}".format(full_subdir)) + mkdir_commands.append(f"echo 'mkdir -p {full_subdir}'") + mkdir_commands.append(f"mkdir -p {full_subdir}") if source is None: # Files being kept should be sym-linked back to the previous @@ -125,30 +125,24 @@ def write_update_script(kgo_dirs, new_dirname, script): # without a source here must have been retired from # the tests continue - keep_description.append("# * {0}".format(kgo_file)) + keep_description.append(f"# * {kgo_file}") # Construct the paths old_file = os.path.join(kgo_dir, kgo_file) new_file = os.path.join(new_kgo_dir, kgo_file) keep_commands.append( - "echo 'ln -s {0} {1}'".format( - os.path.relpath(old_file, os.path.dirname(new_file)), - new_file, - ) + f"echo 'ln -s {os.path.relpath(old_file, os.path.dirname(new_file))} {new_file}'" ) keep_commands.append( - "ln -s {0} {1}".format( - os.path.relpath(old_file, os.path.dirname(new_file)), - new_file, - ) + f"ln -s {os.path.relpath(old_file, os.path.dirname(new_file))} {new_file}" ) else: # Files from the suite should be copied from the suite - copy_description.append("# * {0}".format(kgo_file)) + copy_description.append(f"# * {kgo_file}") new_file = os.path.join(new_kgo_dir, kgo_file) - copy_commands.append("echo 'cp {0} {1}'".format(source, new_file)) - copy_commands.append("cp {0} {1}".format(source, new_file)) + copy_commands.append(f"echo 'cp {source} {new_file}'") + copy_commands.append(f"cp {source} {new_file}") # In this case more disk-space is needed, so update # the running total for reporting later total_filesize += os.path.getsize(source) @@ -179,13 +173,8 @@ def report_space_required(total_filesize, skip=False): units.pop() unit = units[-1] - print( - banner( - "Update will require: {0:.2f} {1} of disk space".format( - total_filesize, unit - ) - ) - ) + print(banner(f"Update will require: {total_filesize:.2f} {unit} of disk space")) + if not confirm("Please confirm this much space is available (y/n)? ", skip=skip): sys.exit("Aborting...") @@ -218,8 +207,8 @@ def group_comparisons_by_dir(comparisons, skip=False): # decide what they want to happen if not os.path.exists(kgo_file) and status.strip() in UPDATE_CRITERIA: if not confirm( - "KGO file {0} doesn't appear to exist, should we " - "install it from the suite (y/n)? ".format(kgo_file), + f"KGO file {kgo_file} doesn't appear to exist, should we " + "install it from the suite (y/n)? ", skip=skip, ): # Note this is negated - i.e. if the user doesn't want to @@ -298,7 +287,7 @@ def connect_to_kgo_database(suite_dir): def get_suite_dir(user, suite_name): "Returns the path to the given suite directory of a given user" - suite_dir = "~{0}/cylc-run/{1}".format(user, suite_name) + suite_dir = f"~{user}/cylc-run/{suite_name}" expansion = os.path.expanduser(suite_dir) if expansion == suite_dir or not os.path.exists(expansion): msg = "ERROR: Unable to find suite ({0})" @@ -317,7 +306,7 @@ def get_site(): retcode = cmd.returncode if retcode == 0: site = stdout.decode("utf-8").split("=")[1].strip() - print("Found site via rose-config: {0}\n".format(site)) + print(f"Found site via rose-config: {site}\n") else: sys.exit("No site was detected via rose config - this setting is required") return site @@ -377,7 +366,7 @@ def update_variables_rc( if os.path.exists(variables_rc_new): print( "WARNING: New variables.rc file for this update already exists " - "at {0} and will be overwritten".format(variables_rc_new) + f"at {variables_rc_new} and will be overwritten" ) if not confirm("Okay to overwrite variables.rc file (y/n)? ", skip=skip): sys.exit("Aborting...") @@ -393,7 +382,7 @@ def update_variables_rc( ticket_suffix = new_kgo_dir.split("_")[1] # Rewrite the variables.rc - with open(variables_rc, "r") as vrc_old: + with open(variables_rc) as vrc_old: with open(variables_rc_new, "w") as vrc_new: for line in vrc_old.readlines(): # Find lines with KGO variables and update them if they are @@ -403,7 +392,7 @@ def update_variables_rc( vrc_new.write( re.sub( r":\s*BASE.*", - r': BASE~"_{}",'.format(ticket_suffix), + rf': BASE~"_{ticket_suffix}",', line, ) ) @@ -423,7 +412,7 @@ class BlankLinesHelpFormatter(argparse.HelpFormatter): "Formatter which adds blank lines between options" def _split_lines(self, text, width): - return super(BlankLinesHelpFormatter, self)._split_lines(text, width) + [""] + return super()._split_lines(text, width) + [""] parser = argparse.ArgumentParser( usage="%(prog)s [--new-release]", @@ -569,11 +558,11 @@ def _split_lines(self, text, width): kgo_dirs = add_untested_kgo_files(kgo_dirs) # Create a file to hold the update script - script_path = os.path.expanduser("~/kgo_update_{0}.sh".format(new_kgo_dir)) + script_path = os.path.expanduser(f"~/kgo_update_{new_kgo_dir}.sh") if os.path.exists(script_path): print( - "WARNING: Script file for this update already exists at {0} " - "and will be overwritten".format(script_path) + f"WARNING: Script file for this update already exists at {script_path} " + "and will be overwritten" ) if not confirm("Okay to overwrite script file (y/n)? ", skip=confirm_skip): sys.exit("Aborting...") @@ -582,7 +571,7 @@ def _split_lines(self, text, width): with open(script_path, "w") as script: total_filesize = write_update_script(kgo_dirs, new_kgo_dir, script) - print("Script file written to: {0}\n".format(script_path)) + print(f"Script file written to: {script_path}\n") # Report on the required space report_space_required(total_filesize, skip=confirm_skip) @@ -605,7 +594,7 @@ def _split_lines(self, text, width): print( f"\n\nOpening {script_path}\nHit Return to Step through, q to print all\n\n" ) - with open(script_path, "r") as f: + with open(script_path) as f: line_count = 0 print_all = False for line in f: @@ -629,7 +618,7 @@ def _split_lines(self, text, width): sys.exit("Aborting...") print( - banner("Running KGO Update commands from {0}".format(script_path)), + banner(f"Running KGO Update commands from {script_path}"), flush=True, ) diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 4047ce5f..df1c1e8b 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -370,7 +370,7 @@ def read_dependencies(self, repo): - str, The source as defined by the dependencies.yaml file """ dependencies_path = os.path.join(self.root_path, "dependencies.yaml") - with open(dependencies_path, "r") as f: + with open(dependencies_path) as f: dependencies = yaml.safe_load(f) return dependencies[repo]["source"], dependencies[repo]["ref"] @@ -626,7 +626,7 @@ def read_meta_imports(self, meta_dir, flag="import"): meta_file = meta_dir imports = [] - with open(meta_file, "r") as f: + with open(meta_file) as f: in_import_list = False for line in f: line = line.strip() diff --git a/lfric_macros/release_lfric.py b/lfric_macros/release_lfric.py index a286fc4f..1a60a342 100755 --- a/lfric_macros/release_lfric.py +++ b/lfric_macros/release_lfric.py @@ -133,7 +133,7 @@ def update_version_number(args): print("[INFO] Updating rose-suite.conf version number") fpath = os.path.join(args.apps, "rose-stem", "rose-suite.conf") - with open(fpath, "r") as f: + with open(fpath) as f: lines = f.readlines() for i, line in enumerate(lines): @@ -160,7 +160,7 @@ def update_variables_files(apps): variables_files.add(os.path.join(meto_path, filename)) for fpath in variables_files: - with open(fpath, "r") as f: + with open(fpath) as f: lines = f.readlines() for i, line in enumerate(lines): @@ -305,7 +305,7 @@ def add_new_import(versions_file, upgrade_name): upgrade_import = upgrade_name.removesuffix(".py") - with open(versions_file, "r") as f: + with open(versions_file) as f: lines = f.readlines() for i, line in enumerate(lines): diff --git a/lfric_macros/validate_rose_meta.py b/lfric_macros/validate_rose_meta.py index 7ea21f70..6b3b197c 100755 --- a/lfric_macros/validate_rose_meta.py +++ b/lfric_macros/validate_rose_meta.py @@ -140,7 +140,7 @@ def check_rose_stem_apps(meta_paths, source_path): continue app_dir = os.path.join(start_dir, app) conf_file = os.path.join(app_dir, "rose-app.conf") - with open(conf_file, "r") as f: + with open(conf_file) as f: for line in f: if line.startswith("meta="): break diff --git a/lfric_styling/lfric_styling.py b/lfric_styling/lfric_styling.py index 8573dc76..82bfdea4 100755 --- a/lfric_styling/lfric_styling.py +++ b/lfric_styling/lfric_styling.py @@ -27,7 +27,7 @@ def lowercase_keywords(file): Lowercase words in a file when they match a word in the keywords set. """ print("Lowercasing keywords in", file) - with open(file, "r") as fp: + with open(file) as fp: lines = fp.read() for keyword in NEW_KEYWORDS: # regex to check if a keyword is preceded with a '!' symbol @@ -55,7 +55,7 @@ def apply_styling(path_to_dir): Identifying fortran files by extension. """ if os.path.exists(path_to_dir): - for root, dirs, files in os.walk(path_to_dir): + for root, _dirs, files in os.walk(path_to_dir): for file in files: if file.endswith((".f90", ".F90")): cur_path = os.path.join(root, file) diff --git a/pyproject.toml b/pyproject.toml index c197ce0e..7818e603 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ select = [ "I", # isort errors: Import sorting issues "UP", # pyupgrade: Modernisation rules, enforce f-strings for formatting "B", # flake8-bugbear: Code quality issues + # "PL", # pylint rules: Various code quality and style issues ] ignore = [ "E501", # line too long diff --git a/script_umdp3_checker/checker_dispatch_tables.py b/script_umdp3_checker/checker_dispatch_tables.py index 51aac77f..12817a1b 100644 --- a/script_umdp3_checker/checker_dispatch_tables.py +++ b/script_umdp3_checker/checker_dispatch_tables.py @@ -9,7 +9,7 @@ Python translation of the original Perl module """ -from typing import Dict, Callable +from collections.abc import Callable from umdp3_checker_rules import UMDP3Checker """ @@ -30,7 +30,7 @@ class CheckerDispatchTables: def __init__(self): self.umdp3_checker = UMDP3Checker() - def get_diff_dispatch_table_fortran(self) -> Dict[str, Callable]: + def get_diff_dispatch_table_fortran(self) -> dict[str, Callable]: """Get dispatch table for Fortran diff tests""" return { # 'Captain Daves doomed test of destruction': @@ -65,7 +65,7 @@ def get_diff_dispatch_table_fortran(self) -> Dict[str, Callable]: def get_file_dispatch_table_fortran( self, filename: str = "" - ) -> Dict[str, Callable]: + ) -> dict[str, Callable]: """Get dispatch table for Fortran file tests""" return { "Warning - used an if-def due for retirement": self.umdp3_checker.retire_if_def, @@ -78,7 +78,7 @@ def get_file_dispatch_table_fortran( + "[1,2,3] form": self.umdp3_checker.array_init_form, } - def get_diff_dispatch_table_c(self) -> Dict[str, Callable]: + def get_diff_dispatch_table_c(self) -> dict[str, Callable]: """Get dispatch table for C diff tests""" return { "Line longer than 80 characters": self.umdp3_checker.line_over_80chars, @@ -87,7 +87,7 @@ def get_diff_dispatch_table_c(self) -> Dict[str, Callable]: + 'between themselves and the string delimiter (the " character)': self.umdp3_checker.c_integral_format_specifiers, } - def get_file_dispatch_table_c(self) -> Dict[str, Callable]: + def get_file_dispatch_table_c(self) -> dict[str, Callable]: """Get dispatch table for C file tests""" return { "Warning - used an if-def due for retirement": self.umdp3_checker.retire_if_def, @@ -108,7 +108,7 @@ def get_file_dispatch_table_c(self) -> Dict[str, Callable]: "C Unit does not end with a final newline character": self.umdp3_checker.c_final_newline, } - def get_file_dispatch_table_all(self) -> Dict[str, Callable]: + def get_file_dispatch_table_all(self) -> dict[str, Callable]: """Get dispatch table for universal file tests""" return { "Line includes trailing whitespace character(s)": self.umdp3_checker.line_trail_whitespace, diff --git a/script_umdp3_checker/umdp3_checker_rules.py b/script_umdp3_checker/umdp3_checker_rules.py index c18270da..58df4d96 100644 --- a/script_umdp3_checker/umdp3_checker_rules.py +++ b/script_umdp3_checker/umdp3_checker_rules.py @@ -11,7 +11,6 @@ import re import threading -from typing import List, Dict from fortran_keywords import fortran_keywords from search_lists import ( obsolescent_intrinsics, @@ -46,7 +45,7 @@ class TestResult: failure_count: int = 0 passed: bool = False output: str = "" - errors: Dict = field(default_factory=dict) + errors: dict = field(default_factory=dict) class UMDP3Checker: @@ -84,7 +83,7 @@ def reset_extra_error_information(self): with self._lock: self._extra_error_info = {} - def get_extra_error_information(self) -> Dict: + def get_extra_error_information(self) -> dict: """ Get extra error information. Dictionary with file names as the keys. """ @@ -106,8 +105,8 @@ def add_extra_error(self, key: str, value: str = ""): self._extra_error_info[key] = value def add_error_log( - self, error_log: Dict, key: str = "no key", value: int = 0 - ) -> Dict: + self, error_log: dict, key: str = "no key", value: int = 0 + ) -> dict: """Add extra error information to the dictionary""" """ TODO: This is a bodge to get more detailed info about @@ -158,7 +157,7 @@ def remove_quoted(self, line: str) -> str: Although, a brief look seems to imply that there are two 'dispatch tables' one for full files and one for changed lines.""" - def capitulated_keywords(self, lines: List[str]) -> TestResult: + def capitulated_keywords(self, lines: list[str]) -> TestResult: """A fake test, put in for testing purposes. Probably not needed any more, but left in case.""" failures = 0 @@ -192,7 +191,7 @@ def capitulated_keywords(self, lines: List[str]) -> TestResult: errors=error_log, ) - def capitalised_keywords(self, lines: List[str]) -> TestResult: + def capitalised_keywords(self, lines: list[str]) -> TestResult: """Check for the presence of lowercase Fortran keywords, which are taken from an imported list 'fortran_keywords'.""" failures = 0 @@ -222,7 +221,7 @@ def capitalised_keywords(self, lines: List[str]) -> TestResult: errors=error_log, ) - def openmp_sentinels_in_column_one(self, lines: List[str]) -> TestResult: + def openmp_sentinels_in_column_one(self, lines: list[str]) -> TestResult: """Check OpenMP sentinels are in column one""" failures = 0 error_log = {} @@ -243,7 +242,7 @@ def openmp_sentinels_in_column_one(self, lines: List[str]) -> TestResult: errors=error_log, ) - def unseparated_keywords(self, lines: List[str]) -> TestResult: + def unseparated_keywords(self, lines: list[str]) -> TestResult: """Check for omitted optional spaces in keywords""" failures = 0 error_log = {} @@ -270,7 +269,7 @@ def unseparated_keywords(self, lines: List[str]) -> TestResult: errors=error_log, ) - def go_to_other_than_9999(self, lines: List[str]) -> TestResult: + def go_to_other_than_9999(self, lines: list[str]) -> TestResult: """Check for GO TO statements other than 9999""" failures = 0 error_log = {} @@ -296,7 +295,7 @@ def go_to_other_than_9999(self, lines: List[str]) -> TestResult: errors=error_log, ) - def write_using_default_format(self, lines: List[str]) -> TestResult: + def write_using_default_format(self, lines: list[str]) -> TestResult: """Check for WRITE without format""" failures = 0 error_log = {} @@ -318,7 +317,7 @@ def write_using_default_format(self, lines: List[str]) -> TestResult: errors=error_log, ) - def lowercase_variable_names(self, lines: List[str]) -> TestResult: + def lowercase_variable_names(self, lines: list[str]) -> TestResult: """Check for lowercase or CamelCase variable names only""" """ TODO: This is a very simplistic check and will not detect many @@ -360,7 +359,7 @@ def lowercase_variable_names(self, lines: List[str]) -> TestResult: errors=error_log, ) - def dimension_forbidden(self, lines: List[str]) -> TestResult: + def dimension_forbidden(self, lines: list[str]) -> TestResult: """Check for use of dimension attribute""" failures = 0 error_log = {} @@ -385,7 +384,7 @@ def dimension_forbidden(self, lines: List[str]) -> TestResult: errors=error_log, ) - def ampersand_continuation(self, lines: List[str]) -> TestResult: + def ampersand_continuation(self, lines: list[str]) -> TestResult: """Check continuation lines shouldn't start with &""" failures = 0 error_log = {} @@ -406,7 +405,7 @@ def ampersand_continuation(self, lines: List[str]) -> TestResult: errors=error_log, ) - def forbidden_keywords(self, lines: List[str]) -> TestResult: + def forbidden_keywords(self, lines: list[str]) -> TestResult: """Check for use of EQUIVALENCE or PAUSE""" """ TODO: Can't believe this will allow a COMMON BLOCK.... @@ -433,7 +432,7 @@ def forbidden_keywords(self, lines: List[str]) -> TestResult: errors=error_log, ) - def forbidden_operators(self, lines: List[str]) -> TestResult: + def forbidden_operators(self, lines: list[str]) -> TestResult: """Check for older form of relational operators""" failures = 0 error_log = {} @@ -460,7 +459,7 @@ def forbidden_operators(self, lines: List[str]) -> TestResult: errors=error_log, ) - def line_over_80chars(self, lines: List[str]) -> TestResult: + def line_over_80chars(self, lines: list[str]) -> TestResult: """Check for lines longer than 80 characters""" failures = 0 error_log = {} @@ -479,7 +478,7 @@ def line_over_80chars(self, lines: List[str]) -> TestResult: errors=error_log, ) - def tab_detection(self, lines: List[str]) -> TestResult: + def tab_detection(self, lines: list[str]) -> TestResult: """Check for tab characters""" failures = 0 error_log = {} @@ -500,7 +499,7 @@ def tab_detection(self, lines: List[str]) -> TestResult: errors=error_log, ) - def printstatus_mod(self, lines: List[str]) -> TestResult: + def printstatus_mod(self, lines: list[str]) -> TestResult: """Check for use of printstatus_mod instead of umPrintMgr""" failures = 0 error_log = {} @@ -521,7 +520,7 @@ def printstatus_mod(self, lines: List[str]) -> TestResult: errors=error_log, ) - def printstar(self, lines: List[str]) -> TestResult: + def printstar(self, lines: list[str]) -> TestResult: """Check for PRINT rather than umMessage and umPrint""" failures = 0 error_log = {} @@ -543,7 +542,7 @@ def printstar(self, lines: List[str]) -> TestResult: errors=error_log, ) - def write6(self, lines: List[str]) -> TestResult: + def write6(self, lines: list[str]) -> TestResult: """Check for WRITE(6) rather than umMessage and umPrint""" failures = 0 error_log = {} @@ -565,7 +564,7 @@ def write6(self, lines: List[str]) -> TestResult: errors=error_log, ) - def um_fort_flush(self, lines: List[str]) -> TestResult: + def um_fort_flush(self, lines: list[str]) -> TestResult: """Check for um_fort_flush rather than umPrintFlush""" failures = 0 error_log = {} @@ -585,7 +584,7 @@ def um_fort_flush(self, lines: List[str]) -> TestResult: errors=error_log, ) - def svn_keyword_subst(self, lines: List[str]) -> TestResult: + def svn_keyword_subst(self, lines: list[str]) -> TestResult: """Check for Subversion keyword substitution""" failures = 0 error_log = {} @@ -605,7 +604,7 @@ def svn_keyword_subst(self, lines: List[str]) -> TestResult: errors=error_log, ) - def omp_missing_dollar(self, lines: List[str]) -> TestResult: + def omp_missing_dollar(self, lines: list[str]) -> TestResult: """Check for !OMP instead of !$OMP""" failures = 0 error_log = {} @@ -624,7 +623,7 @@ def omp_missing_dollar(self, lines: List[str]) -> TestResult: errors=error_log, ) - def cpp_ifdef(self, lines: List[str]) -> TestResult: + def cpp_ifdef(self, lines: list[str]) -> TestResult: """Check for #ifdef/#ifndef rather than #if defined()""" failures = 0 error_log = {} @@ -645,7 +644,7 @@ def cpp_ifdef(self, lines: List[str]) -> TestResult: errors=error_log, ) - def cpp_comment(self, lines: List[str]) -> TestResult: + def cpp_comment(self, lines: list[str]) -> TestResult: """Check for Fortran comments in CPP directives""" """ TODO: This looks like it will incorrectly fail # if !defined(X) @@ -673,7 +672,7 @@ def cpp_comment(self, lines: List[str]) -> TestResult: errors=error_log, ) - def obsolescent_fortran_intrinsic(self, lines: List[str]) -> TestResult: + def obsolescent_fortran_intrinsic(self, lines: list[str]) -> TestResult: """Check for archaic Fortran intrinsic functions""" failures = 0 error_log = {} @@ -698,7 +697,7 @@ def obsolescent_fortran_intrinsic(self, lines: List[str]) -> TestResult: errors=error_log, ) - def exit_stmt_label(self, lines: List[str]) -> TestResult: + def exit_stmt_label(self, lines: list[str]) -> TestResult: """Check that EXIT statements are labelled""" failures = 0 error_log = {} @@ -722,7 +721,7 @@ def exit_stmt_label(self, lines: List[str]) -> TestResult: errors=error_log, ) - def intrinsic_modules(self, lines: List[str]) -> TestResult: + def intrinsic_modules(self, lines: list[str]) -> TestResult: """Check intrinsic modules are USEd with INTRINSIC keyword""" failures = 0 intrinsic_modules = ["ISO_C_BINDING", "ISO_FORTRAN_ENV"] @@ -752,7 +751,7 @@ def intrinsic_modules(self, lines: List[str]) -> TestResult: errors=error_log, ) - def read_unit_args(self, lines: List[str]) -> TestResult: + def read_unit_args(self, lines: list[str]) -> TestResult: """Check READ statements have explicit UNIT= as first argument""" failures = 0 error_log = {} @@ -778,7 +777,7 @@ def read_unit_args(self, lines: List[str]) -> TestResult: errors=error_log, ) - def retire_if_def(self, lines: List[str]) -> TestResult: + def retire_if_def(self, lines: list[str]) -> TestResult: """Check for if-defs due for retirement""" # retired_ifdefs = ['VATPOLES', 'A12_4A', 'A12_3A', 'UM_JULES', # 'A12_2A',] @@ -813,7 +812,7 @@ def retire_if_def(self, lines: List[str]) -> TestResult: errors=error_log, ) - def implicit_none(self, lines: List[str]) -> TestResult: + def implicit_none(self, lines: list[str]) -> TestResult: """Check file has at least one IMPLICIT NONE""" error_log = {} no_implicit_none = True @@ -836,7 +835,7 @@ def implicit_none(self, lines: List[str]) -> TestResult: errors=error_log, ) - def forbidden_stop(self, lines: List[str]) -> TestResult: + def forbidden_stop(self, lines: list[str]) -> TestResult: """Check for STOP or CALL abort""" failures = 0 error_log = {} @@ -860,7 +859,7 @@ def forbidden_stop(self, lines: List[str]) -> TestResult: errors=error_log, ) - def intrinsic_as_variable(self, lines: List[str]) -> TestResult: + def intrinsic_as_variable(self, lines: list[str]) -> TestResult: """Check for Fortran function used as variable name""" failures = 0 error_log = {} @@ -894,7 +893,7 @@ def intrinsic_as_variable(self, lines: List[str]) -> TestResult: errors=error_log, ) - def check_crown_copyright(self, lines: List[str]) -> TestResult: + def check_crown_copyright(self, lines: list[str]) -> TestResult: """Check for crown copyright statement""" """ TODO: This is a very simplistic check and will not detect many @@ -922,7 +921,7 @@ def check_crown_copyright(self, lines: List[str]) -> TestResult: errors=error_log, ) - def check_code_owner(self, lines: List[str]) -> TestResult: + def check_code_owner(self, lines: list[str]) -> TestResult: """Check for correct code owner comment""" """ TODO: oh wow is this test worthless. We don't even guarentee to put @@ -951,7 +950,7 @@ def check_code_owner(self, lines: List[str]) -> TestResult: errors=error_log, ) - def array_init_form(self, lines: List[str]) -> TestResult: + def array_init_form(self, lines: list[str]) -> TestResult: """Check for old array initialization form""" """ TODO: Another instance that assumes continuation lines are @@ -977,7 +976,7 @@ def array_init_form(self, lines: List[str]) -> TestResult: errors=error_log, ) - def line_trail_whitespace(self, lines: List[str]) -> TestResult: + def line_trail_whitespace(self, lines: list[str]) -> TestResult: """Check for trailing whitespace""" failures = 0 error_log = {} @@ -998,7 +997,7 @@ def line_trail_whitespace(self, lines: List[str]) -> TestResult: # C-specific tests - def c_integral_format_specifiers(self, lines: List[str]) -> int: + def c_integral_format_specifiers(self, lines: list[str]) -> int: """Check C integral format specifiers have space""" failures = 0 for line in lines: @@ -1008,7 +1007,7 @@ def c_integral_format_specifiers(self, lines: List[str]) -> int: return failures - def c_deprecated(self, lines: List[str]) -> int: + def c_deprecated(self, lines: list[str]) -> int: """Check for deprecated C identifiers""" failures = 0 for line in lines: @@ -1019,7 +1018,7 @@ def c_deprecated(self, lines: List[str]) -> int: return failures - def c_openmp_define_pair_thread_utils(self, lines: List[str]) -> int: + def c_openmp_define_pair_thread_utils(self, lines: list[str]) -> int: """Check C OpenMP define pairing with thread utils""" failures = 0 for line in lines: @@ -1032,7 +1031,7 @@ def c_openmp_define_pair_thread_utils(self, lines: List[str]) -> int: return failures - def c_openmp_define_no_combine(self, lines: List[str]) -> int: + def c_openmp_define_no_combine(self, lines: list[str]) -> int: """Check C OpenMP defines not combined with third macro""" failures = 0 for line in lines: @@ -1046,7 +1045,7 @@ def c_openmp_define_no_combine(self, lines: List[str]) -> int: return failures - def c_openmp_define_not(self, lines: List[str]) -> int: + def c_openmp_define_not(self, lines: list[str]) -> int: """Check for !defined(_OPENMP) usage""" failures = 0 for line in lines: @@ -1056,7 +1055,7 @@ def c_openmp_define_not(self, lines: List[str]) -> int: return failures - def c_protect_omp_pragma(self, lines: List[str]) -> int: + def c_protect_omp_pragma(self, lines: list[str]) -> int: """Check OMP pragma is protected with ifdef""" failures = 0 in_openmp_block = False @@ -1075,7 +1074,7 @@ def c_protect_omp_pragma(self, lines: List[str]) -> int: return failures - def c_ifdef_defines(self, lines: List[str]) -> int: + def c_ifdef_defines(self, lines: list[str]) -> int: """Check for #ifdef style rather than #if defined()""" failures = 0 for line in lines: @@ -1085,7 +1084,7 @@ def c_ifdef_defines(self, lines: List[str]) -> int: return failures - def c_final_newline(self, lines: List[str]) -> int: + def c_final_newline(self, lines: list[str]) -> int: """Check C unit ends with final newline""" if lines and not lines[-1].endswith("\n"): self.add_extra_error("missing final newline") diff --git a/script_umdp3_checker/umdp3_conformance.py b/script_umdp3_checker/umdp3_conformance.py index 0e91783d..28dfd765 100644 --- a/script_umdp3_checker/umdp3_conformance.py +++ b/script_umdp3_checker/umdp3_conformance.py @@ -1,7 +1,7 @@ import subprocess from abc import ABC, abstractmethod from pathlib import Path -from typing import Callable, List, Dict, Set +from collections.abc import Callable from dataclasses import dataclass, field import argparse from checker_dispatch_tables import CheckerDispatchTables @@ -43,14 +43,14 @@ class CheckResult: file_path: str = "No file provided" tests_failed: int = 0 all_passed: bool = False - test_results: List[TestResult] = field(default_factory=list) + test_results: list[TestResult] = field(default_factory=list) class CMSSystem(ABC): """Abstract base class for CMS systems like git or FCM.""" @abstractmethod - def get_changed_files(self) -> List[Path]: + def get_changed_files(self) -> list[Path]: """Get list of files changed between base_branch and branch.""" pass @@ -75,7 +75,7 @@ def __init__(self, repo_path: Path = Path(".")): self.bdiff_obj = git_bdiff.GitBDiff(repo=self.repo_path) self.info_obj = git_bdiff.GitInfo(repo=self.repo_path) - def get_changed_files(self) -> List[Path]: + def get_changed_files(self) -> list[Path]: """Get list of files changed between base_branch and branch.""" return [Path(f) for f in self.bdiff_obj.files()] @@ -98,7 +98,7 @@ def __init__(self, repo_path: Path = Path(".")): self.repo_path = repo_path self.bdiff_obj = fcm_bdiff.FCMBDiff(repo=self.repo_path) - def get_changed_files(self) -> List[Path]: + def get_changed_files(self) -> list[Path]: """Get list of files changed between base_branch and branch.""" return [Path(f) for f in self.bdiff_obj.files()] @@ -118,7 +118,7 @@ class StyleChecker(ABC): TODO: This is where it might be good to set up a threadsafe class instance to hold the 'expanded' check outputs. One for each file being checked in parallel. - Curently the UMDP3 class holds "_extra_error_info" which + Currently the UMDP3 class holds "_extra_error_info" which was used to provide more detailed error logging. However, this is not threadsafe, so in a multithreaded environment, the extra error info could get mixed up between @@ -127,16 +127,16 @@ class instance to hold the 'expanded' check outputs. a TestResult object directly, which includes the extra error info, so that each thread can work independently.""" name: str - file_extensions: Set[str] - check_functions: Dict[str, Callable] - files_to_check: List[Path] + file_extensions: set[str] + check_functions: dict[str, Callable] + files_to_check: list[Path] def __init__( self, name: str, - file_extensions: Set[str], - check_functions: Dict[str, Callable], - changed_files: List[Path] = [], + file_extensions: set[str], + check_functions: dict[str, Callable], + changed_files: list[Path] = [], ): self.name = name self.file_extensions = file_extensions or set() @@ -161,9 +161,9 @@ def check(self, file_path: Path) -> CheckResult: def from_full_list( cls, name: str, - file_extensions: Set[str], - check_functions: Dict[str, Callable], - all_files: List[Path], + file_extensions: set[str], + check_functions: dict[str, Callable], + all_files: list[Path], ) -> "StyleChecker": """Create a StyleChecker instance filtering files from a full list.""" filtered_files = cls.filter_files(all_files, file_extensions) @@ -171,8 +171,8 @@ def from_full_list( @staticmethod def filter_files( - files: List[Path], file_extensions: Set[str] = set() - ) -> List[Path]: + files: list[Path], file_extensions: set[str] = set() + ) -> list[Path]: """Filter files based on the checker's file extensions.""" if not file_extensions: return files @@ -182,14 +182,14 @@ def filter_files( class UMDP3_checker(StyleChecker): """UMDP3 built-in style checker.""" - files_to_check: List[Path] + files_to_check: list[Path] def __init__( self, name: str, - file_extensions: Set[str], - check_functions: Dict[str, Callable], - changed_files: List[Path] = [], + file_extensions: set[str], + check_functions: dict[str, Callable], + changed_files: list[Path] = [], print_volume: int = 3, ): self.name = name @@ -216,7 +216,7 @@ def check(self, file_path: Path) -> CheckResult: """Run UMDP3 check function on file.""" lines = file_path.read_text().splitlines() file_results = [] # list of TestResult objects - for check_name, check_function in self.check_functions.items(): + for _check_name, check_function in self.check_functions.items(): file_results.append(check_function(lines)) tests_failed = sum([0 if result.passed else 1 for result in file_results]) @@ -237,14 +237,14 @@ class ExternalChecker(StyleChecker): Ideally we should be making callable functions for each check, but that would require more refactoring of the code. Is that a 'factory' method?""" - check_commands: Dict[str, List[str]] + check_commands: dict[str, list[str]] def __init__( self, name: str, - file_extensions: Set[str], - check_functions: Dict[str, List[str]], - changed_files: List[Path], + file_extensions: set[str], + check_functions: dict[str, list[str]], + changed_files: list[Path], print_volume: int = 3, ): self.name = name @@ -323,7 +323,7 @@ class ConformanceChecker: def __init__( self, - checkers: List[StyleChecker], + checkers: list[StyleChecker], max_workers: int = 8, ): self.checkers = checkers @@ -535,7 +535,7 @@ def which_cms_is_it(path: str, print_volume: int = 3) -> CMSSystem: return cms -def detangle_file_types(file_types: Set[str]) -> Set[str]: +def detangle_file_types(file_types: set[str]) -> set[str]: """Process file type arguments to handle 'group' types.""" the_whole_world = set(ALLOWABLE_FILE_TYPES) the_whole_world.update(list(GROUP_FILE_TYPES.keys())) @@ -555,8 +555,8 @@ def detangle_file_types(file_types: Set[str]) -> Set[str]: def create_style_checkers( - file_types: List[str], changed_files: List[Path], print_volume: int = 3 -) -> List[StyleChecker]: + file_types: list[str], changed_files: list[Path], print_volume: int = 3 +) -> list[StyleChecker]: """Create style checkers based on requested file types.""" dispatch_tables = CheckerDispatchTables() checkers = [] @@ -603,7 +603,7 @@ def create_style_checkers( def get_files_to_check( path: str, full_check: bool, print_volume: int = 3 -) -> List[Path]: +) -> list[Path]: """ Docstring for get_files_to_check : A routine to get the list of files to check based on the CMS or the full check override. diff --git a/suite_report.py b/suite_report.py index bff139cb..147d5845 100755 --- a/suite_report.py +++ b/suite_report.py @@ -27,8 +27,6 @@ # pylint: disable=too-many-lines -from __future__ import print_function - import glob import json import os @@ -177,11 +175,11 @@ def _read_file(filename): """Takes filename (str) Return contents of a file, as list of strings.""" if os.path.exists(filename): - with open(filename, "r", encoding="utf-8") as filehandle: + with open(filename, encoding="utf-8") as filehandle: lines = filehandle.readlines() else: print(f'[ERROR] Unable to find file :\n "{filename}"') - raise IOError(f'_read_file got invalid filename : "{filename}"') + raise OSError(f'_read_file got invalid filename : "{filename}"') return lines @@ -219,7 +217,7 @@ def _run_command(command, ignore_fail=False): print(f"[INFO] RC: {retcode}") print(f"[INFO] Stdout: {stdout}") print(f"[INFO] Stderr: {stderr}") - raise IOError("run_command") + raise OSError("run_command") # Reformat stdout into a list stdout = "".join(stdout) stdout = stdout.split("\n") @@ -874,7 +872,7 @@ def clean_tempfile(fname="~/temp.txt"): try: os.remove(os.path.expanduser(fname)) - except EnvironmentError: + except OSError: pass def generate_owner_dictionary(self, mode): @@ -908,7 +906,7 @@ def generate_owner_dictionary(self, mode): # Read through file and generate dictionary try: - with open(file_path, "r", encoding="utf-8") as inp_file: + with open(file_path, encoding="utf-8") as inp_file: owners_dict = {} inside_listing = False for line in inp_file: @@ -932,7 +930,7 @@ def generate_owner_dictionary(self, mode): except IndexError: others = "" owners_dict.update({section.lower(): [owners, others]}) - except EnvironmentError: + except OSError: print("Can't find working copy for Owners File") return None @@ -1124,14 +1122,14 @@ def get_code_owners(self, code_owners): file_path = "" try: - with open(file_path, "r", encoding="utf-8") as inp_file: + with open(file_path, encoding="utf-8") as inp_file: for line in inp_file: if "file belongs in" in line: section = line.strip("\n") break else: section = "" - except EnvironmentError: + except OSError: section = "" # Clean up checked out file @@ -1299,7 +1297,7 @@ def check_lfric_extract_list(self): extract_list_dict[item] += temp_dict[item] else: extract_list_dict[item] = temp_dict[item] - except (EnvironmentError, TypeError, AttributeError): + except (OSError, TypeError, AttributeError): # Potential error here changed type between python2 and 3 extract_list_path = None @@ -2011,7 +2009,7 @@ def print_report(self): # even in event of serious exceptions try: _write_file(trac_log_path, trac_log, newline=True) - except IOError: + except OSError: print(f"[ERROR] Writing to {TRAC_LOG_FILE} file : {trac_log_path}") print( f"{TRAC_LOG_FILE} to this point " + "would have read as follows :\n" diff --git a/umdp3_fixer/ampersands.py b/umdp3_fixer/ampersands.py index 9d8debcb..cf6d39a1 100755 --- a/umdp3_fixer/ampersands.py +++ b/umdp3_fixer/ampersands.py @@ -21,7 +21,7 @@ stdout. Note that this code has made an effort to deal with the possibility of -amersands and exclamation marks within quoted strings or comments, but there +ampersands and exclamation marks within quoted strings or comments, but there may be some cases which are missed. These lines will be left without applying the ampersand shifting, and will be flagged, optionally with a message in stdout. @@ -56,8 +56,8 @@ class CharError(ParsingError): def __init__(self, char, number): self.number = number self.char = char - self.msg = 'There are {0:d} unquoted, uncommented "{1:s}" in this line.'.format( - number, char + self.msg = ( + f'There are {number:d} unquoted, uncommented "{char:s}" in this line.' ) pass @@ -70,7 +70,7 @@ def print_message(errtype, msg, iline=None, line=None, fname=None): if fname is None: fnamestr = "" else: - fnamestr = "{0:s}:".format(fname) + fnamestr = f"{fname:s}:" if iline is None: if fnamestr is None: @@ -78,16 +78,14 @@ def print_message(errtype, msg, iline=None, line=None, fname=None): else: ilinestr = " " else: - ilinestr = "L{0:05d}: ".format(iline) + ilinestr = f"L{iline:05d}: " if line is None: linestr = "" else: - linestr = ": {0:s}".format(line) + linestr = f": {line:s}" - print( - "{0:s}{1:s}{2:s} - {3:s}{4:s}".format(fnamestr, ilinestr, errtype, msg, linestr) - ) + print(f"{fnamestr:s}{ilinestr:s}{errtype:s} - {msg:s}{linestr:s}") def shift_ampersand( @@ -104,7 +102,7 @@ def shift_ampersand( required line length. """ - # return earliy if there are no apersands at all. + # return early if there are no amerinds at all. if "&" not in line: return line @@ -263,11 +261,11 @@ def shift_ampersand( # If the line contains an unquoted or uncommented ampersand, need # to check if it is in the right place. if amp_loc != -1: - # If there is no comment, determin if it is a leading continuation + # If there is no comment, determine if it is a leading continuation # ampersand. If it is remove it; else shift the ampersand (and remove # any trailing whitespace). if comment_loc == -1: - # determin if we are a leading ampersand + # determine if we are a leading ampersand beforeline = workline[lp:amp_loc].rstrip() if len(beforeline) == 0: @@ -329,7 +327,7 @@ def shift_ampersand( # Try cutting down the whitespace one step at a time until # there is only one space left. - for i in range(nloop): + for _i in range(nloop): if workline[amp_location - 1] == " ": # If there is still whitespace that can be removed, # remove it and update the ampersand location. @@ -380,7 +378,7 @@ def apply_ampersand_shift( if debug: print_message( "PARSING ERROR", - "{0:s} Ampersand shifting has not been applied".format(e.msg), + f"{e.msg:s} Ampersand shifting has not been applied", iline + 1, line=line, fname=fname, @@ -435,7 +433,7 @@ def apply_check_line_len(lines, fname=None, maxlinelen=DEFAULT_COL, debug=False) if debug: print_message( "VIOLATION", - "Line > {0:d} columns".format(maxlinelen), + f"Line > {maxlinelen:d} columns", iline + 1, line=line, fname=fname, @@ -499,9 +497,9 @@ def main(): if ilines_too_long is not None: print_message( "WARNING", - "Some lines are longer than {0:d} characters. " + f"Some lines are longer than {opts.col:d} characters. " "Please check and make them " - "shorter.".format(opts.col), + "shorter.", fname=input_file, ) diff --git a/umdp3_fixer/fstring_parse.py b/umdp3_fixer/fstring_parse.py index b557cf92..4a83e963 100644 --- a/umdp3_fixer/fstring_parse.py +++ b/umdp3_fixer/fstring_parse.py @@ -42,8 +42,8 @@ def __init__(self, quote, number): self.msg = ( "There are an odd number of non-commented " - "and non-quoted {1:s} in this line. " - "(From a total of {0:d})".format(number, self.quotemarks[quote]) + f"and non-quoted {self.quotemarks[quote]:s} in this line. " + f"(From a total of {number:d})" ) pass @@ -258,7 +258,7 @@ def is_str_continuation_preparblank(parblanked, line): first_loc = min(apos_loc, quot_loc) - # set the correct return based on the first occurance character + # set the correct return based on the first occurrence character if line[first_loc] == "'": cont[SQUOTE] = True else: @@ -297,7 +297,7 @@ def simplify_line(lines): # Note we will be passed a slice to include the lines after the # current line to the end of the file to allow handling of - # continutations + # continuations iline = 0 line = lines[iline] @@ -321,7 +321,7 @@ def simplify_line(lines): str_cont_test = is_str_continuation(line) if not (str_cont_test[SQUOTE] or str_cont_test[DQUOTE]): print("Indentation simplify line has failed. [2]") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -376,7 +376,7 @@ def simplify_line(lines): line = blank_fstring(line) except ParsingError as e: print("Indentation simplify line has failed. [3]") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) diff --git a/umdp3_fixer/indentation.py b/umdp3_fixer/indentation.py index 8b3747db..fbb6ab16 100755 --- a/umdp3_fixer/indentation.py +++ b/umdp3_fixer/indentation.py @@ -127,7 +127,7 @@ def apply_indentation(lines, debug=False): for iline, line in enumerate(lines): if debug: - print('\n{0:d}: "{1:s}"'.format(iline, line)) + print(f'\n{iline:d}: "{line:s}"') # If this line is continuing a previous preprocessing line, # just ignore indentation @@ -262,11 +262,7 @@ def apply_indentation(lines, debug=False): simple_line = simplify_line(lines[iline:]) if debug: - print( - "??" - + " " * len("{0:d}".format(iline)) - + '"{0:s}"'.format(simple_line) - ) + print("??" + " " * len(f"{iline:d}") + f'"{simple_line:s}"') # Check for ending statements first - since the indentation # shift for the end of a block must also be applied to the @@ -274,7 +270,7 @@ def apply_indentation(lines, debug=False): for pattern in INDENTATION_END: if re.search(pattern, simple_line, flags=re.IGNORECASE): if debug: - print(" (End, matches {0:s})".format(pattern)) + print(f" (End, matches {pattern:s})") indentation -= INDENT # Get the current indentation level of the line @@ -289,11 +285,7 @@ def apply_indentation(lines, debug=False): indented_line = indent_line(line, relative_indent) if debug: - print( - "=>" - + " " * len("{0:d}".format(iline)) - + '"{0:s}"'.format(indented_line) - ) + print("=>" + " " * len(f"{iline:d}") + f'"{indented_line:s}"') new_lines.append(indented_line) @@ -303,7 +295,7 @@ def apply_indentation(lines, debug=False): for pattern in INDENTATION_START: if re.search(pattern, simple_line, flags=re.IGNORECASE): if debug: - print(" (Start, matches {0:s})".format(pattern)) + print(f" (Start, matches {pattern:s})") indentation += INDENT # Finally, detect if the following line is a continuation @@ -335,7 +327,7 @@ def apply_indentation(lines, debug=False): return None if indentation != 0: - print("Final indentation level non-zero ({0:d})".format(indentation)) + print(f"Final indentation level non-zero ({indentation:d})") return None return new_lines diff --git a/umdp3_fixer/rosestem_branch_checker.py b/umdp3_fixer/rosestem_branch_checker.py index 90c9bcc9..0640a6dc 100755 --- a/umdp3_fixer/rosestem_branch_checker.py +++ b/umdp3_fixer/rosestem_branch_checker.py @@ -48,22 +48,23 @@ def copy_working_branch(model_source): shutil.copytree(src_wkcopy, tmp_path) # Directories are the same except shutil.Error as err: - print("Directory not copied. Error: %s" % err) + print(f"Directory not copied. Error: {err}") # Any error saying that the directory doesn't exist except OSError as err: - print("Directory not copied. Error: %s" % err) + print(f"Directory not copied. Error: {err}") return tmp_path, saved_umask, tmpdir def diff_cwd_working(model_source, path, saved_umask, tmpdir): """Diff the tmp dir with the working branch and report diff.""" diff = subprocess.run( - "diff -qr " + model_source + "/src " + path, + ["diff", "-qr", f"{model_source}/src", path], stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, - universal_newlines=True, + capture_output=True, + text=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + # shell=False, ) if diff.returncode == 0: diff --git a/umdp3_fixer/styling.py b/umdp3_fixer/styling.py index a7af2cca..dddffa13 100755 --- a/umdp3_fixer/styling.py +++ b/umdp3_fixer/styling.py @@ -818,7 +818,7 @@ def replace_patterns(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -850,7 +850,7 @@ def replace_patterns(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -887,7 +887,7 @@ def replace_comment_patterns(line, str_continuation): match_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -916,7 +916,7 @@ def replace_comment_patterns(line, str_continuation): match_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -945,7 +945,7 @@ def upcase_keywords(line, str_continuation): simple_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -958,19 +958,15 @@ def upcase_keywords(line, str_continuation): for word in line_words: # Exclude special "__FILE__" or "__LINE__" directives if word.isupper() and not re.match(r"__\w+__", word): - recomp = re.compile(r"(^|\b){0:s}(\b|$)".format(word)) - simple_line = recomp.sub( - r"\g<1>{0:s}" r"\g<2>".format(word.lower()), simple_line - ) + recomp = re.compile(rf"(^|\b){word:s}(\b|$)") + simple_line = recomp.sub(rf"\g<1>{word.lower():s}" r"\g<2>", simple_line) line_words = set([word.lower() for word in line_words]) words_to_upcase = list(line_words.intersection(KEYWORDS)) line = list(line) for keyword in words_to_upcase: - recomp = re.compile(r"(?i)(^|\b){0:s}(\b|$)".format(keyword)) - simple_line = recomp.sub( - r"\g<1>{0:s}\g<2>".format(keyword.upper()), simple_line - ) + recomp = re.compile(rf"(?i)(^|\b){keyword:s}(\b|$)") + simple_line = recomp.sub(rf"\g<1>{keyword.upper():s}\g<2>", simple_line) # Now add back any comments/strings simple_line = list(simple_line) @@ -1002,9 +998,7 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): found_dec_type = None for declare_type in FORTRAN_TYPES: - if re.search( - r"^\s*{0:s}\W".format(declare_type), workline, flags=re.IGNORECASE - ): + if re.search(rf"^\s*{declare_type:s}\W", workline, flags=re.IGNORECASE): found_dec_type = declare_type break @@ -1023,7 +1017,7 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # older "*INT" declaration the first character should # not be a comma search = re.search( - r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format(found_dec_type), + rf"^(\s*{found_dec_type:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)", simple_line, flags=re.IGNORECASE, ) @@ -1044,15 +1038,15 @@ def declaration_double_colon(iline, lines, pp_line_previous, line_previous): # Attempt to fit the double-colon into an existing space to # preserve indentation, otherwise just add it to the line line = re.sub( - r"^{0:s}" r"\s\s\s\s".format(re.escape(statement)), - r"{0:s} :: ".format(statement), + rf"^{re.escape(statement):s}" r"\s\s\s\s", + rf"{statement:s} :: ", line, count=1, ) line = re.sub( - r"^{0:s}\s*" - r"((?".format(statement), + rf"^{re.escape(statement):s}\s*" + r"((?", line, count=1, ) diff --git a/umdp3_fixer/umdp3_fixer.py b/umdp3_fixer/umdp3_fixer.py index 9b95dcb0..eb935710 100755 --- a/umdp3_fixer/umdp3_fixer.py +++ b/umdp3_fixer/umdp3_fixer.py @@ -198,7 +198,7 @@ def main(): if len(f_files) > 0: print("\nProcessing Fortran Files") for input_file in f_files: - print("Processing: {0:s}".format(input_file)) + print(f"Processing: {input_file:s}") sys.stdout.flush() if ( input_file.split(".")[-1] != "F90" @@ -208,17 +208,13 @@ def main(): if input_file.split(".")[-1] == "h": if re.search(r".*\/include\/other\/.*", input_file) is not None: print( - "Input file {0:s} not a " + f"Input file {input_file:s} not a " "Fortran include file," - " skipping".format(input_file) + " skipping" ) continue else: - print( - "Input file {0:s} not a Fortran file, skipping".format( - input_file - ) - ) + print(f"Input file {input_file:s} not a Fortran file, skipping") continue with open(input_file, "r+", errors="replace") as file_in: @@ -241,9 +237,7 @@ def main(): ) if len(amp_not_parsed) > 0: - print( - "Ampersand Alignment Failed for: {0:s}".format(input_file) - ) + print(f"Ampersand Alignment Failed for: {input_file:s}") print("failed on lines:\n") for i in amp_not_parsed: print(str(i) + ': "' + modify_lines[i] + '"') @@ -256,7 +250,7 @@ def main(): white_lines = apply_whitespace_fixes(amp_lines) if white_lines is None: - print("Whitespace Fixes Failed for: {0:s}".format(input_file)) + print(f"Whitespace Fixes Failed for: {input_file:s}") failed = True styled_lines = None @@ -265,7 +259,7 @@ def main(): styled_lines = apply_styling(white_lines) if styled_lines is None: - print("Styling Failed for: {0:s}".format(input_file)) + print(f"Styling Failed for: {input_file:s}") failed = True indented_lines = None @@ -274,7 +268,7 @@ def main(): indented_lines = apply_indentation(styled_lines) if indented_lines is None: - print("Indentation Failed for: {0:s}".format(input_file)) + print(f"Indentation Failed for: {input_file:s}") failed = True amp_lines = None @@ -286,9 +280,7 @@ def main(): ) if len(amp_not_parsed) > 0: - print( - "Ampersand Alignment Failed for: {0:s}".format(input_file) - ) + print(f"Ampersand Alignment Failed for: {input_file:s}") print("failed on lines:\n") for i in amp_not_parsed: print(str(i) + ': "' + indented_lines[i] + '"') @@ -385,7 +377,7 @@ def main(): print("\nProcessing C Files") for input_file in c_files: - print("Processing: {0}".format(input_file)) + print(f"Processing: {input_file}") # Use the clang-format command corresponding to the # installed version on this system args = format_command_map[clang_format_ver].format(input_file) diff --git a/umdp3_fixer/whitespace.py b/umdp3_fixer/whitespace.py index f0f9155b..dc905ef3 100755 --- a/umdp3_fixer/whitespace.py +++ b/umdp3_fixer/whitespace.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -## NOTE ## +# NOTE ## # This module is one of several for which the Master copy is in the UM # repository. When making changes, please ensure the changes are made in the UM # repository or they will be lost during the release process when the UM copy @@ -96,7 +96,7 @@ def keyword_split(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -128,7 +128,7 @@ def keyword_split(line, str_continuation): blank_line = partial_blank_fstring(workline) else: print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) + print(f"{e.msg:s} Line simplification has failed for:") print(line) exit(1) @@ -151,7 +151,7 @@ def apply_whitespace_fixes(lines, striptrailingspace=True, keywordsplit=True): pp_line_previous = "" line_previous = "" - for iline, line in enumerate(lines): + for _iline, line in enumerate(lines): if striptrailingspace: # Strip out trailing spaces ? line = strip_trailing_space(line) @@ -272,9 +272,9 @@ def main(): f'Error opening file:\n "{filename}"\n' "I need a valid filename on which to work...." ) - raise SystemExit(1) + raise SystemExit(1) from None - print("\nLooking at file :\n {0:s}".format(filename)) + print(f"\nLooking at file :\n {filename:s}") # re-open the fortran file, this time to write to it. with open(filename, "r+") as fortran_file: lines_in = fortran_file.read().split("\n") From 73107e83de4ed73b17515a2a97c5a555f541685a Mon Sep 17 00:00:00 2001 From: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:12:58 +0000 Subject: [PATCH 6/6] Fix pytest --- README.md | 6 ++++++ github_scripts/get_git_sources.py | 6 ++++-- github_scripts/tests/test_git_bdiff.py | 15 +++++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f7c9bf8e..76a3c6f6 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,9 @@ ruff check . --config .github/linters/.ruff.toml # Check Format ruff format --check --preview --config .github/linters/.ruff.toml ``` + +## Run Unit Tests + +```sh +pytest -vv +``` diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 8bc5cb89..24777a97 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -186,13 +186,14 @@ def merge_source( if ".git" in str(source): if use_mirrors: remote_path = Path(mirror_loc) / "MetOffice" / repo - fetch = determine_mirror_fetch(source, ref) + fetch = determine_mirror_fetch(str(source), ref) else: remote_path = source fetch = ref else: if not ref: - raise Exception( + raise ValueError( + f"Local source must have a ref. " f"Cannot merge local source '{source}' with empty ref.\n" "Please enter a valid git ref - if you use a branch, then the latest " "commit to that branch will be used." @@ -241,6 +242,7 @@ def handle_merge_conflicts(source: str, ref: str, loc: Path, dependency: str) -> if unmerged: files = "\n".join(f for f in unmerged) raise RuntimeError( + "\nLocal source cannot be merged." "\nA merge conflict has been identified while merging the following branch " f"into the {dependency} source:\n\nsource: {source}\nref: {ref}\n\n" f"with conflicting files:{files}" diff --git a/github_scripts/tests/test_git_bdiff.py b/github_scripts/tests/test_git_bdiff.py index c261cea6..a83b31ff 100644 --- a/github_scripts/tests/test_git_bdiff.py +++ b/github_scripts/tests/test_git_bdiff.py @@ -249,10 +249,14 @@ def find_previous_hash(): Loop over a git log output and extract a hash that isn't the current head """ - result = subprocess.run(["git", "log"], check=True, capture_output=True, text=True) - for line in result.stdout.split("\n"): - if line.startswith("commit") and "HEAD" not in line: - return line.split()[1] + result = subprocess.run( + ["git", "log", "--skip=1", "--format=%H", "-1"], + check=True, + capture_output=True, + text=True, + ) + commit_hash = result.stdout.strip() + return commit_hash if commit_hash else None def test_detached_head(git_repo): @@ -262,6 +266,9 @@ def test_detached_head(git_repo): subprocess.run(["git", "checkout", "main"], check=True) commit_hash = find_previous_hash() + if commit_hash is None: + pytest.skip("No previous commit available for detached head test") + subprocess.run(["git", "checkout", commit_hash], check=True) git_base = GitBase()