Refactor

.github/linters/.ruff.toml

@@ -1,12 +1,16 @@
 cache-dir = "/tmp/.ruff_cache"
 line-length = 88
-output-format = "grouped"
+# output-format = "grouped"
+preview = true
 
 [lint]
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 # undefined-local-with-import-star-usage
-ignore = ["F405"]
+ignore = [
+    "F405", # line too long
+    "I001", # isort: unsorted-imports
+]
 
 [format]
 # Like Black, indent with spaces, rather than tabs.

.gitignore

@@ -1,9 +1,20 @@
-*.sublime-workspace
-*.code-workspace
-.vscode
 __pycache__/
-*.pyc
-*~
-*.swp
 .conda
+.DS_Store
+.env.*
 .idea
+.vscode
+*.bak
+*.code-workspace
+*.egg
+*.egg-info
+*.egg-info/
+*.log
+*.pyc
+*.sublime-workspace
+*.swp
+*.tmp
+*.venv
+*~
+uv.lock
+venv/

LICENSE → LICENCE

@@ -1,4 +1,4 @@
-BSD 3-Clause License
+BSD 3-Clause Licence
 
 Copyright (c) 2023, Met Office
 All rights reserved.

README.md

@@ -10,3 +10,46 @@ owned and maintained by the Simulation Systems and Deployment (SSD) team.
 
 Also contains a copy of `script_updater.sh` which is intended to live in the
 fcm repositories to pull from this repository.
+
+## Install venv with dev tools
+
+- After cloning the repository, run
+
+```sh
+# venv + pip: recommended for Met Office use
+python3.12 -m venv .venv
+.venv/bin/pip install -e ".[dev]"
+
+# Or, via uv (if available)
+uv sync --extra dev --python 3.12
+```
+
+- Activate the virtual environment
+
+```sh
+source .venv/bin/activate
+```
+
+A helper script is also available to do this
+
+```sh
+# via venv + pip
+source ./install-venv.sh
+# via uv
+source ./install-venv.sh [--uv] [--python PYTHON]
+```
+
+## Lint and Format Python
+
+```sh
+# Lint
+ruff check . --config .github/linters/.ruff.toml
+# Check Format
+ruff format --check --preview --config .github/linters/.ruff.toml
+```
+
+## Run Unit Tests
+
+```sh
+pytest -vv
+```

fcm_bdiff/fcm_bdiff.py

@@ -37,10 +37,10 @@ class FCMBase:
     getting information from an instance of the same class.
 
     Note that the version for Git has a small handful of methods, mostly
-    internal and some propeties. These are kept as close as possible to
+    internal and some properties. These are kept as close as possible to
     version in git_bdiff.py.
 
-    Attributes used to navigate the horros of FCM and thus used in this
+    Attributes used to navigate the horrors of FCM and thus used in this
     package are therefore preceded with an '_' and shouldn't be what is
     being referred to outwith this class. Nor should the original
     'functions'...
@@ -85,7 +85,7 @@ def get_branch_name(self):
         """
         Get the branch name from the branch URL.
         Not sure how useful this will be in FCM world.
-        For now, define it to be the contants of the URL after .*/main/ has been
+        For now, define it to be the contents of the URL after .*/main/ has been
         stripped off. i.e. it will start with trunk/... or branches/...
         """
 

gh_review_project/workload.py

@@ -218,12 +218,12 @@ def main(total: bool, test: bool, capture_project: bool, file: Path):
     # Create tables for each combination of reviewers and reposotories
     tables = {}
 
-    ## Table for non-LFRic repositories
+    # Table for non-LFRic repositories
     repo_list = other_repo_list(data, lfric_repositories)
     reviewers = teams["SSD"].get_team_members()
     tables["SSD"] = build_table(data, reviewers, repo_list)
 
-    ## Table for LFRic repositories
+    # Table for LFRic repositories
     repo_list = lfric_repositories
     reviewers = []
     for team in teams.values():

github_scripts/get_git_sources.py

@@ -10,7 +10,6 @@
 import re
 import subprocess
 from datetime import datetime
-from typing import Optional, Union
 from pathlib import Path
 from shutil import rmtree
 import shlex
@@ -21,7 +20,7 @@
 
 def run_command(
     command: str, check: bool = True, capture: bool = True, timeout: int = 600
-) -> Optional[subprocess.CompletedProcess]:
+) -> subprocess.CompletedProcess | None:
     """
     Run a subprocess command and return the result object
     Inputs:
@@ -92,7 +91,7 @@ def datetime_str() -> str:
 
 def clone_and_merge(
     dependency: str,
-    opts: Union[list, dict],
+    opts: list | dict,
     loc: Path,
     use_mirrors: bool,
     mirror_loc: Path,
@@ -167,7 +166,7 @@ def get_source(
 
 
 def merge_source(
-    source: Union[Path, str],
+    source: Path | str,
     ref: str,
     dest: Path,
     repo: str,
@@ -187,13 +186,14 @@ def merge_source(
     if ".git" in str(source):
         if use_mirrors:
             remote_path = Path(mirror_loc) / "MetOffice" / repo
-            fetch = determine_mirror_fetch(source, ref)
+            fetch = determine_mirror_fetch(str(source), ref)
         else:
             remote_path = source
             fetch = ref
     else:
         if not ref:
-            raise Exception(
+            raise ValueError(
+                f"Local source must have a ref. "
                 f"Cannot merge local source '{source}' with empty ref.\n"
                 "Please enter a valid git ref - if you use a branch, then the latest "
                 "commit to that branch will be used."
@@ -242,6 +242,7 @@ def handle_merge_conflicts(source: str, ref: str, loc: Path, dependency: str) ->
     if unmerged:
         files = "\n".join(f for f in unmerged)
         raise RuntimeError(
+            "\nLocal source cannot be merged."
             "\nA merge conflict has been identified while merging the following branch "
             f"into the {dependency} source:\n\nsource: {source}\nref: {ref}\n\n"
             f"with conflicting files:{files}"
@@ -364,7 +365,7 @@ def clone_repo(repo_source: str, repo_ref: str, loc: Path) -> None:
             run_command(command)
 
 
-def sync_repo(repo_source: Union[str, Path], repo_ref: str, loc: Path) -> None:
+def sync_repo(repo_source: str | Path, repo_ref: str, loc: Path) -> None:
     """
     Rsync a local git clone and checkout the provided ref
     """

github_scripts/git_bdiff.py

@@ -168,9 +168,12 @@ def has_diverged(self):
     def files(self):
         """Iterate over files changed on the branch."""
 
-        for line in self.run_git(
-            ["diff", "--name-only", "--diff-filter=AMX", self.ancestor]
-        ):
+        for line in self.run_git([
+            "diff",
+            "--name-only",
+            "--diff-filter=AMX",
+            self.ancestor,
+        ]):
             if line != "":
                 yield line
 

github_scripts/suite_data.py

@@ -16,7 +16,6 @@
 import yaml
 from collections import defaultdict
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Union
 from git_bdiff import GitBDiff, GitInfo
 from get_git_sources import clone_repo, sync_repo
 
@@ -43,7 +42,7 @@ def __init__(self) -> None:
         self.task_states = {}
         self.temp_directory = None
 
-    def get_um_failed_configs(self) -> Set[str]:
+    def get_um_failed_configs(self) -> set[str]:
         """
         Read through failed UM rose_ana tasks
         """
@@ -76,7 +75,7 @@ def read_um_section(self, change: str) -> str:
             section = "Unknown"
         return section
 
-    def get_changed_um_section(self) -> Set[str]:
+    def get_changed_um_section(self) -> set[str]:
         """
         Read through bdiff of UM source and find code owner section for each changed
         file
@@ -113,7 +112,7 @@ def get_changed_um_section(self) -> Set[str]:
 
         return changed_sections
 
-    def get_um_owners(self, filename: str) -> Dict:
+    def get_um_owners(self, filename: str) -> dict:
         """
         Read UM Code Owners file and write to a dictionary
         """
@@ -151,7 +150,7 @@ def get_um_owners(self, filename: str) -> Dict:
 
         return owners
 
-    def parse_tasks(self) -> Dict[str, List[str]]:
+    def parse_tasks(self) -> dict[str, list[str]]:
         """
         Read through the tasks run, sorting by state
         """
@@ -233,7 +232,7 @@ def determine_primary_source(self) -> str:
 
         return "unknown"
 
-    def read_rose_conf(self) -> Dict[str, str]:
+    def read_rose_conf(self) -> dict[str, str]:
         """
         Read the suite rose-suite.conf file into a dictionary
         """
@@ -281,19 +280,19 @@ def find_unknown_dependency(self, dependency: str) -> str:
 
         pattern = re.compile(rf"{dependency.upper()} SOURCE CLONE=(\S+)")
         log_file = self.suite_path / "log" / "scheduler" / "log"
-        with open(log_file, "r") as f:
+        with open(log_file) as f:
             for line in f:
                 if match := pattern.search(line):
                     return match.group(1).rstrip("/")
         raise RuntimeError(f"Unable to find source for dependency {dependency}")
 
-    def read_dependencies(self) -> Dict[str, Dict]:
+    def read_dependencies(self) -> dict[str, dict]:
         """
         Read the suite dependencies from the dependencies.yaml file - this is assumed to
         have been copied to the suite_path directory
         """
 
-        with open(self.suite_path / "dependencies.yaml", "r") as stream:
+        with open(self.suite_path / "dependencies.yaml") as stream:
             dependencies = yaml.safe_load(stream)
         for dependency, data in dependencies.items():
             if data["source"] is None:
@@ -307,7 +306,7 @@ def get_workflow_id(self) -> str:
         Read cylc install log for workflow id
         """
 
-        with open(self.suite_path / "log" / "scheduler" / "log", "r") as f:
+        with open(self.suite_path / "log" / "scheduler" / "log") as f:
             for line in f:
                 match = re.search(r"INFO - Workflow: (\S+\/\w+)", line)
                 try:
@@ -344,7 +343,7 @@ def get_suite_starttime(self) -> str:
             starttime = row[0]
         return starttime.split("+")[0]
 
-    def read_groups_run(self) -> List[str]:
+    def read_groups_run(self) -> list[str]:
         """
         Read in groups run as part of suite from the cylc database file
         """
@@ -360,7 +359,7 @@ def read_groups_run(self) -> List[str]:
             groups = ["suite_default"]
         return groups
 
-    def get_task_states(self) -> Dict[str, str]:
+    def get_task_states(self) -> dict[str, str]:
         """
         Query the database and return a dictionary of states. This is assumed to be in
         suite_path/log/db
@@ -374,8 +373,8 @@ def get_task_states(self) -> Dict[str, str]:
         return data
 
     def query_suite_database(
-        self, database: Path, selections: List[str], source: str
-    ) -> List[tuple]:
+        self, database: Path, selections: list[str], source: str
+    ) -> list[tuple]:
         """
         Create an sql statement and query provided database. Return the result
         """
@@ -392,8 +391,8 @@ def query_suite_database(
         return data
 
     def run_command(
-        self, command: Union[str, List[str]], shell: bool = False, rval: bool = False
-    ) -> Optional[subprocess.CompletedProcess]:
+        self, command: str | list[str], shell: bool = False, rval: bool = False
+    ) -> subprocess.CompletedProcess | None:
         """
         Run a subprocess command and return the result object
         Inputs: