diff --git a/src/cfengine_cli/masterfiles/analyze.py b/src/cfengine_cli/masterfiles/analyze.py index 6eff6e4..1846697 100644 --- a/src/cfengine_cli/masterfiles/analyze.py +++ b/src/cfengine_cli/masterfiles/analyze.py @@ -93,31 +93,25 @@ def finalize_vcf(versions_dict, checksums_dict, files_dict): return versions_dict, checksums_dict, files_dict -def filter_unstable_releases(data): +def get_stable_releases(data): # Filter the data to only include stable releases (not debug, alpha, or beta releases): - filtered_data = [] - - for release_data in data.get("releases", []): - if release_data.get("debug") is True: - continue - if release_data.get("alpha") is True: - continue - if release_data.get("beta") is True: - continue - - filtered_data.append(release_data) - - return filtered_data + return [ + r + for r in data.get("releases", []) + if not (r.get("debug") or r.get("alpha") or r.get("beta")) + ] def sort_release_data(file_checksums_dict): # Newest versions first, and files sorted alphabetically within each version - for v in file_checksums_dict.keys(): - file_checksums_dict[v] = dict_sorted_by_key(file_checksums_dict[v]) + # Work on copy to avoid mutating original dict + file_checksums_dict_copy = file_checksums_dict + for v in file_checksums_dict_copy.keys(): + file_checksums_dict_copy[v] = dict_sorted_by_key(file_checksums_dict_copy[v]) sorted_dict = OrderedDict( sorted( - file_checksums_dict.items(), + file_checksums_dict_copy.items(), key=lambda p: version_as_comparable_list(p[0]), reverse=True, ) diff --git a/src/cfengine_cli/masterfiles/download.py b/src/cfengine_cli/masterfiles/download.py index 388f2e0..f651d2a 100644 --- a/src/cfengine_cli/masterfiles/download.py +++ b/src/cfengine_cli/masterfiles/download.py @@ -11,7 +11,7 @@ ) ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json" - +COMMUNITY_RELEASES_URL = "https://cfengine.com/release-data/community/releases.json" COMMUNITY_ONLY_VERSIONS = ["3.12.0b1", "3.10.0b1"] """Masterfiles versions which do not appear in Enterprise releases but appear in Community releases.""" diff --git a/src/cfengine_cli/masterfiles/generate_release_information.py b/src/cfengine_cli/masterfiles/generate_release_information.py index 6d99395..448b6bd 100644 --- a/src/cfengine_cli/masterfiles/generate_release_information.py +++ b/src/cfengine_cli/masterfiles/generate_release_information.py @@ -1,9 +1,11 @@ +import json from cfengine_cli.masterfiles.download import ( ENTERPRISE_RELEASES_URL, + COMMUNITY_RELEASES_URL, download_all_versions, ) from cfengine_cli.masterfiles.generate_vcf_download import generate_vcf_download -from cfengine_cli.masterfiles.analyze import filter_unstable_releases, sort_release_data +from cfengine_cli.masterfiles.analyze import get_stable_releases, sort_release_data from cfengine_cli.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout from cfengine_cli.masterfiles.check_download_matches_git import ( check_download_matches_git, @@ -72,24 +74,23 @@ def generate_release_information_impl( ) -def generate_release_history(): - print("Generating release history information...") - - releases_data = download_releasedata() - - stable_releases = filter_unstable_releases(releases_data) - - file_checksums_dict = build_release_history(stable_releases) +def download_enterprise_releasedata(): + # Downloading releases.json: + try: + releases_data = get_json(ENTERPRISE_RELEASES_URL) - sorted_releases = sort_release_data(file_checksums_dict) + except CFBSNetworkError: + raise CFBSExitError( + "Downloading CFEngine release data failed - check your Wi-Fi / network settings." + ) - write_json("./cfengine-enterprise/checksums.json", sorted_releases) + return releases_data -def download_releasedata(): - # Downloading releases.json: +def download_community_releasedata(): + # Downloading community/releases.json try: - releases_data = get_json(ENTERPRISE_RELEASES_URL) + releases_data = get_json(COMMUNITY_RELEASES_URL) except CFBSNetworkError: raise CFBSExitError( @@ -99,6 +100,29 @@ def download_releasedata(): return releases_data +def process_release_type(folder, download_func): + # Function for processing either community or enterprise releases + release_data = download_func() + + write_json_pretty(f"./{folder}/releases.json", release_data) + + stable_releases = get_stable_releases(release_data) + + file_checksums_dict = build_release_history(stable_releases) + + write_version_files(stable_releases, folder) + + sorted_releases = sort_release_data(file_checksums_dict) + + write_json(f"./{folder}/checksums.json", sorted_releases) + + +def generate_release_history(): + print("Generating release history information...") + process_release_type("cfengine-enterprise", download_enterprise_releasedata) + process_release_type("cfengine-community", download_community_releasedata) + + def build_release_history(filtered_releases): release_history = {} @@ -152,3 +176,19 @@ def extract_file_info(asset_data): return filename, checksum return None, None + + +def write_version_files(stable_releases, folder): + # Writes versionfiles for each version + for release_data in stable_releases: + version = release_data.get("version") + if not version: + continue + version_data, _ = download_release_version_data(release_data) + write_json(f"./{folder}/versions/{version}.json", version_data) + + +def write_json_pretty(path, data): + # Writes release information in same format as on cfengine.com + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False)