From 95cf5e611562cfc3feaba583ccce79707c8ace58 Mon Sep 17 00:00:00 2001 From: Tedsig42 Date: Sun, 15 Mar 2026 00:31:53 +0000 Subject: [PATCH 01/19] Add CloudVulnDB importer - add CloudVulnDB v2 importer pipeline - register importer - add tests and fixtures - ignore setup.py in pytest collection Signed-off-by: Tedsig42 --- pyproject.toml | 5 + vulnerabilities/importers/__init__.py | 2 + .../v2_importers/cloudvulndb_importer.py | 171 ++++++++++++++++++ .../tests/test_cloudvulndb_importer.py | 64 +++++++ .../cloudvulndb/cloudvulndb_rss_mock.xml | 22 +++ ...expected_cloudvulndb_advisory_output1.json | 21 +++ 6 files changed, 285 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py create mode 100644 vulnerabilities/tests/test_cloudvulndb_importer.py create mode 100644 vulnerabilities/tests/test_data/cloudvulndb/cloudvulndb_rss_mock.xml create mode 100644 vulnerabilities/tests/test_data/cloudvulndb/expected_cloudvulndb_advisory_output1.json diff --git a/pyproject.toml b/pyproject.toml index 6b1d8c0d5..88e1ee5c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,11 @@ addopts = [ "-rfExXw", "--strict-markers", "--doctest-modules", + # setup.py imports setuptools which is not available in the Docker runtime + # image. Without this, pytest (which uses python_files = "*.py") tries to + # collect setup.py as a test module and crashes with exit code 2. + "--ignore=setup.py", + "--ignore-glob=*/setup.py", # Ignore the following doctests until these files are migrated to # import-improve structure "--ignore=vulnerabilities/importers/apache_httpd.py", diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 594021092..6b42b4409 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -47,6 +47,7 @@ from vulnerabilities.pipelines.v2_importers import apache_kafka_importer as apache_kafka_importer_v2 from vulnerabilities.pipelines.v2_importers import apache_tomcat_importer as apache_tomcat_v2 from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2 +from vulnerabilities.pipelines.v2_importers import cloudvulndb_importer as cloudvulndb_importer_v2 from vulnerabilities.pipelines.v2_importers import collect_fix_commits as collect_fix_commits_v2 from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2 from vulnerabilities.pipelines.v2_importers import debian_importer as debian_importer_v2 @@ -109,6 +110,7 @@ project_kb_msr2019_importer_v2.ProjectKBMSR2019Pipeline, ruby_importer_v2.RubyImporterPipeline, epss_importer_v2.EPSSImporterPipeline, + cloudvulndb_importer_v2.CloudVulnDBImporterPipeline, gentoo_importer_v2.GentooImporterPipeline, nginx_importer_v2.NginxImporterPipeline, debian_importer_v2.DebianImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py b/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py new file mode 100644 index 000000000..6b87f7baf --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py @@ -0,0 +1,171 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib +import json +import logging +from typing import Iterable +from urllib.parse import urlparse +from xml.etree import ElementTree + +from dateutil import parser as dateutil_parser + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.utils import fetch_response +from vulnerabilities.utils import find_all_cve + +logger = logging.getLogger(__name__) + +CLOUDVULNDB_RSS_URL = "https://www.cloudvulndb.org/rss/feed.xml" + + +class CloudVulnDBImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + """Collect cloud vulnerabilities from the public CloudVulnDB RSS feed.""" + + pipeline_id = "cloudvulndb_importer" + spdx_license_expression = "CC-BY-4.0" + license_url = "https://github.com/wiz-sec/open-cvdb/blob/main/LICENSE.md" + repo_url = "https://github.com/wiz-sec/open-cvdb" + precedence = 200 + + _cached_items = None + + @classmethod + def steps(cls): + return (cls.collect_and_store_advisories,) + + def get_feed_items(self): + if self._cached_items is None: + response = fetch_response(CLOUDVULNDB_RSS_URL) + self._cached_items = parse_rss_feed(response.text) + return self._cached_items + + def advisories_count(self) -> int: + return len(self.get_feed_items()) + + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: + for item in self.get_feed_items(): + advisory = parse_advisory_data(item) + if advisory: + yield advisory + + +def parse_rss_feed(xml_text: str) -> list: + """ + Parse CloudVulnDB RSS XML and return a list of item dictionaries. + Each dictionary has ``title``, ``link``, ``description``, ``pub_date`` and ``guid`` keys. + """ + try: + root = ElementTree.fromstring(xml_text) + except ElementTree.ParseError as e: + logger.error("Failed to parse CloudVulnDB RSS XML: %s", e) + return [] + + channel = root.find("channel") + if channel is None: + logger.error("CloudVulnDB RSS feed has no element") + return [] + + items = [] + for item_el in channel.findall("item"): + items.append( + { + "title": (item_el.findtext("title") or "").strip(), + "link": (item_el.findtext("link") or "").strip(), + "description": (item_el.findtext("description") or "").strip(), + "pub_date": (item_el.findtext("pubDate") or "").strip(), + "guid": (item_el.findtext("guid") or "").strip(), + } + ) + + return items + + +def parse_advisory_data(item: dict): + """ + Parse one CloudVulnDB item and return an AdvisoryDataV2 object. + Since the RSS feed does not provide package/version coordinates, ``affected_packages`` is empty. + """ + title = item.get("title") or "" + link = item.get("link") or "" + description = item.get("description") or "" + pub_date = item.get("pub_date") or "" + guid = item.get("guid") or "" + + advisory_id = get_advisory_id(guid=guid, link=link, title=title, pub_date=pub_date) + if not advisory_id: + logger.error("Skipping advisory with no usable identifier: %r", item) + return None + + aliases = list(dict.fromkeys(find_all_cve(f"{title}\n{description}"))) + aliases = [alias for alias in aliases if alias != advisory_id] + + date_published = None + if pub_date: + try: + date_published = dateutil_parser.parse(pub_date) + except Exception as e: + logger.warning("Could not parse date %r for advisory %s: %s", pub_date, advisory_id, e) + + references = [] + if link: + references.append(ReferenceV2(url=link)) + + summary = title or description + + return AdvisoryDataV2( + advisory_id=advisory_id, + aliases=aliases, + summary=summary, + affected_packages=[], + references=references, + date_published=date_published, + url=link or CLOUDVULNDB_RSS_URL, + original_advisory_text=json.dumps(item, indent=2, ensure_ascii=False), + ) + + +def get_advisory_id(guid: str, link: str, title: str, pub_date: str) -> str: + """ + Return a stable advisory identifier using the best available source. + Preference order is GUID, link slug, then deterministic content hash fallback. + """ + guid = (guid or "").strip() + if guid: + return guid + + slug = advisory_slug_from_link(link) + if slug: + return slug + + fingerprint_source = "|".join([title.strip(), pub_date.strip()]) + if not fingerprint_source.strip("|"): + return "" + + digest = hashlib.sha256(fingerprint_source.encode("utf-8")).hexdigest()[:16] + return f"cloudvulndb-{digest}" + + +def advisory_slug_from_link(link: str) -> str: + """Extract an advisory slug from a CloudVulnDB URL path.""" + if not link: + return "" + + try: + parsed = urlparse(link) + except Exception: + return "" + + parts = [part for part in parsed.path.split("/") if part] + if not parts: + return "" + + return parts[-1].strip() diff --git a/vulnerabilities/tests/test_cloudvulndb_importer.py b/vulnerabilities/tests/test_cloudvulndb_importer.py new file mode 100644 index 000000000..9f717fe51 --- /dev/null +++ b/vulnerabilities/tests/test_cloudvulndb_importer.py @@ -0,0 +1,64 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from unittest import TestCase + +from vulnerabilities.pipelines.v2_importers.cloudvulndb_importer import advisory_slug_from_link +from vulnerabilities.pipelines.v2_importers.cloudvulndb_importer import get_advisory_id +from vulnerabilities.pipelines.v2_importers.cloudvulndb_importer import parse_advisory_data +from vulnerabilities.pipelines.v2_importers.cloudvulndb_importer import parse_rss_feed +from vulnerabilities.tests import util_tests + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/cloudvulndb") + + +def _load_rss(filename="cloudvulndb_rss_mock.xml"): + with open(os.path.join(TEST_DATA, filename), encoding="utf-8") as f: + return f.read() + + +class TestCloudVulnDBImporter(TestCase): + def test_parse_rss_feed_returns_correct_item_count(self): + items = parse_rss_feed(_load_rss()) + self.assertEqual(len(items), 2) + + def test_parse_advisory_with_guid_and_cves(self): + items = parse_rss_feed(_load_rss()) + result = parse_advisory_data(items[0]) + self.assertIsNotNone(result) + result_dict = result.to_dict() + expected_file = os.path.join(TEST_DATA, "expected_cloudvulndb_advisory_output1.json") + util_tests.check_results_against_json(result_dict, expected_file) + + def test_parse_advisory_without_guid_falls_back_to_link_slug(self): + items = parse_rss_feed(_load_rss()) + result = parse_advisory_data(items[1]) + self.assertIsNotNone(result) + self.assertEqual(result.advisory_id, "azure-imds-ssrf") + self.assertEqual(result.aliases, []) + + def test_get_advisory_id_hash_fallback(self): + advisory_id = get_advisory_id( + guid="", + link="", + title="Example advisory title", + pub_date="Mon, 08 Jul 2024 00:00:00 GMT", + ) + self.assertTrue(advisory_id.startswith("cloudvulndb-")) + self.assertEqual(len(advisory_id), len("cloudvulndb-") + 16) + + def test_parse_rss_feed_invalid_xml_returns_empty(self): + result = parse_rss_feed("not valid xml <>>>") + self.assertEqual(result, []) + + def test_advisory_slug_from_link(self): + slug = advisory_slug_from_link("https://www.cloudvulndb.org/vulnerabilities/aws-example/") + self.assertEqual(slug, "aws-example") diff --git a/vulnerabilities/tests/test_data/cloudvulndb/cloudvulndb_rss_mock.xml b/vulnerabilities/tests/test_data/cloudvulndb/cloudvulndb_rss_mock.xml new file mode 100644 index 000000000..1d2421e57 --- /dev/null +++ b/vulnerabilities/tests/test_data/cloudvulndb/cloudvulndb_rss_mock.xml @@ -0,0 +1,22 @@ + + + + CloudVulnDB RSS + https://www.cloudvulndb.org + Cloud vulnerabilities and security issues + + <![CDATA[AWS Example Privilege Escalation (CVE-2024-11111)]]> + https://www.cloudvulndb.org/vulnerabilities/aws-example-privilege-escalation + CLOUD-2024-0001 + Tue, 04 Jun 2024 12:30:00 GMT + + + + <![CDATA[Azure IMDS SSRF Exposure]]> + https://www.cloudvulndb.org/vulnerabilities/azure-imds-ssrf + + Fri, 05 Jul 2024 08:00:00 GMT + + + + diff --git a/vulnerabilities/tests/test_data/cloudvulndb/expected_cloudvulndb_advisory_output1.json b/vulnerabilities/tests/test_data/cloudvulndb/expected_cloudvulndb_advisory_output1.json new file mode 100644 index 000000000..8baf2b463 --- /dev/null +++ b/vulnerabilities/tests/test_data/cloudvulndb/expected_cloudvulndb_advisory_output1.json @@ -0,0 +1,21 @@ +{ + "advisory_id": "CLOUD-2024-0001", + "aliases": [ + "CVE-2024-11111", + "CVE-2024-22222" + ], + "summary": "AWS Example Privilege Escalation (CVE-2024-11111)", + "affected_packages": [], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://www.cloudvulndb.org/vulnerabilities/aws-example-privilege-escalation" + } + ], + "patches": [], + "severities": [], + "date_published": "2024-06-04T12:30:00+00:00", + "weaknesses": [], + "url": "https://www.cloudvulndb.org/vulnerabilities/aws-example-privilege-escalation" +} From 7faf3cf242a504ba85df17b0d4a81eb8876a1d46 Mon Sep 17 00:00:00 2001 From: Tedsig42 Date: Sun, 15 Mar 2026 00:39:19 +0000 Subject: [PATCH 02/19] Potential fix for pull request finding Signed-off-by: Tedsig42 --- vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py b/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py index 6b87f7baf..9379f1126 100644 --- a/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py +++ b/vulnerabilities/pipelines/v2_importers/cloudvulndb_importer.py @@ -30,7 +30,7 @@ class CloudVulnDBImporterPipeline(VulnerableCodeBaseImporterPipelineV2): """Collect cloud vulnerabilities from the public CloudVulnDB RSS feed.""" - pipeline_id = "cloudvulndb_importer" + pipeline_id = "cloudvulndb_importer_v2" spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/wiz-sec/open-cvdb/blob/main/LICENSE.md" repo_url = "https://github.com/wiz-sec/open-cvdb" From b4ab45743f811eb0173fb1273341402b94efad3e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 4 Mar 2026 19:40:01 +0530 Subject: [PATCH 03/19] Review all V2 pipelines Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- .../v2_importers/alpine_linux_importer.py | 2 ++ .../v2_importers/apache_tomcat_importer.py | 1 + .../v2_importers/archlinux_importer.py | 20 ++++++++++++++++++- .../pipelines/v2_importers/debian_importer.py | 2 ++ .../v2_importers/epss_importer_v2.py | 1 + .../pipelines/v2_importers/istio_importer.py | 1 + .../v2_importers/mattermost_importer.py | 3 +++ .../pipelines/v2_importers/nginx_importer.py | 5 +++-- .../project_kb_msr2019_importer.py | 1 + .../project_kb_statements_importer.py | 1 + .../v2_importers/retiredotnet_importer.py | 1 + .../pipelines/v2_importers/ruby_importer.py | 3 +++ .../v2_importers/suse_score_importer.py | 4 ++++ vulnerabilities/severity_systems.py | 8 ++++++++ .../archlinux_advisoryv2-expected.json | 8 +++++++- 15 files changed, 57 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py index 642a37435..336afa298 100644 --- a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import logging from pathlib import Path from typing import Any @@ -244,4 +245,5 @@ def load_advisories( references=references, affected_packages=affected_packages, url=url, + original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py b/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py index f55665c56..f7dd12f57 100644 --- a/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py +++ b/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py @@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: summary=advisory_list[0].summary, affected_packages=affected_packages, url=page_url, + original_advisory_text=str(content), ) except Exception as e: diff --git a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py index 4f9096f09..b0f005592 100644 --- a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py @@ -14,10 +14,13 @@ from packageurl import PackageURL from univers.version_range import ArchLinuxVersionRange +from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import fetch_response @@ -53,7 +56,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: def parse_advisory(self, record) -> AdvisoryDataV2: affected_packages = [] references = [] + severities = [] avg_name = record.get("name") + severity = record.get("severity") aliases = record.get("issues", []) aliases.extend(record.get("advisories", [])) summary = record.get("type", "") @@ -92,13 +97,26 @@ def parse_advisory(self, record) -> AdvisoryDataV2: ) ) + if severity not in severity_systems.ARCHLINUX.choices: + self.log(f"Unknown severity {severity} for {avg_name}") + severity = None + if severity: + severities = [ + VulnerabilitySeverity( + system=severity_systems.ARCHLINUX, + value=severity, + url="https://security.archlinux.org/{avg_name}.json", + ) + ] + return AdvisoryDataV2( advisory_id=avg_name, aliases=aliases, summary=summary, references=references, affected_packages=affected_packages, + severities=severities, weaknesses=[], url=f"https://security.archlinux.org/{avg_name}.json", - original_advisory_text=json.dumps(record), + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/debian_importer.py b/vulnerabilities/pipelines/v2_importers/debian_importer.py index bdc0770ca..67cba2b11 100644 --- a/vulnerabilities/pipelines/v2_importers/debian_importer.py +++ b/vulnerabilities/pipelines/v2_importers/debian_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import re from typing import Any from typing import Iterable @@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD references=references, weaknesses=weaknesses, url=f"https://security-tracker.debian.org/tracker/{record_identifier}", + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py index e1d6f6eaf..6f8adc6d1 100644 --- a/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py +++ b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py @@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: severities=[severity], references=[references], url=self.advisory_url, + original_advisory_text=",".join(epss_row), ) diff --git a/vulnerabilities/pipelines/v2_importers/istio_importer.py b/vulnerabilities/pipelines/v2_importers/istio_importer.py index a9a3de881..ce7b7d342 100644 --- a/vulnerabilities/pipelines/v2_importers/istio_importer.py +++ b/vulnerabilities/pipelines/v2_importers/istio_importer.py @@ -42,6 +42,7 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "Apache-2.0" license_url = "https://github.com/istio/istio.io/blob/master/LICENSE" repo_url = "git+https://github.com/istio/istio.io" + run_once = True precedence = 200 diff --git a/vulnerabilities/pipelines/v2_importers/mattermost_importer.py b/vulnerabilities/pipelines/v2_importers/mattermost_importer.py index 538bb1d4d..852939fd8 100644 --- a/vulnerabilities/pipelines/v2_importers/mattermost_importer.py +++ b/vulnerabilities/pipelines/v2_importers/mattermost_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json from typing import Iterable from packageurl import PackageURL @@ -122,5 +123,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: summary=details, references=[reference], affected_packages=affected_packages, + severities=severities, url=self.url, + original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/nginx_importer.py b/vulnerabilities/pipelines/v2_importers/nginx_importer.py index 81448166b..f5f3e3c67 100644 --- a/vulnerabilities/pipelines/v2_importers/nginx_importer.py +++ b/vulnerabilities/pipelines/v2_importers/nginx_importer.py @@ -62,7 +62,7 @@ def collect_advisories(self): vulnerability_list = soup.select("li p") for vulnerability_info in vulnerability_list: ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info) - yield to_advisory_data(ngnix_advisory) + yield to_advisory_data(ngnix_advisory, vulnerability_info) class NginxAdvisory(NamedTuple): @@ -79,7 +79,7 @@ def to_dict(self): return self._asdict() -def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2: +def to_advisory_data(nginx_adv: NginxAdvisory, vulnerability_info) -> AdvisoryDataV2: """ Return AdvisoryDataV2 from an NginxAdvisory tuple. """ @@ -150,6 +150,7 @@ def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2: references=nginx_adv.references, patches=nginx_adv.patches, url="https://nginx.org/en/security_advisories.html", + original_advisory_text=str(vulnerability_info), ) diff --git a/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py b/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py index 7dd01233e..1a4411acd 100644 --- a/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py +++ b/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py @@ -90,6 +90,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: patches=patches, references=references, url="https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv", + original_advisory_text=",".join(row), ) def clean_downloads(self): diff --git a/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py b/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py index b67540bce..0c1c8e05d 100644 --- a/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py +++ b/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py @@ -168,6 +168,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: references=references, patches=patches, url=advisory_url, + original_advisory_text=saneyaml.dump(yaml_data, indent=2), ) def clean_downloads(self): diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 478d31323..cb87183e3 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -118,6 +118,7 @@ def collect_advisories(self): affected_packages=affected_packages, references=vuln_reference, url=advisory_url, + original_advisory_text=json.dumps(json_doc, indent=2, ensure_ascii=False), ) @staticmethod diff --git a/vulnerabilities/pipelines/v2_importers/ruby_importer.py b/vulnerabilities/pipelines/v2_importers/ruby_importer.py index db02f4823..455a0a491 100644 --- a/vulnerabilities/pipelines/v2_importers/ruby_importer.py +++ b/vulnerabilities/pipelines/v2_importers/ruby_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import logging from pathlib import Path from typing import Iterable @@ -129,6 +130,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url): severities=get_severities(record), date_published=get_publish_time(record), url=advisory_url, + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) elif schema_type == "rubies": @@ -147,6 +149,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url): references=get_references(record), date_published=get_publish_time(record), url=advisory_url, + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/suse_score_importer.py b/vulnerabilities/pipelines/v2_importers/suse_score_importer.py index b39abcd11..92a534ddc 100644 --- a/vulnerabilities/pipelines/v2_importers/suse_score_importer.py +++ b/vulnerabilities/pipelines/v2_importers/suse_score_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json from typing import Iterable from vulnerabilities import severity_systems @@ -67,4 +68,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: severities=severities, references=[], url=self.url, + original_advisory_text=json.dumps( + self.score_data[cve_id], indent=2, ensure_ascii=False + ), ) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index 27f9d7d1a..6624a656e 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -169,6 +169,14 @@ def get(self, scoring_elements: str) -> dict: "Low", ] +ARCHLINUX.choices = [ + "Critical", + "High", + "Medium", + "Low", + "Very Low", +] + # This is essentially identical to apache_http except for the addition of the "High" score, # which seems to be used interchangeably for "Important". APACHE_TOMCAT = ScoringSystem( diff --git a/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json b/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json index 4902884f5..06cee73f3 100644 --- a/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json +++ b/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json @@ -107,7 +107,13 @@ } ], "patches": [], - "severities": [], + "severities": [ + { + "system": "archlinux", + "value": "Low", + "scoring_elements": "" + } + ], "date_published": null, "weaknesses": [], "url": "https://security.archlinux.org/AVG-4.json" From 90fe20661117bcfbd9b218db1d994962b8ba72aa Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 4 Mar 2026 19:40:31 +0530 Subject: [PATCH 04/19] Add V2 in navbar Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- vulnerabilities/templates/navbar.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index de6dc1a94..3d3fa0e91 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -26,6 +26,9 @@ Vulnerabilities + + V2 + Documentation From 66d2525795103455ca311fb9aa3cba9c7cd0d7e4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 4 Mar 2026 19:40:47 +0530 Subject: [PATCH 05/19] Add V3 info on landing page Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- vulnerabilities/templates/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vulnerabilities/templates/index.html b/vulnerabilities/templates/index.html index 78effa82d..77ad8d1c0 100644 --- a/vulnerabilities/templates/index.html +++ b/vulnerabilities/templates/index.html @@ -33,6 +33,9 @@

+
+ ATTENTION: We will be deprecating V1 and V2 API by 30th June 2026. V3 endpoint is live now. Please migrate to V3 API before the deprecation date. For more details, please refer to this blog. +
{% endblock %} \ No newline at end of file From cae95fede32833050db88eb4cb31639136db9698 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 4 Mar 2026 19:41:01 +0530 Subject: [PATCH 06/19] Prep for release Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- CHANGELOG.rst | 8 ++++- PIPELINES-AVID.rst | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 PIPELINES-AVID.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ba0d0b198..294004e08 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,10 +13,16 @@ Version v37.0.0 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2. - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories. - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories. -- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories. +- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories. - Pipeline Dashboard improvements #1920. - Throttle API requests based on user permissions #1909. - Add pipeline to compute Advisory ToDos #1764 +- Use related advisory severity to calculate exploitibility, weighted severity and risk scores +- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881. +- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099. +- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110. +- Plan storing of exploits and EPSS based advisories #2069. + Version v36.1.3 --------------------- diff --git a/PIPELINES-AVID.rst b/PIPELINES-AVID.rst new file mode 100644 index 000000000..43de21e19 --- /dev/null +++ b/PIPELINES-AVID.rst @@ -0,0 +1,74 @@ +.. list-table:: Pipeline AVID Mapping + :header-rows: 1 + :widths: 35 65 + + * - pipeline name + - AVID + * - alpine_linux_importer_v2 + - {package_name}/{distroversion}/{version}/{vulnerability_id} + * - aosp_dataset_fix_commits + - CVE ID of the record + * - apache_httpd_importer_v2 + - CVE ID of the record + * - apache_kafka_importer_v2 + - CVE ID of the record + * - apache_tomcat_importer_v2 + - {page_id}/{cve_id} + * - archlinux_importer_v2 + - AVG ID of the record + * - curl_importer_v2 + - CURL-CVE ID of the record + * - debian_importer_v2 + - {package_name}/{debian_record_id} + * - elixir_security_importer_v2 + - {package_name}/{file_id} + * - epss_importer_v2 + - CVE ID of the record + * - fireeye_importer_v2 + - {file_id} + * - gentoo_importer_v2 + - GLSA ID of the record + * - github_osv_importer_v2 + - ID of the OSV record + * - gitlab_importer_v2 + - Identifier of the GitLab community advisory record + * - istio_importer_v2 + - ISTIO-SECURITY- + * - mattermost_importer_v2 + - MMSA- + * - mozilla_importer_v2 + - MFSA- + * - nginx_importer_v2 + - First alias of the record + * - nodejs_security_wg + - NPM- + * - nvd_importer_v2 + - CVE ID of the record + * - openssl_importer_v2 + - CVE ID of the record + * - oss_fuzz_importer_v2 + - ID of the OSV record + * - postgresql_importer_v2 + - CVE ID of the record + * - project-kb-msr-2019_v2 + - Vulnerability ID of the record + * - project-kb-statements_v2 + - Vulnerability ID of the record + * - pypa_importer_v2 + - ID of the OSV record + * - pysec_importer_v2 + - ID of the OSV record + * - redhat_importer_v2 + - RHSA ID of the record + * - retiredotnet_importer_v2 + - retiredotnet-{file_id} + * - ruby_importer_v2 + - {file_id} + * - suse_importer_v2 + - CVE ID of the record + * - ubuntu_osv_importer_v2 + - ID of the OSV record + * - vulnrichment_importer_v2 + - CVE ID of the record + * - xen_importer_v2 + - XSA- \ No newline at end of file From 736c1fbc5f186f670c8bfadbd6fff78c21d79f14 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 25 Feb 2026 14:38:19 +0200 Subject: [PATCH 07/19] Fix null constraint violations in multiple v1 exploit collection pipelines Signed-off-by: ziad hany Signed-off-by: Tedsig42 --- vulnerabilities/pipelines/enhance_with_exploitdb.py | 3 ++- vulnerabilities/pipelines/enhance_with_kev.py | 3 ++- vulnerabilities/pipelines/enhance_with_metasploit.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 4d2e966d9..74fc55b09 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -88,7 +88,8 @@ def add_vulnerability_exploit(row, logger): for raw_alias in aliases: try: if alias := Alias.objects.get(alias=raw_alias): - vulnerabilities.add(alias.vulnerability) + if alias.vulnerability: + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index c9fc21a84..00fc72106 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -74,7 +74,8 @@ def add_vulnerability_exploit(kev_vul, logger): vulnerability = None try: if alias := Alias.objects.get(alias=cve_id): - vulnerability = alias.vulnerability + if alias.vulnerability: + vulnerability = alias.vulnerability except Alias.DoesNotExist: logger(f"No vulnerability found for aliases {cve_id}") return 0 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index a9b901400..12437fadc 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -79,7 +79,8 @@ def add_vulnerability_exploit(record, logger): for ref in interesting_references: try: if alias := Alias.objects.get(alias=ref): - vulnerabilities.add(alias.vulnerability) + if alias.vulnerability: + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue From f7a0e9995ff9579cf2b7952ac6b1ce09c10acd1b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 25 Feb 2026 15:43:09 +0200 Subject: [PATCH 08/19] Add a test, and update Kev pipeline Signed-off-by: ziad hany Signed-off-by: Tedsig42 --- vulnerabilities/pipelines/enhance_with_kev.py | 4 ++++ .../pipelines/test_enhance_with_exploitdb.py | 15 +++++++++++++++ .../tests/pipelines/test_enhance_with_kev.py | 15 +++++++++++++++ .../pipelines/test_enhance_with_metasploit.py | 14 ++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 00fc72106..b46daa789 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -80,6 +80,10 @@ def add_vulnerability_exploit(kev_vul, logger): logger(f"No vulnerability found for aliases {cve_id}") return 0 + if not vulnerability: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + Exploit.objects.update_or_create( vulnerability=vulnerability, data_source="KEV", diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py index f54dad55d..1b47591ba 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py @@ -45,3 +45,18 @@ def test_exploit_db_improver(mock_get): # Run Exploit-DB Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + Alias.objects.create(alias="CVE-2009-3699", vulnerability=None) + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py index a93c16555..947d94a0a 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py @@ -45,3 +45,18 @@ def test_kev_improver(mock_get): # Run Kev Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_kev_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = VulnerabilityKevPipeline() + Alias.objects.create(alias="CVE-2021-38647", vulnerability=None) + + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py index eea99e0ca..a4f4dae83 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py @@ -42,3 +42,17 @@ def test_metasploit_improver(mock_get): # Run metasploit Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + Alias.objects.create(alias="CVE-2007-4387", vulnerability=None) # Alias without vulnerability + improver = MetasploitImproverPipeline() + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 From d8436bb4b8a3f634b94cda6cb82f3bb5999525e8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 27 Feb 2026 22:02:40 +0200 Subject: [PATCH 09/19] Update (exploitdb, kev, metasploit ) pipelines to do single db query Signed-off-by: ziad hany Signed-off-by: Tedsig42 --- .../pipelines/enhance_with_exploitdb.py | 20 ++++----- vulnerabilities/pipelines/enhance_with_kev.py | 44 +++++++++---------- .../pipelines/enhance_with_metasploit.py | 15 +++---- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 74fc55b09..70f7b4886 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -78,20 +78,16 @@ def add_exploit(self): def add_vulnerability_exploit(row, logger): - vulnerabilities = set() - aliases = row["codes"].split(";") if row["codes"] else [] if not aliases: return 0 - for raw_alias in aliases: - try: - if alias := Alias.objects.get(alias=raw_alias): - if alias.vulnerability: - vulnerabilities.add(alias.vulnerability) - except Alias.DoesNotExist: - continue + vulnerabilities = ( + Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False) + .values_list("vulnerability_id", flat=True) + .distinct() + ) if not vulnerabilities: logger(f"No vulnerability found for aliases {aliases}") @@ -105,7 +101,7 @@ def add_vulnerability_exploit(row, logger): add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) try: Exploit.objects.update_or_create( - vulnerability=vulnerability, + vulnerability_id=vulnerability, data_source="Exploit-DB", defaults={ "date_added": date_added, @@ -126,7 +122,7 @@ def add_vulnerability_exploit(row, logger): return 1 -def add_exploit_references(ref_id, direct_url, path, vul, logger): +def add_exploit_references(ref_id, direct_url, path, vul_id, logger): url_map = { "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", "direct_url": direct_url, @@ -145,7 +141,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger): if created: VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, + vulnerability_id=vul_id, reference=ref, ) diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index b46daa789..b9b0a84f4 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -71,31 +71,29 @@ def add_vulnerability_exploit(kev_vul, logger): if not cve_id: return 0 - vulnerability = None - try: - if alias := Alias.objects.get(alias=cve_id): - if alias.vulnerability: - vulnerability = alias.vulnerability - except Alias.DoesNotExist: - logger(f"No vulnerability found for aliases {cve_id}") - return 0 + vulnerabilities = ( + Alias.objects.filter(alias=cve_id, vulnerability__isnull=False) + .values_list("vulnerability", flat=True) + .distinct() + ) - if not vulnerability: + if not vulnerabilities: logger(f"No vulnerability found for aliases {cve_id}") return 0 - Exploit.objects.update_or_create( - vulnerability=vulnerability, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) + for vulnerability in vulnerabilities: + Exploit.objects.update_or_create( + vulnerability_id=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 12437fadc..7e28160f9 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -66,7 +66,6 @@ def add_vulnerability_exploits(self): def add_vulnerability_exploit(record, logger): - vulnerabilities = set() references = record.get("references", []) interesting_references = [ @@ -76,13 +75,11 @@ def add_vulnerability_exploit(record, logger): if not interesting_references: return 0 - for ref in interesting_references: - try: - if alias := Alias.objects.get(alias=ref): - if alias.vulnerability: - vulnerabilities.add(alias.vulnerability) - except Alias.DoesNotExist: - continue + vulnerabilities = ( + Alias.objects.filter(alias__in=interesting_references, vulnerability__isnull=False) + .values_list("vulnerability", flat=True) + .distinct() + ) if not vulnerabilities: logger(f"No vulnerability found for aliases {interesting_references}") @@ -108,7 +105,7 @@ def add_vulnerability_exploit(record, logger): for vulnerability in vulnerabilities: Exploit.objects.update_or_create( - vulnerability=vulnerability, + vulnerability_id=vulnerability, data_source="Metasploit", defaults={ "description": description, From ffef300ff0c86b390db7fd33724ca9217ed80793 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 27 Feb 2026 22:07:31 +0200 Subject: [PATCH 10/19] Fix a vulnrichment importer test to correctly mock a CVSSv4 score Signed-off-by: ziad hany Signed-off-by: Tedsig42 --- .../test_vulnrichment_importer_v2.py | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py index 1fb6f190e..f5c251e7f 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py @@ -8,15 +8,16 @@ # import json -from pathlib import Path from unittest.mock import MagicMock from unittest.mock import patch import pytest from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines.v2_importers.vulnrichment_importer import VulnrichImporterPipeline +from vulnerabilities.severity_systems import Cvssv4ScoringSystem @pytest.fixture @@ -58,8 +59,10 @@ def mock_pathlib(tmp_path): "metrics": [ { "cvssV4_0": { - "baseScore": 7.5, - "vectorString": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + "version": "4.0", + "baseScore": 5.3, + "baseSeverity": "MEDIUM", + "vectorString": "CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", } } ], @@ -103,15 +106,20 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) mock_parse.return_value = AdvisoryDataV2( advisory_id="CVE-2021-1234", summary="Sample PyPI vulnerability", - references=[{"url": "https://example.com"}], + references=[ReferenceV2(url="https://example.com")], affected_packages=[], weaknesses=[], url="https://example.com", severities=[ VulnerabilitySeverity( - system="cvssv4", - value=7.5, - scoring_elements="AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + system=Cvssv4ScoringSystem( + identifier="cvssv4", + name="CVSSv4 Base Score", + url="https://www.first.org/cvss/v4-0/", + notes="CVSSv4 base score and vector", + ), + value="5.3", + scoring_elements="CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", ) ], ) @@ -126,6 +134,7 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) assert advisory.advisory_id == "CVE-2021-1234" assert advisory.summary == "Sample PyPI vulnerability" assert advisory.url == "https://example.com" + assert len(advisory.severities) == 1 def test_clean_downloads(mock_vcs_response, mock_fetch_via_vcs): @@ -165,8 +174,10 @@ def test_parse_cve_advisory(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) "metrics": [ { "cvssV4_0": { - "baseScore": 7.5, - "vectorString": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + "version": "4.0", + "baseScore": 5.3, + "baseSeverity": "MEDIUM", + "vectorString": "CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", } } ], @@ -185,7 +196,7 @@ def test_parse_cve_advisory(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) assert advisory.summary == "Sample PyPI vulnerability" assert advisory.url == advisory_url assert len(advisory.severities) == 1 - assert advisory.severities[0].value == 7.5 + assert advisory.severities[0].value == 5.3 def test_collect_advisories_with_invalid_json(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs): From e57efe0fc615aa7dd39edd0a85e75d9fde092bb9 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Feb 2026 01:15:54 +0530 Subject: [PATCH 11/19] Compute content_id from all fields of AdvisoryV2 Resolves: https://github.com/aboutcode-org/vulnerablecode/issues/2186 Signed-off-by: Keshav Priyadarshi Signed-off-by: Tedsig42 --- vulnerabilities/utils.py | 59 +++++++++++++--------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 82f29bcea..0eb1d1258 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -666,53 +666,32 @@ def compute_content_id_v2(advisory_data): """ Compute a unique content_id for an advisory by normalizing its data and hashing it. - :param advisory_data: An AdvisoryData object + :param advisory_data: An AdvisoryDataV2 or AdvisoryV2 object :return: SHA-256 hash digest as content_id """ - - # Normalize fields from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.models import AdvisoryV2 - if isinstance(advisory_data, AdvisoryV2): - normalized_data = { - "aliases": normalize_list(advisory_data.aliases), - "summary": normalize_text(advisory_data.summary), - "impacted_packages": sorted( - [impact.to_dict() for impact in advisory_data.impacted_packages.all()], - key=lambda x: json.dumps(x, sort_keys=True), - ), - "patches": sorted( - [patch.to_patch_data().to_dict() for patch in advisory_data.patches.all()], - key=lambda x: json.dumps(x, sort_keys=True), - ), - "references": [ref for ref in normalize_list(advisory_data.references) if ref], - "weaknesses": normalize_list(advisory_data.weaknesses), - } - normalized_data["url"] = advisory_data.url - - elif isinstance(advisory_data, AdvisoryDataV2): - normalized_data = { - "advisory_id": normalize_text(advisory_data.advisory_id), - "aliases": normalize_list(advisory_data.aliases), - "summary": normalize_text(advisory_data.summary), - "affected_packages": [ - pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg - ], - "references": [ - ref.to_dict() for ref in normalize_list(advisory_data.references) if ref - ], - "severities": [ - sev.to_dict() for sev in normalize_list(advisory_data.severities) if sev - ], - "weaknesses": normalize_list(advisory_data.weaknesses), - "patches": [patch.to_dict() for patch in normalize_list(advisory_data.patches)], - } - normalized_data["url"] = advisory_data.url - - else: + if not isinstance(advisory_data, (AdvisoryV2, AdvisoryDataV2)): raise ValueError("Unsupported advisory data type for content ID computation") + if isinstance(advisory_data, AdvisoryV2): + advisory_data = advisory_data.to_advisory_data() + + normalized_data = { + "advisory_id": normalize_text(advisory_data.advisory_id), + "aliases": normalize_list(advisory_data.aliases), + "summary": normalize_text(advisory_data.summary), + "affected_packages": [ + pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg + ], + "references": [ref.to_dict() for ref in normalize_list(advisory_data.references) if ref], + "severities": [sev.to_dict() for sev in normalize_list(advisory_data.severities) if sev], + "weaknesses": normalize_list(advisory_data.weaknesses), + "patches": [patch.to_dict() for patch in normalize_list(advisory_data.patches)], + } + normalized_data["url"] = advisory_data.url + normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) content_id = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() From b77c68d61cbb720e683811225707a06924818a1f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Feb 2026 12:16:28 +0530 Subject: [PATCH 12/19] Add tests for compute_content_id_v2 Signed-off-by: Keshav Priyadarshi Signed-off-by: Tedsig42 --- vulnerabilities/tests/test_utils.py | 100 ++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/vulnerabilities/tests/test_utils.py b/vulnerabilities/tests/test_utils.py index c9ba98e79..0460715be 100644 --- a/vulnerabilities/tests/test_utils.py +++ b/vulnerabilities/tests/test_utils.py @@ -7,12 +7,27 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from datetime import datetime +from datetime import timedelta + +from django.test import TestCase from fetchcode.package_versions import PackageVersion from packageurl import PackageURL from univers.version_constraint import VersionConstraint from univers.version_range import GemVersionRange +from univers.version_range import VersionRange from univers.versions import RubygemsVersion +from vulnerabilities import utils +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import PackageCommitPatchData +from vulnerabilities.importer import PatchData +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.pipelines import insert_advisory_v2 +from vulnerabilities.references import XsaReferenceV2 +from vulnerabilities.references import ZbxReferenceV2 from vulnerabilities.utils import AffectedPackage from vulnerabilities.utils import get_item from vulnerabilities.utils import get_severity_range @@ -151,3 +166,88 @@ def test_resolve_version_range_without_ignorable_versions(): def test_get_severity_range(): assert get_severity_range({""}) is None assert get_severity_range({}) is None + + +class TestComputeContentIdV2(TestCase): + def setUp(self): + self.advisory1 = AdvisoryDataV2( + summary="Test advisory", + aliases=["CVE-2025-0001", "CVE-2024-0001"], + references=[ + XsaReferenceV2.from_number(248), + ZbxReferenceV2.from_id("ZBX-000"), + ], + severities=[ + VulnerabilitySeverity.from_dict( + { + "system": "cvssv4", + "value": "7.5", + "scoring_elements": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + } + ), + VulnerabilitySeverity.from_dict( + { + "system": "cvssv3", + "value": "6.5", + "scoring_elements": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + } + ), + ], + weaknesses=[296, 233], + affected_packages=[ + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=0.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/0.2.4"), + introduced_by_commit_patches=[ + PackageCommitPatchData( + vcs_url="https://foobar.vcs/", + commit_hash="662f801f", + ), + PackageCommitPatchData( + vcs_url="https://foobar.vcs/", + commit_hash="001f801f", + ), + ], + fixed_by_commit_patches=[ + PackageCommitPatchData( + vcs_url="https://foobar.vcs/", + commit_hash="982f801f", + ), + PackageCommitPatchData( + vcs_url="https://foobar.vcs/", + commit_hash="081f801f", + ), + ], + ), + ], + patches=[ + PatchData(patch_url="https://foo.bar/", patch_text="test patch"), + PatchData(patch_url="https://yet-another-foo.bar/", patch_text="some test patch"), + ], + advisory_id="ADV-001", + date_published=datetime.now() - timedelta(days=10), + url="https://example.com/advisory/1", + ) + insert_advisory_v2( + advisory=self.advisory1, + pipeline_id="test_pipeline_v2", + ) + + def test_compute_content_id_v2(self): + result = utils.compute_content_id_v2(self.advisory1) + self.assertEqual(result, "5211f1e6c3d935759fb288d79a865eeacc06e3e0e352ab7f5b4cb0e76a43a955") + + def test_content_id_from_adv_data_and_adv_model_are_same(self): + id_from_data = utils.compute_content_id_v2(self.advisory1) + advisory_model = AdvisoryV2.objects.first() + id_from_model = utils.compute_content_id_v2(advisory_model) + + self.assertEqual(id_from_data, id_from_model) From c5de735575d876dc8963c6e7b41075570f9a5365 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Feb 2026 18:12:51 +0530 Subject: [PATCH 13/19] Do not update related field of an immutable AdvisoryV2 Resolves: https://github.com/aboutcode-org/vulnerablecode/issues/2187 Signed-off-by: Keshav Priyadarshi Signed-off-by: Tedsig42 --- vulnerabilities/pipes/advisory.py | 117 +++++++++--------- .../test_federate_vulnerabilities.py | 2 + .../v2_improvers/test_unfurl_version_range.py | 3 + vulnerabilities/tests/test_models.py | 15 ++- vulnerabilities/tests/test_utils.py | 3 + 5 files changed, 80 insertions(+), 60 deletions(-) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 96e1d4871..c812fdf86 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -291,7 +291,7 @@ def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = def insert_advisory_v2( advisory: AdvisoryDataV2, pipeline_id: str, - logger: Callable = None, + logger: Callable, precedence: int = 0, ): from vulnerabilities.models import ImpactedPackage @@ -299,13 +299,8 @@ def insert_advisory_v2( from vulnerabilities.utils import compute_content_id_v2 advisory_obj = None - aliases = get_or_create_advisory_aliases(aliases=advisory.aliases) - references = get_or_create_advisory_references(references=advisory.references) - severities = get_or_create_advisory_severities(severities=advisory.severities) - patches = get_or_create_advisory_patches(patches=advisory.patches) - weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses) - content_id = compute_content_id_v2(advisory_data=advisory) created = False + content_id = compute_content_id_v2(advisory_data=advisory) try: default_data = { "datasource_id": pipeline_id, @@ -324,65 +319,73 @@ def insert_advisory_v2( unique_content_id=content_id, defaults=default_data, ) - related_fields = { - "aliases": aliases, - "references": references, - "severities": severities, - "weaknesses": weaknesses, - "patches": patches, - } - - for field_name, values in related_fields.items(): - if values: - getattr(advisory_obj, field_name).add(*values) - - except Advisory.MultipleObjectsReturned: + except AdvisoryV2.MultipleObjectsReturned: logger( f"Multiple Advisories returned: unique_content_id: {content_id}, url: {advisory.url}, advisory: {advisory!r}" ) raise except Exception as e: - if logger: - logger( - f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) + logger( + f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + raise - if created: - for affected_pkg in advisory.affected_packages: - impact = ImpactedPackage.objects.create( - advisory=advisory_obj, - base_purl=str(affected_pkg.package), - affecting_vers=str(affected_pkg.affected_version_range) - if affected_pkg.affected_version_range - else None, - fixed_vers=str(affected_pkg.fixed_version_range) - if affected_pkg.fixed_version_range - else None, - ) - package_affected_purls, package_fixed_purls = get_exact_purls_v2( - affected_package=affected_pkg, - logger=logger, - ) + if not created: + return advisory_obj - affected_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls( - purls=package_affected_purls - ) - fixed_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls( - purls=package_fixed_purls - ) + aliases = get_or_create_advisory_aliases(aliases=advisory.aliases) + references = get_or_create_advisory_references(references=advisory.references) + severities = get_or_create_advisory_severities(severities=advisory.severities) + patches = get_or_create_advisory_patches(patches=advisory.patches) + weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses) - impact.affecting_packages.add(*affected_packages_v2) - impact.fixed_by_packages.add(*fixed_packages_v2) + related_fields = { + "aliases": aliases, + "references": references, + "severities": severities, + "weaknesses": weaknesses, + "patches": patches, + } + + for field_name, values in related_fields.items(): + if values: + getattr(advisory_obj, field_name).add(*values) + + for affected_pkg in advisory.affected_packages: + impact = ImpactedPackage.objects.create( + advisory=advisory_obj, + base_purl=str(affected_pkg.package), + affecting_vers=str(affected_pkg.affected_version_range) + if affected_pkg.affected_version_range + else None, + fixed_vers=str(affected_pkg.fixed_version_range) + if affected_pkg.fixed_version_range + else None, + ) + package_affected_purls, package_fixed_purls = get_exact_purls_v2( + affected_package=affected_pkg, + logger=logger, + ) - introduced_commit_v2 = get_or_create_advisory_package_commit_patches( - affected_pkg.introduced_by_commit_patches - ) - fixed_commit_v2 = get_or_create_advisory_package_commit_patches( - affected_pkg.fixed_by_commit_patches - ) - impact.introduced_by_package_commit_patches.add(*introduced_commit_v2) - impact.fixed_by_package_commit_patches.add(*fixed_commit_v2) + affected_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls( + purls=package_affected_purls + ) + fixed_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls( + purls=package_fixed_purls + ) + + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + + introduced_commit_v2 = get_or_create_advisory_package_commit_patches( + affected_pkg.introduced_by_commit_patches + ) + fixed_commit_v2 = get_or_create_advisory_package_commit_patches( + affected_pkg.fixed_by_commit_patches + ) + impact.introduced_by_package_commit_patches.add(*introduced_commit_v2) + impact.fixed_by_package_commit_patches.add(*fixed_commit_v2) return advisory_obj diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py index a0bbbb772..af36cc711 100644 --- a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -86,10 +86,12 @@ def setUp(self): insert_advisory_v2( advisory=advisory1, pipeline_id="test_pipeline_v2", + logger=self.logger.write, ) insert_advisory_v2( advisory=advisory2, pipeline_id="test_pipeline_v2", + logger=self.logger.write, ) @patch( diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py b/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py index 2319f27d0..a1927a426 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py @@ -22,10 +22,12 @@ from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines.v2_improvers.unfurl_version_range import UnfurlVersionRangePipeline from vulnerabilities.pipes.advisory import insert_advisory_v2 +from vulnerabilities.tests.pipelines import TestLogger class TestUnfurlVersionRangePipeline(TestCase): def setUp(self): + self.logger = TestLogger() advisory1 = AdvisoryDataV2( summary="Test advisory", aliases=["CVE-2025-0001"], @@ -49,6 +51,7 @@ def setUp(self): insert_advisory_v2( advisory=advisory1, pipeline_id="test_pipeline_v2", + logger=self.logger.write, ) @patch("vulnerabilities.pipelines.v2_improvers.unfurl_version_range.get_purl_versions") diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 96dc9b0ff..69237b806 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -39,6 +39,7 @@ from vulnerabilities.severity_systems import CVSSV3 from vulnerabilities.severity_systems import CVSSV4 from vulnerabilities.severity_systems import ScoringSystem +from vulnerabilities.tests.pipelines import TestLogger from vulnerabilities.utils import compute_content_id @@ -747,6 +748,7 @@ def test_constraint_none(self): class TestAdvisoryV2Model(DjangoTestCase): def setUp(self): + self.logger = TestLogger() self.advisoryv2_data1 = AdvisoryDataV2( advisory_id="test_adv", aliases=[], @@ -770,7 +772,9 @@ def setUp(self): def test_advisoryv2_to_advisory_data_patch_seralization(self): from vulnerabilities.pipes.advisory import insert_advisory_v2 - insert_advisory_v2(advisory=self.advisoryv2_data1, pipeline_id="test_pipeline") + insert_advisory_v2( + advisory=self.advisoryv2_data1, pipeline_id="test_pipeline", logger=self.logger.write + ) result = models.AdvisoryV2.objects.first().to_advisory_data() self.assertEqual(result, self.advisoryv2_data1) @@ -778,6 +782,7 @@ def test_advisoryv2_to_advisory_data_patch_seralization(self): class TestAdvisoryV2ModelDuplication(DjangoTestCase): def setUp(self): + self.logger = TestLogger() self.advisoryv2_data1 = AdvisoryDataV2( advisory_id="CVE-2023-0401", aliases=[], @@ -813,8 +818,12 @@ def setUp(self): def test_advisoryv2_duplication_data(self): from vulnerabilities.pipes.advisory import insert_advisory_v2 - insert_advisory_v2(advisory=self.advisoryv2_data1, pipeline_id="test_pipeline") - insert_advisory_v2(advisory=self.advisoryv2_data2, pipeline_id="test_pipeline") + insert_advisory_v2( + advisory=self.advisoryv2_data1, pipeline_id="test_pipeline", logger=self.logger.write + ) + insert_advisory_v2( + advisory=self.advisoryv2_data2, pipeline_id="test_pipeline", logger=self.logger.write + ) result = models.AdvisoryV2.objects.count() self.assertEqual(result, 2) diff --git a/vulnerabilities/tests/test_utils.py b/vulnerabilities/tests/test_utils.py index 0460715be..a100cbd74 100644 --- a/vulnerabilities/tests/test_utils.py +++ b/vulnerabilities/tests/test_utils.py @@ -28,6 +28,7 @@ from vulnerabilities.pipelines import insert_advisory_v2 from vulnerabilities.references import XsaReferenceV2 from vulnerabilities.references import ZbxReferenceV2 +from vulnerabilities.tests.pipelines import TestLogger from vulnerabilities.utils import AffectedPackage from vulnerabilities.utils import get_item from vulnerabilities.utils import get_severity_range @@ -170,6 +171,7 @@ def test_get_severity_range(): class TestComputeContentIdV2(TestCase): def setUp(self): + self.logger = TestLogger() self.advisory1 = AdvisoryDataV2( summary="Test advisory", aliases=["CVE-2025-0001", "CVE-2024-0001"], @@ -239,6 +241,7 @@ def setUp(self): insert_advisory_v2( advisory=self.advisory1, pipeline_id="test_pipeline_v2", + logger=self.logger.write, ) def test_compute_content_id_v2(self): From daa490e52ae005cb1ee230eb75e326ae15c5ade4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 14:43:52 +0530 Subject: [PATCH 14/19] Store advisory content hash Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- vulnerabilities/improvers/__init__.py | 2 + .../0116_advisoryv2_advisory_content_hash.py | 23 +++++++ vulnerabilities/models.py | 36 ++-------- .../v2_importers/alpine_linux_importer.py | 3 +- .../v2_importers/apache_httpd_importer.py | 15 +++-- .../v2_importers/elixir_security_importer.py | 1 + .../pipelines/v2_importers/gitlab_importer.py | 16 +++-- .../pipelines/v2_importers/ruby_importer.py | 4 +- .../compute_advisory_content_hash.py | 67 +++++++++++++++++++ vulnerabilities/pipes/advisory.py | 3 + vulnerabilities/utils.py | 33 ++++++++- 11 files changed, 157 insertions(+), 46 deletions(-) create mode 100644 vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py create mode 100644 vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 97c18e6f9..17923280b 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -33,6 +33,7 @@ from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 +from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash from vulnerabilities.utils import create_registry IMPROVERS_REGISTRY = create_registry( @@ -74,5 +75,6 @@ compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, + compute_advisory_content_hash.ComputeAdvisoryContentHash, ] ) diff --git a/vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py b/vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py new file mode 100644 index 000000000..2844f034d --- /dev/null +++ b/vulnerabilities/migrations/0116_advisoryv2_advisory_content_hash.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.11 on 2026-03-11 08:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0115_impactedpackageaffecting_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="advisoryv2", + name="advisory_content_hash", + field=models.CharField( + blank=True, + help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.", + max_length=64, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 1981a2861..d1c88f285 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3010,6 +3010,13 @@ class AdvisoryV2(models.Model): help_text="Related advisories that are used to calculate the severity of this advisory.", ) + advisory_content_hash = models.CharField( + max_length=64, + blank=True, + null=True, + help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.", + ) + @property def risk_score(self): """ @@ -3078,35 +3085,6 @@ def get_aliases(self): """ return self.aliases.all() - def compute_advisory_content(self): - """ - Compute a unique content hash for an advisory by normalizing its data and hashing it. - - :param advisory: An Advisory object - :return: SHA-256 hash digest as content hash - """ - normalized_data = { - "summary": normalize_text(self.summary), - "impacted_packages": sorted( - [impact.to_dict() for impact in self.impacted_packages.all()], - key=lambda x: json.dumps(x, sort_keys=True), - ), - "patches": sorted( - [patch.to_patch_data().to_dict() for patch in self.patches.all()], - key=lambda x: json.dumps(x, sort_keys=True), - ), - "severities": sorted( - [sev.to_vulnerability_severity_data().to_dict() for sev in self.severities.all()], - key=lambda x: (x.get("system"), x.get("value")), - ), - "weaknesses": normalize_list([weakness.cwe_id for weakness in self.weaknesses.all()]), - } - - normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) - content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - - return content_hash - alias = get_aliases diff --git a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py index 336afa298..4c176732e 100644 --- a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py @@ -193,7 +193,8 @@ def load_advisories( fixed_version_range = None try: - fixed_version_range = AlpineLinuxVersionRange.from_versions([version]) + if version: + fixed_version_range = AlpineLinuxVersionRange.from_versions([version]) except InvalidVersion as e: logger( f"{version!r} is not a valid AlpineVersion {e!r}", diff --git a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py index 51e4b5e77..3987eea73 100644 --- a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py +++ b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py @@ -330,7 +330,7 @@ def to_version_ranges(self, versions_data, fixed_versions): "=": "=", } comparator = comparator_by_range_expression.get(range_expression) - if comparator: + if comparator and version_value and version_value not in self.ignorable_versions: constraints.append( VersionConstraint(comparator=comparator, version=SemverVersion(version_value)) ) @@ -338,11 +338,12 @@ def to_version_ranges(self, versions_data, fixed_versions): for fixed_version in fixed_versions: # The VersionConstraint method `invert()` inverts the fixed_version's comparator, # enabling inclusion of multiple fixed versions with the `affected_version_range` values. - constraints.append( - VersionConstraint( - comparator="=", - version=SemverVersion(fixed_version), - ).invert() - ) + if fixed_version and fixed_version not in self.ignorable_versions: + constraints.append( + VersionConstraint( + comparator="=", + version=SemverVersion(fixed_version), + ).invert() + ) return ApacheVersionRange(constraints=constraints) diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py index 8d7cdfc9e..3b9f86d8e 100644 --- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py +++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py @@ -35,6 +35,7 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC0-1.0" license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt" repo_url = "git+https://github.com/dependabot/elixir-security-advisories" + run_once = True precedence = 200 diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py index 7c164b1c0..24c25679f 100644 --- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py @@ -57,14 +57,14 @@ def steps(cls): purl_type_by_gitlab_scheme = { "conan": "conan", - "gem": "gem", + # "gem": "gem", # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 # "go": "golang", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "packagist": "composer", - "pypi": "pypi", + # "maven": "maven", + # "npm": "npm", + # "nuget": "nuget", + # "packagist": "composer", + # "pypi": "pypi", } gitlab_scheme_by_purl_type = {v: k for k, v in purl_type_by_gitlab_scheme.items()} @@ -252,6 +252,7 @@ def parse_gitlab_advisory( original_advisory_text=json.dumps(gitlab_advisory, indent=2, ensure_ascii=False), ) affected_version_range = None + fixed_version_range = None fixed_versions = gitlab_advisory.get("fixed_versions") or [] affected_range = gitlab_advisory.get("affected_range") gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"]) @@ -285,7 +286,8 @@ def parse_gitlab_advisory( if affected_version_range: vrc = affected_version_range.__class__ - fixed_version_range = vrc.from_versions(parsed_fixed_versions) + if parsed_fixed_versions: + fixed_version_range = vrc.from_versions(parsed_fixed_versions) if not fixed_version_range and not affected_version_range: return diff --git a/vulnerabilities/pipelines/v2_importers/ruby_importer.py b/vulnerabilities/pipelines/v2_importers/ruby_importer.py index 455a0a491..fad09a1b5 100644 --- a/vulnerabilities/pipelines/v2_importers/ruby_importer.py +++ b/vulnerabilities/pipelines/v2_importers/ruby_importer.py @@ -162,7 +162,9 @@ def get_affected_packages(record, purl): affected_packages = [] for unaffected_version in record.get("unaffected_versions", []): try: - affected_version_range = GemVersionRange.from_native(unaffected_version).invert() + if unaffected_version: + unaffected_version = unaffected_version.strip() + affected_version_range = GemVersionRange.from_native(unaffected_version).invert() validate_comparators(affected_version_range.constraints) affected_packages.append( AffectedPackageV2( diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py new file mode 100644 index 000000000..59655183c --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py @@ -0,0 +1,67 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.utils import compute_advisory_content + + +class ComputeAdvisoryContentHash(VulnerableCodePipeline): + """Compute Advisory Content Hash for Advisory.""" + + pipeline_id = "compute_advisory_content_hash_v2" + + @classmethod + def steps(cls): + return (cls.compute_advisory_content_hash,) + + def compute_advisory_content_hash(self): + """Create ToDos for missing summary, affected and fixed packages.""" + + advisories = AdvisoryV2.objects.filter(advisory_content_hash__isnull=True) + + advisories_count = advisories.count() + + self.log( + f"Checking missing summary, affected and fixed packages in {advisories_count} Advisories" + ) + progress = LoopProgress( + total_iterations=advisories_count, + logger=self.log, + progress_step=1, + ) + + to_update = [] + batch_size = 5000 + + for advisory in progress.iter( + advisories.iterator(chunk_size=batch_size) + ): + advisory.advisory_content_hash = compute_advisory_content(advisory) + to_update.append(advisory) + + if len(to_update) >= batch_size: + AdvisoryV2.objects.bulk_update( + to_update, + ["advisory_content_hash"], + batch_size=batch_size, + ) + to_update.clear() + + if to_update: + AdvisoryV2.objects.bulk_update( + to_update, + ["advisory_content_hash"], + batch_size=batch_size, + ) + + self.log("Finished computing advisory_content_hash") diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index c812fdf86..00414a259 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,6 +48,7 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 +from vulnerabilities.utils import compute_advisory_content def get_or_create_aliases(aliases: List) -> QuerySet: @@ -301,6 +302,7 @@ def insert_advisory_v2( advisory_obj = None created = False content_id = compute_content_id_v2(advisory_data=advisory) + advisory_content_hash = compute_advisory_content(advisory_data=advisory) try: default_data = { "datasource_id": pipeline_id, @@ -311,6 +313,7 @@ def insert_advisory_v2( "original_advisory_text": advisory.original_advisory_text, "url": advisory.url, "precedence": precedence, + "advisory_content_hash": advisory_content_hash, } advisory_obj, created = AdvisoryV2.objects.get_or_create( diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 0eb1d1258..f90d42401 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -848,7 +848,11 @@ def group_advisories_by_content(advisories): grouped = {} for advisory in advisories: - content_hash = advisory.compute_advisory_content() + content_hash = ( + advisory.advisory_content_hash + if advisory.advisory_content_hash + else compute_advisory_content(advisory) + ) entry = grouped.setdefault( content_hash, @@ -867,3 +871,30 @@ def group_advisories_by_content(advisories): entry["secondary"].add(advisory) return grouped + + +def compute_advisory_content(advisory_data): + """ + Compute a unique content hash for an advisory by normalizing its data and hashing it. + + :param advisory_data: An AdvisoryData object + :return: SHA-256 hash digest as content hash + """ + from vulnerabilities.models import AdvisoryV2 + + if isinstance(advisory_data, AdvisoryV2): + advisory_data = advisory_data.to_advisory_data() + normalized_data = { + "summary": normalize_text(advisory_data.summary), + "affected_packages": [ + pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg + ], + "severities": [sev.to_dict() for sev in normalize_list(advisory_data.severities) if sev], + "weaknesses": normalize_list(advisory_data.weaknesses), + "patches": [patch.to_dict() for patch in normalize_list(advisory_data.patches)], + } + + normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) + content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() + + return content_hash From 472f17bf67323994506cddd39b3aefc1cd85434e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 14:48:43 +0530 Subject: [PATCH 15/19] Allow all ecosystems in gitlab Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- .../pipelines/v2_importers/gitlab_importer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py index 24c25679f..2c12f3a1a 100644 --- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py @@ -57,14 +57,14 @@ def steps(cls): purl_type_by_gitlab_scheme = { "conan": "conan", - # "gem": "gem", + "gem": "gem", # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 # "go": "golang", - # "maven": "maven", - # "npm": "npm", - # "nuget": "nuget", - # "packagist": "composer", - # "pypi": "pypi", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "packagist": "composer", + "pypi": "pypi", } gitlab_scheme_by_purl_type = {v: k for k, v in purl_type_by_gitlab_scheme.items()} From 93e1b744fa3e1042e419b61b6f4df6d36c7454ba Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 14:50:21 +0530 Subject: [PATCH 16/19] Fix formatting issues Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- .../pipelines/v2_improvers/compute_advisory_content_hash.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py index 59655183c..cd91db771 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py @@ -43,9 +43,7 @@ def compute_advisory_content_hash(self): to_update = [] batch_size = 5000 - for advisory in progress.iter( - advisories.iterator(chunk_size=batch_size) - ): + for advisory in progress.iter(advisories.iterator(chunk_size=batch_size)): advisory.advisory_content_hash = compute_advisory_content(advisory) to_update.append(advisory) From 857c692a30399c1975e9cf522e492223b209b125 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 14:51:39 +0530 Subject: [PATCH 17/19] Fix formatting issues Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- vulnerabilities/improvers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 17923280b..982b4bbd8 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -20,6 +20,7 @@ from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees +from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -33,7 +34,6 @@ from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 -from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash from vulnerabilities.utils import create_registry IMPROVERS_REGISTRY = create_registry( From 1991ce837effb6a8dfb124633f4843934de6450b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 15:21:40 +0530 Subject: [PATCH 18/19] Add tests Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- .../test_compute_advisory_content_hash.py | 88 +++++++++++++++++++ vulnerabilities/tests/test_api_v2.py | 8 +- 2 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py new file mode 100644 index 000000000..5b7f0c186 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py @@ -0,0 +1,88 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from unittest.mock import patch + +import pytest + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash import ( + ComputeAdvisoryContentHash, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def advisory_factory(): + def _create(count, with_hash=False, start=0): + objs = [] + for i in range(start, start + count): + objs.append( + AdvisoryV2( + summary=f"summary {i}", + advisory_content_hash="existing_hash" if with_hash else None, + unique_content_id=f"unique_id_{i}", + advisory_id=f"ADV-{i}", + datasource_id="ds", + avid=f"ds/ADV-{i}", + url=f"https://example.com/ADV-{i}", + ) + ) + return AdvisoryV2.objects.bulk_create(objs) + + return _create + + +def run_pipeline(): + pipeline = ComputeAdvisoryContentHash() + pipeline.compute_advisory_content_hash() + + +@patch( + "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" +) +def test_pipeline_updates_only_missing_hash(mock_compute, advisory_factory): + advisory_factory(3, with_hash=False, start=0) + advisory_factory(2, with_hash=True, start=100) + + mock_compute.return_value = "new_hash" + + run_pipeline() + + updated = AdvisoryV2.objects.filter(advisory_content_hash="new_hash").count() + untouched = AdvisoryV2.objects.filter(advisory_content_hash="existing_hash").count() + + assert updated == 3 + assert untouched == 2 + assert mock_compute.call_count == 3 + + +@patch( + "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" +) +def test_pipeline_bulk_update_batches(mock_compute, advisory_factory): + advisory_factory(6000, with_hash=False) + + mock_compute.return_value = "batch_hash" + + run_pipeline() + + assert AdvisoryV2.objects.filter(advisory_content_hash="batch_hash").count() == 6000 + + assert mock_compute.call_count == 6000 + + +@patch( + "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" +) +def test_pipeline_no_advisories(mock_compute): + run_pipeline() + + assert mock_compute.call_count == 0 diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index ea1a689e6..6968123c7 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -859,7 +859,7 @@ def setUp(self): def test_list_with_purl_filter(self): url = reverse("package-v3-list") - with self.assertNumQueries(29): + with self.assertNumQueries(31): response = self.client.get(url, {"purl": "pkg:pypi/sample@1.0.0"}) assert response.status_code == 200 assert "packages" in response.data["results"] @@ -868,7 +868,7 @@ def test_list_with_purl_filter(self): def test_bulk_lookup(self): url = reverse("package-v3-bulk-lookup") - with self.assertNumQueries(28): + with self.assertNumQueries(30): response = self.client.post(url, {"purls": ["pkg:pypi/sample@1.0.0"]}, format="json") assert response.status_code == 200 assert "packages" in response.data @@ -878,7 +878,7 @@ def test_bulk_lookup(self): def test_bulk_search_plain(self): url = reverse("package-v3-bulk-search") payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": True, "purl_only": False} - with self.assertNumQueries(28): + with self.assertNumQueries(30): response = self.client.post(url, payload, format="json") assert response.status_code == 200 assert "packages" in response.data @@ -894,7 +894,7 @@ def test_bulk_search_purl_only(self): def test_lookup_single_package(self): url = reverse("package-v3-lookup") - with self.assertNumQueries(21): + with self.assertNumQueries(23): response = self.client.post(url, {"purl": "pkg:pypi/sample@1.0.0"}, format="json") assert response.status_code == 200 assert any(pkg["purl"] == "pkg:pypi/sample@1.0.0" for pkg in response.data) From ce442422daa1de407b040b96b6fa8f8da9461443 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 15:29:40 +0530 Subject: [PATCH 19/19] Fix typos Signed-off-by: Tushar Goel Signed-off-by: Tedsig42 --- .../pipelines/v2_improvers/compute_advisory_content_hash.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py index cd91db771..fe5a3c97e 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py @@ -25,15 +25,12 @@ def steps(cls): return (cls.compute_advisory_content_hash,) def compute_advisory_content_hash(self): - """Create ToDos for missing summary, affected and fixed packages.""" + """Compute Advisory Content Hash for Advisory.""" advisories = AdvisoryV2.objects.filter(advisory_content_hash__isnull=True) advisories_count = advisories.count() - self.log( - f"Checking missing summary, affected and fixed packages in {advisories_count} Advisories" - ) progress = LoopProgress( total_iterations=advisories_count, logger=self.log,