diff --git a/api_v3_usage.rst b/api_v3_usage.rst new file mode 100644 index 000000000..26ed9377f --- /dev/null +++ b/api_v3_usage.rst @@ -0,0 +1,246 @@ +Package Endpoint +================ + +We are migrating from **API v1** to **API v3**. + +Previously, the ``/api/packages`` endpoint exposed multiple routes: + +- ``bulk_search`` +- ``bulk_lookup`` +- ``lookup`` +- ``all`` + +In **API v3**, all these capabilities are consolidated into a **single endpoint**: + +:: + + POST /api/v3/packages + + +Pagination +---------- + +Responses from the package endpoint are **always paginated**, with **10 results per page**. + +Each response includes: + +- ``count`` — total number of results +- ``next`` — URL for the next page +- ``previous`` — URL for the previous page + +If a package is associated with **more than 100 advisories**, the response will include: + +- ``affected_by_vulnerabilities_url`` instead of ``affected_by_vulnerabilities`` +- ``fixing_vulnerabilities_url`` instead of ``fixing_vulnerabilities`` + + +Getting All Vulnerable Packages +------------------------------- + +Instead of calling ``/api/packages/all``, call the v3 endpoint with an empty ``purls`` list. + +:: + + POST /api/v3/packages + + { + "purls": [] + } + +Example response: + +:: + + { + "count": 596, + "next": "http://example.com/api/v3/packages?page=2", + "previous": null, + "results": [ + "pkg:npm/626@1.1.1", + "pkg:npm/aedes@0.35.0", + "pkg:npm/airbrake@0.3.8", + "pkg:npm/angular-http-server@1.4.3", + "pkg:npm/apex-publish-static-files@2.0.0", + "pkg:npm/atob@2.0.3", + "pkg:npm/augustine@0.2.3", + "pkg:npm/backbone@0.3.3", + "pkg:npm/base64-url@1.3.3", + "pkg:npm/base64url@2.0.0" + ] + } + + +Bulk Search (Replacement) +------------------------- + +Instead of calling ``/api/packages/bulk_search``, use: + +:: + + POST /api/v3/packages + +Parameters: + +- ``purls`` — list of package URLs to query +- ``details`` — boolean (default: ``false``) +- ``approximate`` — boolean (default: ``false``) + +The ``approximate`` flag replaces the previous ``plain_purl`` parameter. +When set to ``true``, qualifiers and subpaths in PURLs are ignored. + + +Get Only Vulnerable PURLs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": false + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + "pkg:npm/atob@2.0.3" + ] + } + + +Get Detailed Vulnerability Information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": true + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] + } + + +Using Approximate Matching +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3?foo=bar"], + "approximate": true, + "details": true + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] + } + + +Advisory Endpoint +================= + +Retrieve advisories for one or more PURLs: + +:: + + POST /api/v3/advisories + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"] + } + +Responses are paginated (10 results per page) and include ``next`` and ``previous`` links. + + +Affected-By Advisories Endpoint +=============================== + +Retrieve advisories that **affect (impact)** a given PURL: + +:: + + GET /api/v3/affected-by-advisories?purl= + +Example: + +:: + + GET /api/v3/affected-by-advisories?purl=pkg:npm/atob@2.0.3 + + +Fixing Advisories Endpoint +========================== + +Retrieve advisories that are **fixed by** a given PURL: + +:: + + GET /api/v3/fixing-advisories?purl= + +Example: + +:: + + GET /api/v3/fixing-advisories?purl=pkg:npm/atob@2.1.0 \ No newline at end of file diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index 74975b819..6e0ab9213 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -9,6 +9,7 @@ from django.db.models import Prefetch +from django.db.models import Q from django_filters import rest_framework as filters from drf_spectacular.utils import OpenApiParameter from drf_spectacular.utils import extend_schema @@ -25,15 +26,9 @@ from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle -from vulnerabilities.models import AdvisoryReference -from vulnerabilities.models import AdvisorySeverity -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import AdvisoryWeakness from vulnerabilities.models import CodeFix from vulnerabilities.models import CodeFixV2 -from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import Package -from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.models import Vulnerability @@ -41,7 +36,6 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.throttling import PermissionBasedUserRateThrottle -from vulnerabilities.utils import group_advisories_by_content class CharInFilter(filters.BaseInFilter, filters.CharFilter): @@ -58,16 +52,6 @@ class Meta: fields = ["cwe_id", "name", "description"] -class AdvisoryWeaknessSerializer(serializers.ModelSerializer): - cwe_id = serializers.CharField() - name = serializers.CharField() - description = serializers.CharField() - - class Meta: - model = AdvisoryWeakness - fields = ["cwe_id", "name", "description"] - - class VulnerabilityReferenceV2Serializer(serializers.ModelSerializer): url = serializers.CharField() reference_type = serializers.CharField() @@ -78,29 +62,6 @@ class Meta: fields = ["url", "reference_type", "reference_id"] -class AdvisoryReferenceSerializer(serializers.ModelSerializer): - url = serializers.CharField() - reference_type = serializers.CharField() - reference_id = serializers.CharField() - - class Meta: - model = AdvisoryReference - fields = ["url", "reference_type", "reference_id"] - - -class AdvisorySeveritySerializer(serializers.ModelSerializer): - class Meta: - model = AdvisorySeverity - fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] - - def to_representation(self, instance): - data = super().to_representation(instance) - published_at = data.get("published_at", None) - if not published_at: - data.pop("published_at") - return data - - class VulnerabilitySeverityV2Serializer(serializers.ModelSerializer): class Meta: model = VulnerabilitySeverity @@ -141,58 +102,6 @@ def get_aliases(self, obj): return [alias.alias for alias in obj.aliases.all()] -class AdvisoryV2Serializer(serializers.ModelSerializer): - aliases = serializers.SerializerMethodField() - weaknesses = AdvisoryWeaknessSerializer(many=True) - references = AdvisoryReferenceSerializer(many=True) - severities = AdvisorySeveritySerializer(many=True) - advisory_id = serializers.CharField(source="avid", read_only=True) - related_ssvc_trees = serializers.SerializerMethodField() - - def get_related_ssvc_trees(self, obj): - related_ssvcs = obj.related_ssvcs.all().select_related("source_advisory") - source_ssvcs = obj.source_ssvcs.all().select_related("source_advisory") - - seen = set() - result = [] - - for ssvc in list(related_ssvcs) + list(source_ssvcs): - key = (ssvc.vector, ssvc.source_advisory_id) - if key in seen: - continue - seen.add(key) - - result.append( - { - "vector": ssvc.vector, - "decision": ssvc.decision, - "options": ssvc.options, - "source_url": ssvc.source_advisory.url, - } - ) - - return result - - class Meta: - model = AdvisoryV2 - fields = [ - "advisory_id", - "url", - "aliases", - "summary", - "severities", - "weaknesses", - "references", - "exploitability", - "weighted_severity", - "risk_score", - "related_ssvc_trees", - ] - - def get_aliases(self, obj): - return [alias.alias for alias in obj.aliases.all()] - - class VulnerabilityListSerializer(serializers.ModelSerializer): url = serializers.SerializerMethodField() @@ -333,107 +242,6 @@ def get_fixing_vulnerabilities(self, obj): return [vuln.vulnerability_id for vuln in obj.fixing_vulnerabilities.all()] -class PackageV3Serializer(serializers.ModelSerializer): - purl = serializers.CharField(source="package_url") - risk_score = serializers.FloatField(read_only=True) - affected_by_vulnerabilities = serializers.SerializerMethodField() - fixing_vulnerabilities = serializers.SerializerMethodField() - next_non_vulnerable_version = serializers.SerializerMethodField() - latest_non_vulnerable_version = serializers.SerializerMethodField() - - class Meta: - model = Package - fields = [ - "purl", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", - "next_non_vulnerable_version", - "latest_non_vulnerable_version", - "risk_score", - ] - - def get_affected_by_vulnerabilities(self, package): - """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" - impacts = package.affected_in_impacts.select_related("advisory").prefetch_related( - "fixed_by_packages" - ) - - avids = {impact.advisory.avid for impact in impacts if impact.advisory_id} - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_by_avid = {adv.avid: adv for adv in latest_advisories} - impact_by_avid = {} - - advisories = [] - for impact in impacts: - avid = impact.advisory.avid - advisory = advisory_by_avid.get(avid) - if not advisory: - continue - advisories.append(advisory) - impact_by_avid[avid] = impact - - grouped_advisories = group_advisories_by_content(advisories=advisories) - - advs = [] - - for hash in grouped_advisories: - advs.append(grouped_advisories[hash]) - - result = [] - - for advisory in advs: - primary_advisory = advisory["primary"] - avid = primary_advisory.avid - impact = impact_by_avid.get(avid) - if not impact: - continue - result.append( - { - "advisory_id": primary_advisory.avid, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], - "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]], - } - ) - - return result - - def get_fixing_vulnerabilities(self, package): - impacts = package.fixed_in_impacts.select_related("advisory") - - avids = {impact.advisory.avid for impact in impacts if impact.advisory_id} - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - - grouped_advisories = group_advisories_by_content(advisories=latest_advisories) - - advs = [] - - for hash in grouped_advisories: - advs.append(grouped_advisories[hash]) - - result = [] - - for advisory in advs: - primary_advisory = advisory["primary"] - result.append( - { - "advisory_id": primary_advisory.avid, - "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]], - } - ) - - return result - - def get_next_non_vulnerable_version(self, package): - if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: - return next_non_vulnerable.version - - def get_latest_non_vulnerable_version(self, package): - if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: - return latest_non_vulnerable.version - - class PackageurlListSerializer(serializers.Serializer): purls = serializers.ListField( child=serializers.CharField(), @@ -462,27 +270,6 @@ class PackageV2FilterSet(filters.FilterSet): purl = filters.CharFilter(field_name="package_url") -class AdvisoryPackageV2FilterSet(filters.FilterSet): - affected_by_advisory = filters.CharFilter( - field_name="affected_in_impacts__advisory__avid", - label="Affected By Advisory ID", - help_text="Filter packages affected by a specific Advisory ID.", - ) - - fixing_advisory = filters.CharFilter( - field_name="fixed_in_impacts__advisory__avid", - label="Fixed By Advisory ID", - help_text="Filter packages fixed by a specific Advisory ID.", - ) - - purls = CharInFilter( - field_name="package_url", - lookup_expr="in", - label="Package URL", - help_text="Filter by one or more Package URLs. Multi-value supported (comma-separated).", - ) - - class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): queryset = Package.objects.all().prefetch_related( Prefetch( @@ -1062,339 +849,3 @@ def get_view_name(self): if self.detail: return "Pipeline Instance" return "Pipeline Jobs" - - -class PackageV3ViewSet(viewsets.ReadOnlyModelViewSet): - queryset = PackageV2.objects.all() - serializer_class = PackageV3Serializer - filter_backends = [filters.DjangoFilterBackend] - filterset_class = AdvisoryPackageV2FilterSet - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_queryset(self): - return ( - super() - .get_queryset() - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - - def list(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - page = self.paginate_queryset(queryset) - - packages = page if page is not None else queryset - - avids = set() - - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - - advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in latest_advisories} - - serializer = self.get_serializer(packages, many=True) - - if page is not None: - return self.get_paginated_response( - { - "packages": serializer.data, - "advisories_by_id": advisory_data, - } - ) - - return Response( - { - "packages": serializer.data, - "advisories_by_id": advisory_data, - } - ) - - @extend_schema( - request=PackageurlListSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageurlListSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_lookup(self, request): - """ - Return the response for exact PackageURLs requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - - purls = serializer.validated_data.get("purls") - - packages = ( - PackageV2.objects.for_purls(purls) - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages" - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - - avids = set() - - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } - - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) - - @extend_schema( - request=PackageBulkSearchRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageBulkSearchRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_search(self, request): - """ - Lookup for vulnerable packages using many Package URLs at once. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - - validated_data = serializer.validated_data - purls = validated_data.get("purls") - purl_only = validated_data.get("purl_only", False) - plain_purl = validated_data.get("plain_purl", False) - - if plain_purl: - purl_objects = [PackageURL.from_string(purl) for purl in purls] - plain_purl_objects = [ - PackageURL( - type=purl.type, - namespace=purl.namespace, - name=purl.name, - version=purl.version, - ) - for purl in purl_objects - ] - plain_purls = [str(purl) for purl in plain_purl_objects] - - query = ( - PackageV2.objects.filter(plain_package_url__in=plain_purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related( - "advisory" - ).prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - - packages = query - - avids = set() - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } - - if not purl_only: - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) - - # Using order by and distinct because there will be - # many fully qualified purl for a single plain purl - vulnerable_purls = query.vulnerable().only("plain_package_url") - vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - query = ( - PackageV2.objects.filter(package_url__in=purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - packages = query - - avids = set() - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } - - if not purl_only: - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) - - vulnerable_purls = query.vulnerable().only("package_url") - vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - @action(detail=False, methods=["get"]) - def all(self, request): - """ - Return a list of Package URLs of vulnerable packages. - """ - vulnerable_purls = ( - PackageV2.objects.vulnerable() - .only("package_url") - .order_by("package_url") - .distinct() - .values_list("package_url", flat=True) - ) - return Response(vulnerable_purls) - - @extend_schema( - request=LookupRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=LookupRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def lookup(self, request): - """ - Return the response for exact PackageURL requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A 'purl' is required.", - }, - ) - validated_data = serializer.validated_data - purl = validated_data.get("purl") - - qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() - return Response(PackageV3Serializer(qs, many=True, context={"request": request}).data) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py new file mode 100644 index 000000000..fb9847a1b --- /dev/null +++ b/vulnerabilities/api_v3.py @@ -0,0 +1,547 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from urllib.parse import urlencode + +from django.db.models import Exists +from django.db.models import OuterRef +from django.db.models import Prefetch +from django_filters import rest_framework as filters +from packageurl import PackageURL +from rest_framework import serializers +from rest_framework import viewsets +from rest_framework.reverse import reverse +from rest_framework.throttling import AnonRateThrottle + +from vulnerabilities.models import AdvisoryReference +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import AdvisoryWeakness +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import PackageV2 +from vulnerabilities.throttling import PermissionBasedUserRateThrottle +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import merge_and_save_grouped_advisories + + +class PackageQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + ) + details = serializers.BooleanField(default=False) + approximate = serializers.BooleanField(default=False) + + def validate(self, data): + if not data["purls"]: + if data["details"] or data["approximate"]: + raise serializers.ValidationError( + "details and approximate must be false when purls is empty" + ) + return data + + +class AdvisoryQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + ) + + def validate(self, data): + if not data["purls"]: + raise serializers.ValidationError("purls is required") + return data + + +class AdvisoryReferenceSerializer(serializers.ModelSerializer): + url = serializers.CharField() + reference_type = serializers.CharField() + reference_id = serializers.CharField() + + class Meta: + model = AdvisoryReference + fields = ["url", "reference_type", "reference_id"] + + +class AdvisorySeveritySerializer(serializers.ModelSerializer): + class Meta: + model = AdvisorySeverity + fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] + + def to_representation(self, instance): + data = super().to_representation(instance) + published_at = data.get("published_at", None) + if not published_at: + data.pop("published_at") + return data + + +class AdvisoryWeaknessSerializer(serializers.ModelSerializer): + cwe_id = serializers.CharField() + name = serializers.CharField() + description = serializers.CharField() + + class Meta: + model = AdvisoryWeakness + fields = ["cwe_id", "name", "description"] + + +class AdvisoryV3Serializer(serializers.ModelSerializer): + aliases = serializers.SlugRelatedField( + many=True, + read_only=True, + slug_field="alias", + ) + weaknesses = AdvisoryWeaknessSerializer(many=True) + references = AdvisoryReferenceSerializer(many=True) + severities = AdvisorySeveritySerializer(many=True) + advisory_id = serializers.CharField(source="avid", read_only=True) + related_ssvc_trees = serializers.SerializerMethodField() + + def get_related_ssvc_trees(self, obj): + seen = set() + result = [] + + all_ssvcs = list(obj.related_ssvcs.all()) + list(obj.source_ssvcs.all()) + + for ssvc in all_ssvcs: + key = (ssvc.vector, ssvc.source_advisory_id) + if key in seen: + continue + seen.add(key) + + result.append( + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + ) + + return result + + class Meta: + model = AdvisoryV2 + fields = [ + "advisory_id", + "url", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + "related_ssvc_trees", + ] + + def get_aliases(self, obj): + return [alias.alias for alias in obj.aliases.all()] + + +class PackageV3Serializer(serializers.ModelSerializer): + purl = serializers.CharField(source="package_url") + risk_score = serializers.FloatField(read_only=True) + affected_by_vulnerabilities = serializers.SerializerMethodField() + affected_by_vulnerabilities_url = serializers.SerializerMethodField() + fixing_vulnerabilities = serializers.SerializerMethodField() + fixing_vulnerabilities_url = serializers.SerializerMethodField() + next_non_vulnerable_version = serializers.SerializerMethodField() + latest_non_vulnerable_version = serializers.SerializerMethodField() + + class Meta: + model = PackageV2 + fields = [ + "purl", + "affected_by_vulnerabilities", + "affected_by_vulnerabilities_url", + "fixing_vulnerabilities", + "fixing_vulnerabilities_url", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", + "risk_score", + ] + + def to_representation(self, instance): + data = super().to_representation(instance) + + if data.get("affected_by_vulnerabilities") is None: + data.pop("affected_by_vulnerabilities", None) + else: + data.pop("affected_by_vulnerabilities_url", None) + + if data.get("fixing_vulnerabilities") is None: + data.pop("fixing_vulnerabilities", None) + else: + data.pop("fixing_vulnerabilities_url", None) + + return data + + def get_affected_by_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("affected-by-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + + def get_fixing_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("fixing-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + + def get_affected_by_vulnerabilities(self, package): + """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + + advisories = [] + + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_ids = advisories_qs.only("id") + + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: + return None + + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} + avids = advisory_by_avid.keys() + + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + result = [] + + for advisory in advisories_qs: + impact = impact_by_avid.get(advisory.avid) + if not impact: + continue + + result.append( + { + "advisory_id": advisory.advisory_id.split("/")[-1], + "aliases": [alias.alias for alias in advisory.aliases.all()], + "summary": advisory.summary, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "severity": advisory.weighted_severity, + "exploitability": advisory.exploitability, + "risk_score": advisory.risk_score, + } + ) + + return result + + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() + + if is_grouped: + affected_by_advisories_qs = AdvisorySet.objects.filter( + package=package, relation_type="affecting" + ).select_related("primary_advisory") + + affected_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") + for adv in affected_by_advisories_qs + ] + + advisories = get_advisories_from_groups(affected_groups) + return self.return_advisories_data(package, advisories_qs, advisories) + + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = advisories_qs.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + advisories = merge_and_save_grouped_advisories(package, advisories_qs, "affecting") + return self.return_advisories_data(package, advisories_qs, advisories) + + def get_fixing_vulnerabilities(self, package): + advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) + + if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_ids = advisories_qs.only("id") + + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: + return None + + results = [] + + for advisory in advisories_qs: + results.append( + { + "advisory_id": advisory.advisory_id.split("/")[-1], + } + ) + return results + + advisories = [] + + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() + + if is_grouped: + fixing_advisories_qs = AdvisorySet.objects.filter( + package=package, relation_type="fixing" + ).select_related("primary_advisory") + + fixing_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ] + + advisories = get_advisories_from_groups(fixing_groups) + return self.return_fixing_advisories_data(advisories) + + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = advisories_qs.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + advisories = merge_and_save_grouped_advisories(package, advisories_qs, "fixing") + return self.return_fixing_advisories_data(advisories) + + def return_fixing_advisories_data(self, advisories): + result = [] + for advisory in advisories: + result.append( + { + "advisory_id": advisory["identifier"], + } + ) + + return result + + def return_advisories_data(self, package, advisories_qs, advisories): + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} + avids = advisory_by_avid.keys() + + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + result = [] + for advisory in advisories: + impact = impact_by_avid.get(advisory["advisory"].avid) + if not impact: + continue + + result.append( + { + "advisory_id": advisory["identifier"], + "aliases": [alias.alias for alias in advisory["aliases"]], + "summary": advisory["advisory"].summary, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + } + ) + + return result + + def get_next_non_vulnerable_version(self, package): + if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: + return next_non_vulnerable.version + + def get_latest_non_vulnerable_version(self, package): + if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: + return latest_non_vulnerable.version + + +class PackageV3ViewSet(viewsets.GenericViewSet): + queryset = PackageV2.objects.all() + serializer_class = PackageV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + purls = serializer.validated_data["purls"] + details = serializer.validated_data["details"] + approximate = serializer.validated_data["approximate"] + + if not purls: + impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) + + query = ( + PackageV2.objects.annotate(has_vuln=Exists(impacted)) + .filter(has_vuln=True) + .values_list("package_url", flat=True) + .order_by("package_url") + ) + page = self.paginate_queryset(query) + return self.get_paginated_response(page) + + plain_purls = None + + if approximate: + plain_purls = [ + str( + PackageURL( + type=p.type, + namespace=p.namespace, + name=p.name, + version=p.version, + ) + ) + for p in map(PackageURL.from_string, purls) + ] + + if not details: + if approximate: + query = ( + PackageV2.objects.filter(plain_package_url__in=plain_purls) + .values_list("plain_package_url", flat=True) + .distinct() + .order_by("plain_package_url") + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .distinct() + .order_by("package_url") + .values_list("package_url", flat=True) + ) + + page = self.paginate_queryset(query) + return self.get_paginated_response(page) + + if approximate: + query = ( + PackageV2.objects.filter(plain_package_url__in=plain_purls) + .order_by("plain_package_url") + .distinct("plain_package_url") + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .order_by("package_url") + .distinct("package_url") + ) + + page = self.paginate_queryset(query) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) + + +class AffectedByAdvisoryV3Serializer(AdvisoryV3Serializer): + fixed_by_packages = serializers.SerializerMethodField() + + def get_fixed_by_packages(self, obj): + return list( + obj.impacted_packages.values_list("fixed_by_packages__package_url", flat=True) + .exclude(fixed_by_packages__package_url__isnull=True) + .distinct() + ) + + class Meta: + model = AdvisoryV2 + fields = [ + "advisory_id", + "url", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + "related_ssvc_trees", + "fixed_by_packages", + ] + + +class AdvisoryV3ViewSet(viewsets.GenericViewSet): + queryset = AdvisoryV2.objects.all() + serializer_class = AdvisoryV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + purls = serializer.validated_data["purls"] + + latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls( + purls=purls + ).prefetch_related( + Prefetch( + "references", + queryset=AdvisoryReference.objects.only( + "id", + "url", + "reference_type", + "reference_id", + ), + ), + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only( + "id", + "url", + "value", + "scoring_system", + "scoring_elements", + "published_at", + ), + ), + "weaknesses", + "aliases", + "related_ssvcs__source_advisory", + "source_ssvcs__source_advisory", + ) + + page = self.paginate_queryset(latest_advisories) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) + + +class PackageAdvisoriesViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = AdvisoryV3Serializer + relation = None + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def get_queryset(self): + purl = self.request.query_params.get("purl") + + if not purl: + return AdvisoryV2.objects.none() + + return AdvisoryV2.objects.filter(**{self.relation: purl}).latest_per_avid() + + +class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__fixed_by_packages__package_url" + + +class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__affecting_packages__package_url" + serializer_class = AffectedByAdvisoryV3Serializer diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 982b4bbd8..3e991d658 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -20,7 +20,6 @@ from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees -from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -32,6 +31,7 @@ enhance_with_metasploit as enhance_with_metasploit_v2, ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 +from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 from vulnerabilities.utils import create_registry @@ -75,6 +75,6 @@ compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, - compute_advisory_content_hash.ComputeAdvisoryContentHash, + group_advisories_for_packages.GroupAdvisoriesForPackages, ] ) diff --git a/vulnerabilities/migrations/0117_advisoryv2_risk_score.py b/vulnerabilities/migrations/0117_advisoryv2_risk_score.py new file mode 100644 index 000000000..47733da5e --- /dev/null +++ b/vulnerabilities/migrations/0117_advisoryv2_risk_score.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.11 on 2026-03-17 09:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0116_advisoryv2_advisory_content_hash"), + ] + + operations = [ + migrations.AddField( + model_name="advisoryv2", + name="risk_score", + field=models.DecimalField( + blank=True, + decimal_places=1, + help_text="Risk expressed as a number ranging from 0 to 10. Risk is calculated from weighted severity and exploitability values. It is the maximum value of (the weighted severity multiplied by its exploitability) or 10. Risk = min(weighted severity * exploitability, 10)", + max_digits=3, + null=True, + ), + ), + ] diff --git a/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py b/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py new file mode 100644 index 000000000..467d7b36c --- /dev/null +++ b/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py @@ -0,0 +1,71 @@ +# Generated by Django 5.2.11 on 2026-03-25 10:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0117_advisoryv2_risk_score"), + ] + + operations = [ + migrations.CreateModel( + name="AdvisorySet", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "relation_type", + models.CharField( + choices=[("affecting", "Affecting"), ("fixing", "Fixing")], max_length=20 + ), + ), + ("identifiers", models.JSONField()), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.packagev2" + ), + ), + ( + "primary_advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="vulnerabilities.advisoryv2" + ), + ), + ], + ), + migrations.CreateModel( + name="AdvisorySetMember", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("is_primary", models.BooleanField(default=False)), + ( + "advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.advisoryv2" + ), + ), + ( + "advisory_set", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="members", + to="vulnerabilities.advisoryset", + ), + ), + ], + ), + ] diff --git a/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py b/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py new file mode 100644 index 000000000..503e14f8d --- /dev/null +++ b/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.11 on 2026-03-30 08:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0118_advisoryset_advisorysetmember"), + ] + + operations = [ + migrations.RemoveField( + model_name="advisoryset", + name="identifiers", + ), + migrations.RemoveField( + model_name="advisoryv2", + name="advisory_content_hash", + ), + migrations.AddField( + model_name="advisoryset", + name="aliases", + field=models.ManyToManyField( + help_text="A list of serializable Alias objects", + related_name="advisory_sets", + to="vulnerabilities.advisoryalias", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index d1c88f285..b8a16a686 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2862,19 +2862,114 @@ def latest_for_avid(self, avid: str): ) def latest_per_avid(self): - latest_ids = ( - self.filter(avid=OuterRef("avid")) - .order_by( - F("date_collected").desc(nulls_last=True), - "-id", + return self.order_by( + "avid", + F("date_collected").desc(nulls_last=True), + "-id", + ).distinct("avid") + + def latest_for_avids(self, avids): + return self.filter(avid__in=avids).latest_per_avid() + + def latest_affecting_advisories_for_purl(self, purl): + adv_ids = ImpactedPackageAffecting.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, + ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + + def latest_affecting_advisories_for_purls(self, purls): + adv_ids = ImpactedPackageAffecting.objects.filter( + package__package_url__in=purls + ).values_list( + "impacted_package__advisory_id", + flat=True, + ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + + def latest_fixed_by_advisories_for_purl(self, purl): + adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, + ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + + def latest_fixed_by_advisories_for_purls(self, purls): + adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( + "impacted_package__advisory_id", + flat=True, + ) + + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + + def latest_advisories_for_purls(self, purls): + adv_ids = ( + ImpactedPackageAffecting.objects.filter(package__package_url__in=purls) + .values_list( + "impacted_package__advisory_id", + flat=True, + ) + .union( + ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( + "impacted_package__advisory_id", + flat=True, + ) ) - .values("id")[:1] ) - return self.filter(id=Subquery(latest_ids)) + qs = self.filter(id__in=Subquery(adv_ids)) + return qs.latest_per_avid() - def latest_for_avids(self, avids): - return self.filter(avid__in=avids).latest_per_avid() + def latest_advisories_for_purl(self, purl): + adv_ids = ( + ImpactedPackageAffecting.objects.filter(package__package_url=purl) + .values_list( + "impacted_package__advisory_id", + flat=True, + ) + .union( + ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, + ) + ) + ) + + qs = self.filter(id__in=Subquery(adv_ids)) + return qs.latest_per_avid() + + +class AdvisorySet(models.Model): + + RELATION_TYPE_CHOICES = [ + ("affecting", "Affecting"), + ("fixing", "Fixing"), + ] + + package = models.ForeignKey("PackageV2", on_delete=models.CASCADE) + relation_type = models.CharField(max_length=20, choices=RELATION_TYPE_CHOICES) + + aliases = models.ManyToManyField( + AdvisoryAlias, + related_name="advisory_sets", + help_text="A list of serializable Alias objects", + ) + + primary_advisory = models.ForeignKey("AdvisoryV2", on_delete=models.PROTECT) + + created_at = models.DateTimeField(auto_now_add=True) + + +class AdvisorySetMember(models.Model): + + advisory_set = models.ForeignKey( + AdvisorySet, + on_delete=models.CASCADE, + related_name="members", + ) + + advisory = models.ForeignKey("AdvisoryV2", on_delete=models.CASCADE) + is_primary = models.BooleanField(default=False) class AdvisoryV2(models.Model): @@ -3010,25 +3105,14 @@ class AdvisoryV2(models.Model): help_text="Related advisories that are used to calculate the severity of this advisory.", ) - advisory_content_hash = models.CharField( - max_length=64, - blank=True, + risk_score = models.DecimalField( null=True, - help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.", + blank=True, + max_digits=3, + decimal_places=1, + help_text="Risk expressed as a number ranging from 0 to 10. Risk is calculated from weighted severity and exploitability values. It is the maximum value of (the weighted severity multiplied by its exploitability) or 10. Risk = min(weighted severity * exploitability, 10)", ) - @property - def risk_score(self): - """ - Risk expressed as a number ranging from 0 to 10. - Risk is calculated from weighted severity and exploitability values. - It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 - Risk = min(weighted severity * exploitability, 10) - """ - if self.exploitability and self.weighted_severity: - risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) - return round(risk_score, 1) - objects = AdvisoryV2QuerySet.as_manager() class Meta: @@ -3045,6 +3129,9 @@ def save(self, *args, **kwargs): self.full_clean() return super().save(*args, **kwargs) + def __str__(self): + return self.avid + @property def get_status_label(self): label_by_status = {choice[0]: choice[1] for choice in VulnerabilityStatusType.choices} @@ -3221,7 +3308,7 @@ def search(self, query: str = None): except ValueError: # otherwise use query as a plain string qs = qs.filter(package_url__icontains=query) - return qs.order_by("package_url") + return qs.order_by("package_url").order_by("-version_rank") def with_vulnerability_counts(self): return self.annotate( @@ -3325,7 +3412,7 @@ def vulnerable(self): """ Return only packages that are vulnerable. """ - return self.filter(affected_in_impacts__isnull=False) + return self.filter(id__in=ImpactedPackageAffecting.objects.values("package_id").distinct()) def with_is_vulnerable(self): """ diff --git a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py index b0f005592..b666e48a2 100644 --- a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py @@ -105,7 +105,7 @@ def parse_advisory(self, record) -> AdvisoryDataV2: VulnerabilitySeverity( system=severity_systems.ARCHLINUX, value=severity, - url="https://security.archlinux.org/{avg_name}.json", + url=f"https://security.archlinux.org/{avg_name}.json", ) ] diff --git a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py index cfe92d93f..33acaf7f8 100644 --- a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py +++ b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py @@ -31,7 +31,7 @@ class GithubOSVImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" repo_url = "git+https://github.com/github/advisory-database/" - precedence = 100 + precedence = 200 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/pypa_importer.py b/vulnerabilities/pipelines/v2_importers/pypa_importer.py index 90599e99d..7a80ed70f 100644 --- a/vulnerabilities/pipelines/v2_importers/pypa_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pypa_importer.py @@ -29,7 +29,7 @@ class PyPaImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" repo_url = "git+https://github.com/pypa/advisory-database" - precedence = 200 + precedence = 500 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/pysec_importer.py b/vulnerabilities/pipelines/v2_importers/pysec_importer.py index 05614b961..e9225a4f5 100644 --- a/vulnerabilities/pipelines/v2_importers/pysec_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pysec_importer.py @@ -29,7 +29,7 @@ class PyPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" spdx_license_expression = "CC-BY-4.0" - precedence = 100 + precedence = 300 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py index 2b82b667c..213904815 100644 --- a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py +++ b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py @@ -36,7 +36,8 @@ def steps(cls): def collect_ssvc_data(self): vulnrichment_advisories = ( - AdvisoryV2.objects.filter( + AdvisoryV2.objects.latest_per_avid() + .filter( severities__scoring_system=SCORING_SYSTEMS["ssvc"], ) .distinct() diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py deleted file mode 100644 index fe5a3c97e..000000000 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.utils import compute_advisory_content - - -class ComputeAdvisoryContentHash(VulnerableCodePipeline): - """Compute Advisory Content Hash for Advisory.""" - - pipeline_id = "compute_advisory_content_hash_v2" - - @classmethod - def steps(cls): - return (cls.compute_advisory_content_hash,) - - def compute_advisory_content_hash(self): - """Compute Advisory Content Hash for Advisory.""" - - advisories = AdvisoryV2.objects.filter(advisory_content_hash__isnull=True) - - advisories_count = advisories.count() - - progress = LoopProgress( - total_iterations=advisories_count, - logger=self.log, - progress_step=1, - ) - - to_update = [] - batch_size = 5000 - - for advisory in progress.iter(advisories.iterator(chunk_size=batch_size)): - advisory.advisory_content_hash = compute_advisory_content(advisory) - to_update.append(advisory) - - if len(to_update) >= batch_size: - AdvisoryV2.objects.bulk_update( - to_update, - ["advisory_content_hash"], - batch_size=batch_size, - ) - to_update.clear() - - if to_update: - AdvisoryV2.objects.bulk_update( - to_update, - ["advisory_content_hash"], - batch_size=batch_size, - ) - - self.log("Finished computing advisory_content_hash") diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index 9caaaeb95..dacf7e6c8 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -7,14 +7,15 @@ # See https://aboutcode.org for more information about nexB OSS projects. # from aboutcode.pipeline import LoopProgress +from django.db.models import Max from django.db.models import Prefetch -from django.db.models import Q +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.risk import compute_package_risk_v2 from vulnerabilities.risk import compute_vulnerability_risk_factors @@ -36,104 +37,146 @@ def steps(cls): ) def compute_and_store_vulnerability_risk_score(self): + affected_advisories = ( - AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False) + AdvisoryV2.objects.latest_per_avid() + .filter(impacted_packages__affecting_packages__isnull=False) + .only("id") .prefetch_related( - "references", - "severities", - "exploits", + Prefetch( + "references", queryset=AdvisoryReference.objects.only("id", "reference_type") + ), + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only("id", "value", "url", "scoring_system"), + ), + Prefetch("exploits", queryset=AdvisoryExploit.objects.only("id")), Prefetch( "related_advisory_severities", - queryset=AdvisoryV2.objects.prefetch_related("severities"), + queryset=AdvisoryV2.objects.only("id").prefetch_related( + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only( + "id", "value", "url", "scoring_system" + ), + ) + ), ), ) .distinct() ) + estimated_vulnerability_count = affected_advisories.count() + self.log( - f"Calculating risk for {affected_advisories.count():,d} advisory with a affected packages records" + f"Calculating risk for {estimated_vulnerability_count:,d} advisory with a affected packages records" ) - progress = LoopProgress(total_iterations=affected_advisories.count(), logger=self.log) + progress = LoopProgress( + logger=self.log, total_iterations=estimated_vulnerability_count, progress_step=5 + ) updatables = [] updated_vulnerability_count = 0 batch_size = 5000 for advisory in progress.iter(affected_advisories.iterator(chunk_size=batch_size)): + references = advisory.references.all() exploits = advisory.exploits.all() - severities = AdvisorySeverity.objects.filter( - Q(advisories=advisory) | Q(advisories__related_to_advisory_severities=advisory) - ).distinct() + severities = list(advisory.severities.all()) - weighted_severity, exploitability = compute_vulnerability_risk_factors( - references=references, - severities=severities, - exploits=exploits, - ) - advisory.weighted_severity = weighted_severity - advisory.exploitability = exploitability - updatables.append(advisory) + for rel in advisory.related_advisory_severities.all(): + severities.extend(rel.severities.all()) + + try: + weighted_severity, exploitability = compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + + advisory.weighted_severity = weighted_severity + advisory.exploitability = exploitability + if advisory.exploitability and advisory.weighted_severity: + risk_score = min( + float(advisory.exploitability * advisory.weighted_severity), 10.0 + ) + advisory.risk_score = round(risk_score, 1) + updatables.append(advisory) + except Exception as e: + self.log(f"Error computing risk score for advisory {advisory.advisory_id}: {e}") if len(updatables) >= batch_size: updated_vulnerability_count += bulk_update( model=AdvisoryV2, items=updatables, - fields=["weighted_severity", "exploitability"], + fields=["weighted_severity", "exploitability", "risk_score"], logger=self.log, ) - - updated_vulnerability_count += bulk_update( - model=AdvisoryV2, - items=updatables, - fields=["weighted_severity", "exploitability"], - logger=self.log, - ) + updatables.clear() + + if updatables: + updated_vulnerability_count += bulk_update( + model=AdvisoryV2, + items=updatables, + fields=["weighted_severity", "exploitability", "risk_score"], + logger=self.log, + ) self.log( f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" ) def compute_and_store_package_risk_score(self): - affected_packages = (PackageV2.objects.filter(affected_in_impacts__isnull=False)).distinct() - self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") + latest_advisories = AdvisoryV2.objects.latest_per_avid() + + qs = ( + PackageV2.objects.filter( + affected_in_impacts__advisory__risk_score__isnull=False, + affected_in_impacts__advisory__in=latest_advisories, + ) + .annotate(computed_risk=Max("affected_in_impacts__advisory__risk_score")) + .only("id") + ) + + estimated = qs.count() progress = LoopProgress( - total_iterations=affected_packages.count(), + total_iterations=estimated, logger=self.log, progress_step=5, ) - updatables = [] - updated_package_count = 0 - batch_size = 10000 + self.log(f"Computing risk for {estimated:,d} packages") - for package in progress.iter(affected_packages.iterator(chunk_size=batch_size)): - risk_score = compute_package_risk_v2(package) + batch = [] + batch_size = 5000 + updated = 0 - if not risk_score: - continue + for pkg in progress.iter(qs.iterator(chunk_size=batch_size)): - package.risk_score = risk_score - updatables.append(package) + pkg.risk_score = round(float(pkg.computed_risk), 1) + batch.append(pkg) - if len(updatables) >= batch_size: - updated_package_count += bulk_update( + if len(batch) >= batch_size: + updated += bulk_update( model=PackageV2, - items=updatables, + items=batch, fields=["risk_score"], logger=self.log, ) - updated_package_count += bulk_update( + batch.clear() + + updated += bulk_update( model=PackageV2, - items=updatables, + items=batch, fields=["risk_score"], logger=self.log, ) - self.log(f"Successfully added risk score for {updated_package_count:,d} package") + self.log(f"Successfully added risk score for {updated:,d} package") def bulk_update(model, items, fields, logger): diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py index c306502d8..70afa4ef1 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py @@ -89,7 +89,7 @@ def add_vulnerability_exploit(row, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=raw_alias) + advs = AdvisoryV2.objects.filter(advisory_id=raw_alias).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py index 486d79232..5274378da 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py @@ -78,7 +78,7 @@ def add_vulnerability_exploit(kev_vul, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=cve_id) + advs = AdvisoryV2.objects.filter(advisory_id=cve_id).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py index fbfea5150..3ce1ff7c9 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py @@ -83,7 +83,7 @@ def add_advisory_exploit(record, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=ref) + advs = AdvisoryV2.objects.filter(advisory_id=ref).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py new file mode 100644 index 000000000..d2c8f6296 --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -0,0 +1,57 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import merge_advisories + + +class GroupAdvisoriesForPackages(VulnerableCodePipeline): + """Detect and flag packages that do not exist upstream.""" + + pipeline_id = "group_advisories_for_packages" + + @classmethod + def steps(cls): + return (cls.group_advisories_for_packages,) + + def group_advisories_for_packages(self): + group_advisoris_for_packages(logger=self.log) + + +def group_advisoris_for_packages(logger=None): + for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): + print(f"Grouping advisories for package {package.purl}") + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + try: + affected_groups = merge_advisories(affecting_advisories, package) + fixed_by_groups = merge_advisories(fixed_by_advisories, package) + delete_and_save_advisory_set(affected_groups, package, relation="affecting") + delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") + except Exception as e: + print(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + continue diff --git a/vulnerabilities/pipelines/v2_improvers/relate_severities.py b/vulnerabilities/pipelines/v2_improvers/relate_severities.py index 97a86404b..9ce3e0a30 100644 --- a/vulnerabilities/pipelines/v2_improvers/relate_severities.py +++ b/vulnerabilities/pipelines/v2_improvers/relate_severities.py @@ -61,8 +61,8 @@ def relate_severities(self): severity_score_advisories = ( AdvisoryV2.objects.filter(datasource_id__in=self.pipelines) .filter(severities__scoring_system__in=self.SUPPORTED_SYSTEMS) - .distinct() .latest_per_avid() + .distinct() ) total = severity_score_advisories.count() @@ -70,14 +70,21 @@ def relate_severities(self): advisory_id_map = {} - qs = AdvisoryV2.objects.filter( - advisory_id__in=severity_score_advisories.values("advisory_id") - ).values("id", "advisory_id") - - alias_qs = AdvisoryV2.objects.filter( - aliases__alias__in=severity_score_advisories.values("advisory_id") - ).values("id", "aliases__alias") + qs = ( + AdvisoryV2.objects.filter( + advisory_id__in=severity_score_advisories.values("advisory_id") + ) + .latest_per_avid() + .values("id", "advisory_id") + ) + alias_qs = ( + AdvisoryV2.objects.filter( + aliases__alias__in=severity_score_advisories.values("advisory_id") + ) + .latest_per_avid() + .values("id", "aliases__alias") + ) for row in qs: advisory_id_map.setdefault(row["advisory_id"], set()).add(row["id"]) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 00414a259..c812fdf86 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,7 +48,6 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 -from vulnerabilities.utils import compute_advisory_content def get_or_create_aliases(aliases: List) -> QuerySet: @@ -302,7 +301,6 @@ def insert_advisory_v2( advisory_obj = None created = False content_id = compute_content_id_v2(advisory_data=advisory) - advisory_content_hash = compute_advisory_content(advisory_data=advisory) try: default_data = { "datasource_id": pipeline_id, @@ -313,7 +311,6 @@ def insert_advisory_v2( "original_advisory_text": advisory.original_advisory_text, "url": advisory.url, "precedence": precedence, - "advisory_content_hash": advisory_content_hash, } advisory_obj, created = AdvisoryV2.objects.get_or_create( diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py new file mode 100644 index 000000000..d66365706 --- /dev/null +++ b/vulnerabilities/pipes/group_advisories.py @@ -0,0 +1,50 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db import transaction + + +@transaction.atomic +def delete_and_save_advisory_set(groups, package, relation=None): + from vulnerabilities.models import AdvisorySet + from vulnerabilities.models import AdvisorySetMember + + AdvisorySet.objects.filter(package=package, relation_type=relation).delete() + + membership_to_create = [] + + for identifiers, primary, secondary in groups: + + advisory_set = AdvisorySet.objects.create( + package=package, + relation_type=relation, + primary_advisory=primary, + ) + + advisory_set.aliases.add(*identifiers) + advisory_set.save() + + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=primary, + is_primary=True, + ) + ) + + for adv in secondary: + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=adv, + is_primary=False, + ) + ) + + AdvisorySetMember.objects.bulk_create(membership_to_create) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 0628422bb..dd7401d80 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -8,6 +8,9 @@ # from urllib.parse import urlparse +from django.db.models import Max + +from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import VulnerabilityReference from vulnerabilities.severity_systems import EPSS from vulnerabilities.weight_config import WEIGHT_CONFIG @@ -123,12 +126,14 @@ def compute_package_risk_v2(package): Calculate the risk for a package by iterating over all vulnerabilities that affects this package and determining the associated risk. """ - result = [] - for impact in package.affected_in_impacts.all(): - if risk := impact.advisory.risk_score: - result.append(float(risk)) - if not result: + max_risk = ( + AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.purl).aggregate( + max_risk=Max("risk_score") + ) + )["max_risk"] + + if max_risk is None: return - return round(max(result), 1) + return round(float(max_risk), 1) diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html index 595412df4..90f1d6d8b 100644 --- a/vulnerabilities/templates/advisory_detail.html +++ b/vulnerabilities/templates/advisory_detail.html @@ -137,7 +137,11 @@ applications, or networks. This metric is determined automatically based on the discovery of known exploits."> Exploitability - {{ advisory.exploitability }} + {% if advisory.exploitability is not None %} + {{ advisory.exploitability }} + {% else %} + {{ "" }} + {% endif %} @@ -146,7 +150,11 @@ data-tooltip="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10." >Weighted Severity - {{ advisory.weighted_severity }} + {% if advisory.weighted_severity is not None %} + {{ advisory.weighted_severity }} + {% else %} + {{ "" }} + {% endif %} @@ -157,7 +165,11 @@ " >Risk - {{ advisory.risk_score }} + {% if advisory.risk_score is not None %} + {{ advisory.risk_score }} + {% else %} + {{ "" }} + {% endif %} @@ -214,9 +226,20 @@ {% for severity in severities %} {{ severity.scoring_system }} - {{ severity.value }} - + + {% if severity.value is not None %} + {{ severity.value }} + {% else %} + {{ "" }} + {% endif %} + + + {% if severity.url is not None %} + {{ severity.url }} + {% else %} + {{ "" }} + {% endif %} {% empty %} @@ -483,7 +506,11 @@
{% for severity_vector in severity_vectors %} {% if severity_vector.vector.version == '2.0' %} - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} + Vector: {{ severity_vector.vector.vectorString }} + {% if severity_vector.origin %} + Found at + {{ severity_vector.origin }} + {% endif %} diff --git a/vulnerabilities/templates/affected_by_advisories.html b/vulnerabilities/templates/affected_by_advisories.html new file mode 100644 index 000000000..01721b84f --- /dev/null +++ b/vulnerabilities/templates/affected_by_advisories.html @@ -0,0 +1,116 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load url_filters %} +{% load utils %} + +{% block content %} +
+
+
+
+ {{ page_obj.paginator.count|intcomma }} results +
+ {% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+
+
+ +
+
+
Exploitability (E)
+ + + + + + + + + + + + {% for advisory in page_obj %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryFixed in package version
+ + {{advisory.avid }} + +
+ {% if advisory.alias|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.alias %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} + + {% if advisory.secondary|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary %} + + {{secondary.avid }} + + {% endfor %} + {% endif %} +
+ {{advisory.url}} + + {{advisory.date_published}} + + {{ advisory.summary }} + + {% with fixed=fixed_package_details|get_item:advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} +
+ This package is not known to be subject of any advisories. +
+
+ +{% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} +{% endif %} +{% endblock %} + diff --git a/vulnerabilities/templates/fixing_advisories.html b/vulnerabilities/templates/fixing_advisories.html new file mode 100644 index 000000000..64af4fc65 --- /dev/null +++ b/vulnerabilities/templates/fixing_advisories.html @@ -0,0 +1,76 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} + +{% block content %} +
+
+
+
+ {{ page_obj.paginator.count|intcomma }} results +
+ {% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+
+
+ +
+
+ + + + + + + + + + + + {% for advisory in page_obj %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryAliases
+ + {{advisory.avid }} + + + {{advisory.url}} + + {{advisory.date_published}} + + {{ advisory.summary }} + + {% for alias in advisory.alias %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ This package is not known to fix any advisories. +
+
+ +{% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} +{% endif %} +{% endblock %} +
diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 9cc9ea343..a6c07c352 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -45,7 +45,7 @@
- {% if affected_by_advisories_v2|length != 0 %} + {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %}
{% else %}
@@ -82,7 +82,7 @@
- {% if affected_by_advisories_v2|length != 0 %} + {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %}
@@ -118,7 +118,11 @@ Risk score @@ -128,34 +132,33 @@ {% endif %}
+ {% if affected_by_advisories_v2|length != 0 %}
Vulnerabilities affecting this package ({{ affected_by_advisories_v2|length }})
-
+ {% if package.risk_score is not None %} {{package.risk_score}} + {% else %} + {{""}} + {% endif %}
- - + {% if grouped %} {% for advisory in affected_by_advisories_v2 %} - + + {% empty %} + + + + {% endfor %} + + {% else %} + + {% for advisory in affected_by_advisories_v2 %} + + {% endfor %} + {% endif %}
AdvisorySourceDate Published Summary Fixed in package version
- - {{advisory.primary.avid }} + + {{advisory.identifier }}
- {% if advisory.primary.alias|length != 0 %} + {% if advisory.aliases|length != 0 %} Aliases: {% endif %}
- {% for alias in advisory.primary.alias %} + {% for alias in advisory.aliases %} {% if alias.url %} {{ alias }} @@ -166,35 +169,88 @@ {% endif %} {% endfor %} - {% if advisory.secondary|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary %} - - {{secondary.avid }} - - {% endfor %} - {% endif %}
- {{advisory.primary.url}} + {{ advisory.advisory.summary|truncatewords:20 }} - {{advisory.primary.date_published}} + + {% with fixed=fixed_package_details|get_item:advisory.advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} +
+ This package is not known to be subject of any advisories. +
+ + {{advisory.advisory_id }} + +
+ {% if advisory.aliases.all|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.aliases.all %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
- {{ advisory.primary.summary }} + {{ advisory.summary|truncatewords:20 }} - {% with fixed=fixed_package_details|get_item:advisory.primary.avid %} + {% with fixed=fixed_package_details|get_item:advisory.avid %} {% if fixed %} {% for item in fixed %}
{{ item.pkg.version }}
- - Subject of {{ item.affected_count }} other advisories. + {% if item.pkg.is_vulnerable %} + + Vulnerable + {% else %} + + Not vulnerable + + {% endif %}
{% endfor %} {% else %} @@ -211,10 +267,22 @@
+ {% elif affected_by_advisories_v2_url %} +
+ This package is subject to more than 100 advisories. Please refer to the following + URL for vulnerabilities affecting this package: Advisories +
+ {% else %} +
+ This package is not known to be subject of any advisories. +
+ {% endif %}
+ {% if fixing_advisories_v2|length != 0 %}
Vulnerabilities fixed by this package ({{ fixing_advisories_v2|length }})
@@ -223,40 +291,61 @@ Advisory - Source - Date Published Summary Aliases + + {% if grouped %} {% for advisory in fixing_advisories_v2 %} - - {{advisory.primary.avid }} + + {{advisory.identifier }}
- {% if advisory.secondary|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary %} - - {{secondary.avid }} - - {% endfor %} - {% endif %} - {{advisory.primary.url}} + {{ advisory.advisory.summary|truncatewords:20 }} - {{advisory.primary.date_published}} + {% for alias in advisory.aliases %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} + + + {% empty %} + + + This package is not known to fix any advisories. + + + {% endfor %} + + + {% else %} + + {% for advisory in fixing_advisories_v2 %} + + + + {{advisory.advisory_id }} + +
- {{ advisory.primary.summary }} + {{ advisory.summary|truncatewords:20 }} - {% for alias in advisory.primary.alias %} + {% for alias in advisory.aliases.all %} {% if alias.url %} {{ alias }} @@ -277,8 +366,18 @@ {% endfor %} - + {% endif %}
+ {% elif fixing_advisories_v2_url %} +
+ This package is known to fix more than 100 advisories. Please refer to the following + URL for vulnerabilities fixed by this package: Advisories +
+ {% else %} +
+ This package is not known to fix any advisories. +
+ {% endif %}
diff --git a/vulnerabilities/templates/package_details_v3.html b/vulnerabilities/templates/package_details_v3.html new file mode 100644 index 000000000..44ec1c297 --- /dev/null +++ b/vulnerabilities/templates/package_details_v3.html @@ -0,0 +1,367 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load url_filters %} +{% load utils %} + +{% block title %} +VulnerableCode Package Details - {{ package.purl }} +{% endblock %} + +{% block content %} +
+ {% include "package_search_box_v2.html"%} +
+ +{% if package %} +
+
+
+
+ Package details: + {{ package.purl }} + +
+
+ +
+ +
+ +
+
+
+ {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} +
+ {% else %} +
+ {% endif %} + + + + + + + {% if package.is_ghost %} + + + + + {% endif %} + +
+ + purl + + + {{ package.purl }} +
+ Tags + + + Ghost + +
+
+ {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} + +
+ + + + + + + + + + + + + + + +
+ Next non-vulnerable version + + {% if next_non_vulnerable.version %} + {{ next_non_vulnerable.version }} + {% else %} + None. + {% endif %} +
+ Latest non-vulnerable version + + {% if latest_non_vulnerable.version %} + {{ latest_non_vulnerable.version }} + {% else %} + None. + {% endif %} +
+ Risk score + + {{package.risk_score}} +
+
+ + {% endif %} + +
+ {% if affected_by_advisories_v2|length != 0 %} +
+ Vulnerabilities affecting this package ({{ affected_by_advisories_v2|length }}) +
+ + + + + + + + + + + + + {% for advisory in affected_by_advisories_v2 %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryFixed in package version
+ + {{advisory.primary_advisory.advisory_id }} + +
+ {% if advisory.identifiers|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.identifiers %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ {% if advisory.secondary_members|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary_members %} + + {{secondary.advisory.avid }}
+
+ {% endfor %} + {% endif %} +
+ {{advisory.primary_advisory.url}} + + {{advisory.primary_advisory.date_published}} + + {{ advisory.primary_advisory.summary }} + + {% with fixed=fixed_package_details|get_item:advisory.primary_advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} +
+ This package is not known to be subject of any advisories. +
+ {% elif affected_by_advisories_v2_url %} +
+ This package is subject to more than 100 advisories. Please refer to the following + URL for vulnerabilities affecting this package: Advisories +
+ {% else %} +
+ This package is not known to be subject of any advisories. +
+ {% endif %} +
+ +
+ {% if fixing_advisories_v2|length != 0 %} +
+ Vulnerabilities fixed by this package ({{ fixing_advisories_v2|length }}) +
+ + + + + + + + + + + + + {% for advisory in fixing_advisories_v2 %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryAliases
+ + {{advisory.primary_advisory.advisory_id }} + +
+ {% if advisory.secondary_members|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary_members %} + + {{secondary.advisory.avid }}
+
+ {% endfor %} + {% endif %} +
+ {{advisory.primary_advisory.url}} + + {{advisory.primary_advisory.date_published}} + + {{ advisory.primary_advisory.summary }} + + {% for alias in advisory.identifiers %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ This package is not known to fix any advisories. +
+ +
+ {% elif fixing_advisories_v2_url %} +
+ This package is known to fix more than 100 advisories. Please refer to the following + URL for vulnerabilities fixed by this package: Advisories +
+ {% else %} +
+ This package is not known to fix any advisories. +
+ {% endif %} +
+
+
+ + +
+
+
+
+ +{% endif %} +{% endblock %} diff --git a/vulnerabilities/templates/packages_v2.html b/vulnerabilities/templates/packages_v2.html index fe2b05abe..f114a7159 100644 --- a/vulnerabilities/templates/packages_v2.html +++ b/vulnerabilities/templates/packages_v2.html @@ -40,15 +40,15 @@ - Affected by vulnerabilities + data-tooltip="This is the status of the package. If it is vulnerable, it means that there are known vulnerabilities associated with this package."> + Vulnerable - Fixing vulnerabilities + data-tooltip="This is the risk score of the package based on its vulnerabilities."> + Risk Score @@ -61,8 +61,14 @@ href="{{ package.get_absolute_url }}?search={{ search }}" target="_self">{{ package.purl }} - {{ package.vulnerability_count }} - {{ package.patched_vulnerability_count }} + {{ package.is_vulnerable|yesno:"Yes,No" }} + + {% if package.risk_score is not None %} + {{ package.risk_score }} + {% else %} + {{ "" }} + {% endif %} + {% empty %} diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py deleted file mode 100644 index 5b7f0c186..000000000 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from unittest.mock import patch - -import pytest - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash import ( - ComputeAdvisoryContentHash, -) - -pytestmark = pytest.mark.django_db - - -@pytest.fixture -def advisory_factory(): - def _create(count, with_hash=False, start=0): - objs = [] - for i in range(start, start + count): - objs.append( - AdvisoryV2( - summary=f"summary {i}", - advisory_content_hash="existing_hash" if with_hash else None, - unique_content_id=f"unique_id_{i}", - advisory_id=f"ADV-{i}", - datasource_id="ds", - avid=f"ds/ADV-{i}", - url=f"https://example.com/ADV-{i}", - ) - ) - return AdvisoryV2.objects.bulk_create(objs) - - return _create - - -def run_pipeline(): - pipeline = ComputeAdvisoryContentHash() - pipeline.compute_advisory_content_hash() - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_updates_only_missing_hash(mock_compute, advisory_factory): - advisory_factory(3, with_hash=False, start=0) - advisory_factory(2, with_hash=True, start=100) - - mock_compute.return_value = "new_hash" - - run_pipeline() - - updated = AdvisoryV2.objects.filter(advisory_content_hash="new_hash").count() - untouched = AdvisoryV2.objects.filter(advisory_content_hash="existing_hash").count() - - assert updated == 3 - assert untouched == 2 - assert mock_compute.call_count == 3 - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_bulk_update_batches(mock_compute, advisory_factory): - advisory_factory(6000, with_hash=False) - - mock_compute.return_value = "batch_hash" - - run_pipeline() - - assert AdvisoryV2.objects.filter(advisory_content_hash="batch_hash").count() == 6000 - - assert mock_compute.call_count == 6000 - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_no_advisories(mock_compute): - run_pipeline() - - assert mock_compute.call_count == 0 diff --git a/vulnerabilities/tests/test_advisory_merge.py b/vulnerabilities/tests/test_advisory_merge.py new file mode 100644 index 000000000..ddcc3cadb --- /dev/null +++ b/vulnerabilities/tests/test_advisory_merge.py @@ -0,0 +1,192 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib + +import pytest + +from vulnerabilities.models import AdvisoryAlias +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageV2 +from vulnerabilities.utils import compute_advisory_content_hash +from vulnerabilities.utils import delete_and_save_advisory_set +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import get_merged_identifier_groups +from vulnerabilities.utils import merge_advisories +from vulnerabilities.utils import merge_and_save_grouped_advisories + + +@pytest.mark.django_db +class TestAdvisoryMerge: + def create_advisory(self, advisory_id, affected_versions, fixed_versions=None, precedence=None): + unique_content_id = hashlib.sha256(advisory_id.encode()).hexdigest() + + adv = AdvisoryV2.objects.create( + datasource_id="ghsa", + advisory_id=advisory_id, + avid=f"ghsa/{advisory_id}", + unique_content_id=unique_content_id, + url="https://example.com/advisory", + date_collected="2025-07-01T00:00:00Z", + precedence=precedence, + ) + + pkg = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + impact = ImpactedPackage.objects.create( + advisory=adv, + base_purl="pkg:pypi/sample", + ) + + # affected + for v in affected_versions: + p = PackageV2.objects.from_purl(f"pkg:pypi/sample@{v}") + impact.affecting_packages.add(p) + + # fixed + if fixed_versions: + for v in fixed_versions: + p = PackageV2.objects.from_purl(f"pkg:pypi/sample@{v}") + impact.fixed_by_packages.add(p) + + return adv + + def test_content_hash_same(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) + + h1 = compute_advisory_content_hash(adv1, package) + h2 = compute_advisory_content_hash(adv2, package) + + assert h1 == h2 + + def test_content_hash_different(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["3.0"]) + + assert compute_advisory_content_hash(adv1, package) != compute_advisory_content_hash( + adv2, package + ) + + def test_identifier_merging(self): + adv1 = self.create_advisory("A1", ["1.0"]) + adv2 = self.create_advisory("A2", ["1.0"]) + + alias = AdvisoryAlias.objects.create(alias="CVE-123") + + adv1.aliases.add(alias) + adv2.aliases.add(alias) + + groups = get_merged_identifier_groups([adv1, adv2]) + + assert len(groups) == 1 + identifiers, primary, secondary = groups[0] + + assert len(secondary) == 1 + assert primary in [adv1, adv2] + + def test_transitive_merge(self): + a1 = self.create_advisory("A1", ["1.0"]) + a2 = self.create_advisory("A2", ["1.0"]) + a3 = self.create_advisory("A3", ["1.0"]) + + alias_1 = AdvisoryAlias.objects.create(alias="CVE-1") + alias_2 = AdvisoryAlias.objects.create(alias="CVE-2") + + a1.aliases.add(alias_1) + a2.aliases.add(alias_1) + a2.aliases.add(alias_2) + a3.aliases.add(alias_2) + + groups = get_merged_identifier_groups([a1, a2, a3]) + + assert len(groups) == 1 + + def test_primary_selection_by_precedence(self): + a1 = self.create_advisory("A1", ["1.0"], precedence=1) + a2 = self.create_advisory("A2", ["1.0"], precedence=5) + + alias_1 = AdvisoryAlias.objects.create(alias="CVE-1") + + a1.aliases.add(alias_1) + a2.aliases.add(alias_1) + + groups = get_merged_identifier_groups([a1, a2]) + _, primary, _ = groups[0] + + assert primary == a2 + + def test_get_advisories_from_groups(self): + adv = self.create_advisory("GHSA-ABC-123", ["1.0"]) + adv.aliases.create(alias="CVE-999") + + groups = get_merged_identifier_groups([adv]) + result = get_advisories_from_groups(groups) + + assert result[0]["identifier"] == "GHSA-ABC-123" + assert len(result[0]["aliases"]) == 1 + + def test_delete_and_save_advisory_set(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"]) + adv2 = self.create_advisory("A2", ["1.0"]) + + adv1.aliases.create(alias="CVE-1") + + groups = [(set(adv1.aliases.all()), adv1, [adv2])] + + delete_and_save_advisory_set(groups, package, relation="affecting") + + assert AdvisorySet.objects.count() == 1 + assert AdvisorySetMember.objects.count() == 2 + + advisory_set = AdvisorySet.objects.first() + members = AdvisorySetMember.objects.filter(advisory_set=advisory_set) + + assert any(m.is_primary for m in members) + assert any(not m.is_primary for m in members) + + def test_merge_and_save_integration(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) + + alias = AdvisoryAlias.objects.create(alias="CVE-1") + + adv1.aliases.add(alias) + adv2.aliases.add(alias) + + result = merge_and_save_grouped_advisories( + package, + [adv1, adv2], + relation="test", + ) + + assert len(result) == 1 + assert AdvisorySet.objects.count() == 1 + assert AdvisorySetMember.objects.count() == 2 + + def test_merge_advisories_separates_different_content(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["3.0"]) + + groups = merge_advisories([adv1, adv2], package) + + assert len(groups) == 2 diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index 6968123c7..c4abe3b97 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -834,74 +834,3 @@ def test_filter_codefix_by_advisory_id_not_found(self): response = self.client.get(self.url, {"advisory_id": "nonexistent/ADVISORY-ID"}) assert response.status_code == status.HTTP_200_OK assert response.data["count"] == 0 - - -class AdvisoriesPackageV2Tests(APITestCase): - def setUp(self): - from vulnerabilities.models import ImpactedPackage - - self.advisory = AdvisoryV2.objects.create( - datasource_id="ghsa", - advisory_id="GHSA-1234", - avid="ghsa/GHSA-1234", - unique_content_id="f" * 64, - url="https://example.com/advisory", - date_collected="2025-07-01T00:00:00Z", - ) - - self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") - self.impact = ImpactedPackage.objects.create( - advisory=self.advisory, base_purl="pkg:pypi/sample" - ) - self.impact.affecting_packages.add(self.package) - - self.client = APIClient(enforce_csrf_checks=True) - - def test_list_with_purl_filter(self): - url = reverse("package-v3-list") - with self.assertNumQueries(31): - response = self.client.get(url, {"purl": "pkg:pypi/sample@1.0.0"}) - assert response.status_code == 200 - assert "packages" in response.data["results"] - assert "advisories_by_id" in response.data["results"] - assert self.advisory.avid in response.data["results"]["advisories_by_id"] - - def test_bulk_lookup(self): - url = reverse("package-v3-bulk-lookup") - with self.assertNumQueries(30): - response = self.client.post(url, {"purls": ["pkg:pypi/sample@1.0.0"]}, format="json") - assert response.status_code == 200 - assert "packages" in response.data - assert "advisories_by_id" in response.data - assert self.advisory.avid in response.data["advisories_by_id"] - - def test_bulk_search_plain(self): - url = reverse("package-v3-bulk-search") - payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": True, "purl_only": False} - with self.assertNumQueries(30): - response = self.client.post(url, payload, format="json") - assert response.status_code == 200 - assert "packages" in response.data - assert "advisories_by_id" in response.data - - def test_bulk_search_purl_only(self): - url = reverse("package-v3-bulk-search") - payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": False, "purl_only": True} - with self.assertNumQueries(17): - response = self.client.post(url, payload, format="json") - assert response.status_code == 200 - assert "pkg:pypi/sample@1.0.0" in response.data - - def test_lookup_single_package(self): - url = reverse("package-v3-lookup") - with self.assertNumQueries(23): - response = self.client.post(url, {"purl": "pkg:pypi/sample@1.0.0"}, format="json") - assert response.status_code == 200 - assert any(pkg["purl"] == "pkg:pypi/sample@1.0.0" for pkg in response.data) - - def test_get_all_vulnerable_purls(self): - url = reverse("package-v3-all") - with self.assertNumQueries(3): - response = self.client.get(url) - assert response.status_code == 200 - assert "pkg:pypi/sample@1.0.0" in response.data diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py new file mode 100644 index 000000000..fa8a08b33 --- /dev/null +++ b/vulnerabilities/tests/test_api_v3.py @@ -0,0 +1,233 @@ +from django.urls import reverse +from packageurl import PackageURL +from rest_framework import status +from rest_framework.test import APIClient +from rest_framework.test import APITestCase +from univers.version_range import PypiVersionRange + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipes.advisory import insert_advisory_v2 + + +class APIV3TestCase(APITestCase): + def setUp(self): + from vulnerabilities.models import ImpactedPackage + + self.advisory = AdvisoryV2.objects.create( + datasource_id="ghsa", + advisory_id="GHSA-1234", + avid="ghsa/GHSA-1234", + unique_content_id="f" * 64, + url="https://example.com/advisory", + date_collected="2025-07-01T00:00:00Z", + ) + + self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") + self.impact = ImpactedPackage.objects.create( + advisory=self.advisory, base_purl="pkg:pypi/sample" + ) + self.impact.affecting_packages.add(self.package) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_packages_post_without_details(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(4): + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0"], + "details": False, + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0], "pkg:pypi/sample@1.0.0") + + def test_packages_post_with_details(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(33): + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0"], + "details": True, + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + pkg = response.data["results"][0] + self.assertEqual(pkg["purl"], "pkg:pypi/sample@1.0.0") + + def test_advisories_post(self): + url = reverse("advisory-v3-list") + + with self.assertNumQueries(10): + response = self.client.post( + url, + data={"purls": ["pkg:pypi/sample@1.0.0"]}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + advisory = response.data["results"][0] + self.assertEqual(advisory["advisory_id"], "ghsa/GHSA-1234") + + def test_affected_by_advisories_list(self): + url = reverse("affected-by-advisories-list") + + with self.assertNumQueries(11): + response = self.client.get( + url, + {"purl": "pkg:pypi/sample@1.0.0"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["advisory_id"], "ghsa/GHSA-1234") + + def test_fixing_advisories_list_empty(self): + url = reverse("fixing-advisories-list") + + with self.assertNumQueries(3): + response = self.client.get( + url, + {"purl": "pkg:pypi/sample@1.0.0"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]), 0) + + def test_packages_pagination(self): + url = reverse("package-v3-list") + + response = self.client.post( + url, + data={"purls": []}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertIn("results", response.data) + self.assertIn("next", response.data) + + def test_packages_approximate(self): + url = reverse("package-v3-list") + + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0?foo=bar"], + "approximate": True, + "details": False, + }, + format="json", + ) + + self.assertEqual(response.status_code, 200) + self.assertGreaterEqual(len(response.data["results"]), 1) + self.assertIn("pkg:pypi/sample@1.0.0", response.data["results"]) + + +class APIV3TestCaseOnePackageMultipleAdvisories(APITestCase): + def setUp(self): + from vulnerabilities.importer import AdvisoryDataV2 + from vulnerabilities.importer import AffectedPackageV2 + + affected_packages = [] + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="pypi", name="sample"), + affected_version_range=PypiVersionRange.from_string("vers:pypi/=1.0.0"), + ) + ) + + for i in range(1, 102): + advisory = AdvisoryDataV2( + advisory_id=f"GHSA-1234{i}", + aliases=[f"CVE-2021-1234{i}"], + summary="Sample advisory", + affected_packages=affected_packages, + url="https://example.com/advisory", + original_advisory_text="Sample advisory text", + ) + + insert_advisory_v2(advisory, "ghsa_importer", print, 100) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_advisories_post(self): + url = reverse("advisory-v3-list") + + with self.assertNumQueries(10): + response = self.client.post( + url, + data={"purls": ["pkg:pypi/sample@1.0.0"]}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]), 10) + advisory = response.data["results"][0] + self.assertEqual(advisory["advisory_id"], "ghsa_importer/GHSA-12341") + + +class APIV3TestCaseOneAdvisoryMultiplePackages(APITestCase): + def setUp(self): + from vulnerabilities.importer import AdvisoryDataV2 + from vulnerabilities.importer import AffectedPackageV2 + + affected_packages = [] + for i in range(1, 102): + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="pypi", name=f"sample{i}"), + affected_version_range=PypiVersionRange.from_string("vers:pypi/=1.0.0"), + ) + ) + + advisory = AdvisoryDataV2( + advisory_id=f"GHSA-1234{i}", + aliases=[f"CVE-2021-1234{i}"], + summary="Sample advisory", + affected_packages=affected_packages, + url="https://example.com/advisory", + original_advisory_text="Sample advisory text", + ) + + insert_advisory_v2(advisory, "ghsa_importer", print, 100) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_get_all_vulnerable_purls(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(4): + response = self.client.post( + url, + data={ + "purls": [], + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 10) + self.assertIn("next", response.data) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index f90d42401..5f791d30b 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -41,6 +41,7 @@ from univers.version_range import VersionRange from aboutcode.hashid import build_vcid +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set logger = logging.getLogger(__name__) @@ -611,6 +612,7 @@ def normalize_text(text): def normalize_list(lst): """Sort a list to ensure consistent ordering.""" + lst = [x for x in lst if x] return sorted(lst) if lst else [] @@ -844,57 +846,152 @@ def compute_patch_checksum(patch_text: str): return hashlib.sha512(patch_text.encode("utf-8")).hexdigest() -def group_advisories_by_content(advisories): - grouped = {} +def merge_advisories(advisories, package): + """ + Merge advisories based on their content hash and identifiers. + """ - for advisory in advisories: - content_hash = ( - advisory.advisory_content_hash - if advisory.advisory_content_hash - else compute_advisory_content(advisory) - ) + advisories = list(advisories) - entry = grouped.setdefault( - content_hash, - {"primary": advisory, "secondary": set()}, - ) + content_hash_map = defaultdict(list) - primary = entry["primary"] + for adv in advisories: + content_hash = compute_advisory_content_hash(adv, package) + content_hash_map[content_hash].append(adv) - if advisory is primary: - continue + final_groups = [] - if advisory.precedence > primary.precedence: - entry["primary"] = advisory - entry["secondary"].add(primary) - else: - entry["secondary"].add(advisory) + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) - return grouped + return final_groups -def compute_advisory_content(advisory_data): - """ - Compute a unique content hash for an advisory by normalizing its data and hashing it. +def compute_advisory_content_hash(adv, package): + """Compute a content hash for an advisory based on its affected and fixed packages for a given package. + This is used to determine if two advisories are the same based on their content.""" + affected = [] + fixed = [] - :param advisory_data: An AdvisoryData object - :return: SHA-256 hash digest as content hash - """ - from vulnerabilities.models import AdvisoryV2 + version_less_purl = PackageURL( + type=package.type, + namespace=package.namespace, + name=package.name, + qualifiers=package.qualifiers, + subpath=package.subpath, + ) + + for impact in adv.impacted_packages.filter(base_purl=str(version_less_purl)): + affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) + fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) - if isinstance(advisory_data, AdvisoryV2): - advisory_data = advisory_data.to_advisory_data() normalized_data = { - "summary": normalize_text(advisory_data.summary), - "affected_packages": [ - pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg - ], - "severities": [sev.to_dict() for sev in normalize_list(advisory_data.severities) if sev], - "weaknesses": normalize_list(advisory_data.weaknesses), - "patches": [patch.to_dict() for patch in normalize_list(advisory_data.patches)], + "affected_packages": normalize_list(affected), + "fixed_packages": normalize_list(fixed), } normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - return content_hash + + +def get_merged_identifier_groups(advisories): + """ + Merge advisories based on their identifiers (advisory_id and aliases). + Example: If two advisories share ``advisory_id`` or share an alias, they will be merged together. + """ + + identifier_groups = defaultdict(set) + + advisories = list(advisories) + + for adv in advisories: + + identifier_groups[adv.advisory_id].add(adv) + + for alias in adv.aliases.values_list("alias", flat=True): + identifier_groups[alias].add(adv) + + groups = [set(advs) for advs in identifier_groups.values() if len(advs) > 1] + + merged = [] + + for group in groups: + group = set(group) + + i = 0 + while i < len(merged): + if group & merged[i]: + group |= merged[i] + merged.pop(i) + else: + i += 1 + + merged.append(group) + + all_grouped = set() + for g in merged: + all_grouped |= g + + for adv in advisories: + if adv not in all_grouped: + merged.append({adv}) + + final_groups = [] + + for group in merged: + identifiers = set() + for adv in group: + for alias in adv.aliases.all().order_by("alias"): + identifiers.add(alias) + + primary = max(group, key=lambda a: a.precedence if a.precedence is not None else -1) + + secondary = [a for a in group if a != primary] + + final_groups.append((identifiers, primary, secondary)) + + return final_groups + + +def get_advisories_from_groups(groups): + """ + Return a list of advisories from the merged groups of advisories. + """ + advisories = [] + for aliases, primary, _ in groups: + identifier = primary.advisory_id.split("/")[-1] + + filtered_aliases = [alias for alias in aliases if alias.alias != identifier] + + advisories.append( + {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} + ) + + return advisories + + +def merge_and_save_grouped_advisories(package, advisories, relation): + """ + Merge advisories based on their content and identifiers and save the merged advisories to the database. + """ + groups = merge_advisories(advisories, package) + delete_and_save_advisory_set(groups, package, relation) + advisories = get_advisories_from_groups(groups) + + return advisories + + +TYPES_WITH_MULTIPLE_IMPORTERS = [ + "pypi", + "maven", + "nuget", + "golang", + "npm", + "composer", + "hex", + "cargo", + "gem", + "conan", +] diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 860bde8eb..63d02c5b1 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging +from collections import defaultdict from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -15,8 +16,8 @@ from django.contrib.auth.views import LoginView from django.core.exceptions import ValidationError from django.core.mail import send_mail -from django.db.models import Count -from django.db.models import F +from django.db.models import Exists +from django.db.models import OuterRef from django.db.models import Prefetch from django.http.response import Http404 from django.shortcuts import get_object_or_404 @@ -36,17 +37,21 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import PipelineSchedulePackageForm from vulnerabilities.forms import VulnerabilitySearchForm -from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import group_advisories_by_content +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import merge_and_save_grouped_advisories from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env -PAGE_SIZE = 20 +PAGE_SIZE = 10 class PackageSearch(ListView): @@ -77,34 +82,6 @@ def get_queryset(self, query=None): ) -class PackageSearchV2(ListView): - model = models.PackageV2 - template_name = "packages_v2.html" - ordering = ["type", "namespace", "name", "version"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["package_search_form"] = PackageSearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - """ - Return a Package queryset for the ``query``. - Make a best effort approach to find matching packages either based - on exact purl, partial purl or just name and namespace. - """ - query = query or self.request.GET.get("search") or "" - return ( - self.model.objects.search(query) - .with_vulnerability_counts() - .prefetch_related() - .order_by("package_url") - ) - - class VulnerabilitySearch(ListView): model = models.Vulnerability template_name = "vulnerabilities.html" @@ -123,24 +100,6 @@ def get_queryset(self, query=None): return self.model.objects.search(query=query).with_package_counts() -class AdvisorySearch(ListView): - model = models.AdvisoryV2 - template_name = "vulnerabilities.html" - ordering = ["advisory_id"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["advisory_search_form"] = VulnerabilitySearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - query = query or self.request.GET.get("search") or "" - return self.model.objects.search(query=query).with_package_counts() - - class PackageDetails(DetailView): model = models.Package template_name = "package_details.html" @@ -182,6 +141,64 @@ def get_object(self, queryset=None): return package +class PackageSearchV2(ListView): + model = models.PackageV2 + template_name = "packages_v2.html" + ordering = ["type", "namespace", "name", "version"] + paginate_by = PAGE_SIZE + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + request_query = self.request.GET + context["package_search_form"] = PackageSearchForm(request_query) + context["search"] = request_query.get("search") + return context + + def get_queryset(self, query=None): + """ + Return a Package queryset for the ``query``. + Make a best effort approach to find matching packages either based + on exact purl, partial purl or just name and namespace. + """ + query = query or self.request.GET.get("search") or "" + return self.model.objects.search(query).prefetch_related().with_is_vulnerable() + + +class AffectedByAdvisoriesListView(ListView): + model = models.AdvisoryV2 + template_name = "affected_by_advisories.html" + paginate_by = PAGE_SIZE + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + purl = self.kwargs.get("purl") + package = models.PackageV2.objects.for_purl(purl).first() + context["fixed_package_details"] = get_fixed_package_details(package) + return context + + def get_queryset(self): + purl = self.kwargs.get("purl") + return ( + models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl) + .only("advisory_id", "summary", "url", "date_published") + .prefetch_related("aliases") + ) + + +class FixingAdvisoriesListView(ListView): + model = models.AdvisoryV2 + template_name = "fixing_advisories.html" + paginate_by = PAGE_SIZE + + def get_queryset(self): + purl = self.kwargs.get("purl") + return ( + models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl) + .only("advisory_id", "summary", "url", "date_published") + .prefetch_related("aliases") + ) + + class PackageV2Details(DetailView): model = models.PackageV2 template_name = "package_details_v2.html" @@ -191,103 +208,228 @@ class PackageV2Details(DetailView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) package = self.object + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() - ( - fixed_pkg_details, - affected_by_advisories, - fixing_advisories, - ) = self.get_fixed_package_details(package) + context["package"] = package + context["next_non_vulnerable"] = next_non_vulnerable + context["latest_non_vulnerable"] = latest_non_vulnerable + context["package_search_form"] = PackageSearchForm(self.request.GET) + + if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ) - affected_avid_by_hash = {} - fixing_avid_by_hash = {} + context["grouped"] = False - affected_avid_by_hash = group_advisories_by_content(affected_by_advisories) - fixing_avid_by_hash = group_advisories_by_content(fixing_advisories) + affected_by_advisories_url = None + fixing_advisories_url = None - affecting_advs = [] + affected_by_advisories_qs_ids = affecting_advisories.only("id") + fixing_advisories_qs_ids = fixed_by_advisories.only("id") - for hash in affected_avid_by_hash: - affecting_advs.append(affected_avid_by_hash[hash]) + affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) + if len(affected_by_advisories) > 100: + affected_by_advisories_url = reverse_lazy( + "affected_by_advisories_v2", kwargs={"purl": package.package_url} + ) + context["affected_by_advisories_v2_url"] = affected_by_advisories_url + + else: + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2"] = affecting_advisories + context["affected_by_advisories_v2_url"] = None + + fixing_advisories = list(fixing_advisories_qs_ids[:101]) + if len(fixing_advisories) > 100: + fixing_advisories_url = reverse_lazy( + "fixing_advisories_v2", kwargs={"purl": package.package_url} + ) + context["fixing_advisories_v2_url"] = fixing_advisories_url + context["fixing_advisories_v2"] = [] - fixing_advs = [] + else: + context["fixing_advisories_v2"] = fixed_by_advisories - for hash in fixing_avid_by_hash: - fixing_advs.append(fixing_avid_by_hash[hash]) + return context - context["package"] = package - context["next_non_vulnerable"] = next_non_vulnerable - context["latest_non_vulnerable"] = latest_non_vulnerable - context["affected_by_advisories_v2"] = affecting_advs - context["fixing_advisories_v2"] = fixing_advs + is_grouped = models.AdvisorySet.objects.filter(package=package).exists() - context["package_search_form"] = PackageSearchForm(self.request.GET) - context["fixed_package_details"] = fixed_pkg_details + if is_grouped: + context["grouped"] = True + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details - return context + affected_by_advisories_qs = models.AdvisorySet.objects.filter( + package=package, relation_type="affecting" + ).select_related("primary_advisory") - def get_fixed_package_details(self, package): - affected_impacts = package.affected_in_impacts.select_related("advisory").prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=( - models.PackageV2.objects.annotate(affected_count=Count("affected_in_impacts")) - ), - ) - ) + fixing_advisories_qs = models.AdvisorySet.objects.filter( + package=package, relation_type="fixing" + ).select_related("primary_advisory") - fixed_impacts = package.fixed_in_impacts.select_related("advisory") + affected_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") + for adv in affected_by_advisories_qs + ] + fixing_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ] - affected_avids = {impact.advisory.avid for impact in affected_impacts if impact.advisory_id} + affected_advisories = get_advisories_from_groups(affected_groups) + fixing_advisories = get_advisories_from_groups(fixing_groups) - fixed_avids = {impact.advisory.avid for impact in fixed_impacts if impact.advisory_id} + context["affected_by_advisories_v2"] = affected_advisories + context["fixing_advisories_v2"] = fixing_advisories - all_avids = affected_avids | fixed_avids + return context - advisories = models.AdvisoryV2.objects.latest_for_avids(all_avids) - advisory_by_avid = {adv.avid: adv for adv in advisories} + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ) - fixed_pkg_details = {} + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ) + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + context["grouped"] = True + + affecting_advisories = affecting_advisories.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) - for impact in affected_impacts: - advisory = advisory_by_avid.get(impact.advisory.avid) - if not advisory: - continue + affected_by_advisories = merge_and_save_grouped_advisories( + package, affecting_advisories, "affecting" + ) - fixed_pkg_details.setdefault(impact.advisory.avid, []).extend( - { - "pkg": pkg, - "affected_count": pkg.affected_count, - } - for pkg in impact.fixed_by_packages.all() + fixed_by_advisories = fixed_by_advisories.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) - affected_by_advisories = { - advisory_by_avid[avid] for avid in affected_avids if avid in advisory_by_avid - } + fixing_advisories = merge_and_save_grouped_advisories( + package, fixed_by_advisories, "fixing" + ) - fixing_advisories = { - advisory_by_avid[avid] for avid in fixed_avids if avid in advisory_by_avid - } + context["affected_by_advisories_v2"] = affected_by_advisories + context["fixing_advisories_v2"] = fixing_advisories + return context - return fixed_pkg_details, affected_by_advisories, fixing_advisories + def get_object(self, queryset=None): + if queryset is None: + queryset = self.get_queryset() - def get_queryset(self): - return ( - super() - .get_queryset() + purl = self.kwargs.get(self.slug_url_kwarg) + if purl: + queryset = queryset.for_purl(purl) + else: + cls = self.__class__.__name__ + raise AttributeError( + f"Package details view {cls} must be called with a purl, " f"but got: {purl!r}" + ) + + try: + package = queryset.get() + except queryset.model.DoesNotExist: + raise Http404(f"No Package found for purl: {purl}") + return package + + +class PackageV3Details(DetailView): + model = models.PackageV2 + template_name = "package_details_v3.html" + slug_url_kwarg = "purl" + slug_field = "purl" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + package = self.object + + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() + + context["package"] = package + context["next_non_vulnerable"] = next_non_vulnerable + context["latest_non_vulnerable"] = latest_non_vulnerable + context["package_search_form"] = PackageSearchForm(self.request.GET) + + affected_by_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") .prefetch_related( Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + fixing_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) ) ) + print(affected_by_advisories_qs) + print(fixing_advisories_qs) + + affected_by_advisories_url = None + fixing_advisories_url = None + + affected_by_advisories_qs_ids = affected_by_advisories_qs.only("id") + fixing_advisories_qs_ids = fixing_advisories_qs.only("id") + + # affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) + # if len(affected_by_advisories) > 100: + # affected_by_advisories_url = reverse_lazy( + # "affected_by_advisories_v2", kwargs={"purl": package.package_url} + # ) + # context["affected_by_advisories_v2_url"] = affected_by_advisories_url + # context["affected_by_advisories_v2"] = [] + # context["fixed_package_details"] = {} + + # else: + fixed_pkg_details = get_fixed_package_details(package) + + context["affected_by_advisories_v2"] = affected_by_advisories_qs + context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2_url"] = None + + # fixing_advisories = list(fixing_advisories_qs_ids[:101]) + # if len(fixing_advisories) > 100: + # fixing_advisories_url = reverse_lazy( + # "fixing_advisories_v2", kwargs={"purl": package.package_url} + # ) + # context["fixing_advisories_v2_url"] = fixing_advisories_url + # context["fixing_advisories_v2"] = [] + + # else: + context["fixing_advisories_v2"] = fixing_advisories_qs + context["fixing_advisories_v2_url"] = None + + return context + def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() @@ -308,6 +450,43 @@ def get_object(self, queryset=None): return package +def get_fixed_package_details(package): + rows = package.affected_in_impacts.values_list( + "advisory__avid", + "fixed_by_packages", + ) + + pkg_ids = {pkg_id for _, pkg_id in rows if pkg_id} + + pkg_map = { + p.id: p + for p in models.PackageV2.objects.filter(id__in=pkg_ids, is_ghost=False).annotate( + is_vulnerable=Exists( + models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) + ) + ) + } + + fixed_pkg_details = defaultdict(list) + + for avid, pkg_id in rows: + if not pkg_id: + continue + + pkg = pkg_map.get(pkg_id) + if not pkg: + continue + + fixed_pkg_details[avid].append( + { + "pkg": pkg, + "is_vulnerable": pkg.is_vulnerable, + } + ) + + return fixed_pkg_details + + class VulnerabilityDetails(DetailView): model = models.Vulnerability template_name = "vulnerability_details.html" diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index ae6638b76..435cb8953 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -54,6 +54,9 @@ # WARNING: Set this to False in production STAGING = env.bool("STAGING", default=True) +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +USE_X_FORWARDED_HOST = True + EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" EMAIL_HOST = env.str("EMAIL_HOST", default="") EMAIL_USE_TLS = True diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 49948a3b9..44cacd9b0 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -23,19 +23,25 @@ from vulnerabilities.api_v2 import CodeFixV2ViewSet from vulnerabilities.api_v2 import CodeFixViewSet from vulnerabilities.api_v2 import PackageV2ViewSet -from vulnerabilities.api_v2 import PackageV3ViewSet from vulnerabilities.api_v2 import PipelineScheduleV2ViewSet from vulnerabilities.api_v2 import VulnerabilityV2ViewSet +from vulnerabilities.api_v3 import AdvisoryV3ViewSet +from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet +from vulnerabilities.api_v3 import FixingAdvisoriesViewSet +from vulnerabilities.api_v3 import PackageV3ViewSet from vulnerabilities.views import AdminLoginView from vulnerabilities.views import AdvisoryDetails from vulnerabilities.views import AdvisoryPackagesDetails +from vulnerabilities.views import AffectedByAdvisoriesListView from vulnerabilities.views import ApiUserCreateView +from vulnerabilities.views import FixingAdvisoriesListView from vulnerabilities.views import HomePage from vulnerabilities.views import HomePageV2 from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch from vulnerabilities.views import PackageSearchV2 from vulnerabilities.views import PackageV2Details +from vulnerabilities.views import PackageV3Details from vulnerabilities.views import PipelineRunDetailView from vulnerabilities.views import PipelineRunListView from vulnerabilities.views import PipelineScheduleListView @@ -70,6 +76,11 @@ def __init__(self, *args, **kwargs): api_v3_router = OptionalSlashRouter() api_v3_router.register("packages", PackageV3ViewSet, basename="package-v3") +api_v3_router.register("advisories", AdvisoryV3ViewSet, basename="advisory-v3") +api_v3_router.register( + "affected-by-advisories", AffectedByAdvisoriesViewSet, basename="affected-by-advisories" +) +api_v3_router.register("fixing-advisories", FixingAdvisoriesViewSet, basename="fixing-advisories") urlpatterns = [ path("admin/login/", AdminLoginView.as_view(), name="admin-login"), @@ -134,6 +145,16 @@ def __init__(self, *args, **kwargs): PackageV2Details.as_view(), name="package_details_v2", ), + re_path( + r"^fixing-advisories/v2/(?Ppkg:.+)$", + FixingAdvisoriesListView.as_view(), + name="fixing_advisories_v2", + ), + re_path( + r"^affected-by-advisories/v2/(?Ppkg:.+)$", + AffectedByAdvisoriesListView.as_view(), + name="affected_by_advisories_v2", + ), path( "vulnerabilities/search/", VulnerabilitySearch.as_view(), @@ -174,10 +195,10 @@ def __init__(self, *args, **kwargs): TemplateView.as_view(template_name="tos.html"), name="api_tos", ), - path( - "admin/", - admin.site.urls, - ), + # path( + # "admin/", + # admin.site.urls, + # ), ] if DEBUG: